001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.io.compress;
020
021import java.io.IOException;
022import java.io.InputStream;
023import java.io.OutputStream;
024
025import org.apache.hadoop.conf.Configurable;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.io.compress.lz4.Lz4Compressor;
028import org.apache.hadoop.io.compress.lz4.Lz4Decompressor;
029import org.apache.hadoop.fs.CommonConfigurationKeys;
030import org.apache.hadoop.util.NativeCodeLoader;
031
032/**
033 * This class creates lz4 compressors/decompressors.
034 */
035public class Lz4Codec implements Configurable, CompressionCodec {
036
037  static {
038    NativeCodeLoader.isNativeCodeLoaded();
039  }
040
041  Configuration conf;
042
043  /**
044   * Set the configuration to be used by this object.
045   *
046   * @param conf the configuration object.
047   */
048  @Override
049  public void setConf(Configuration conf) {
050    this.conf = conf;
051  }
052
053  /**
054   * Return the configuration used by this object.
055   *
056   * @return the configuration object used by this objec.
057   */
058  @Override
059  public Configuration getConf() {
060    return conf;
061  }
062
063  /**
064   * Are the native lz4 libraries loaded & initialized?
065   *
066   * @return true if loaded & initialized, otherwise false
067   */
068  public static boolean isNativeCodeLoaded() {
069    return NativeCodeLoader.isNativeCodeLoaded();
070  }
071
072  /**
073   * Create a {@link CompressionOutputStream} that will write to the given
074   * {@link OutputStream}.
075   *
076   * @param out the location for the final output stream
077   * @return a stream the user can write uncompressed data to have it compressed
078   * @throws IOException
079   */
080  @Override
081  public CompressionOutputStream createOutputStream(OutputStream out)
082      throws IOException {
083    return createOutputStream(out, createCompressor());
084  }
085
086  /**
087   * Create a {@link CompressionOutputStream} that will write to the given
088   * {@link OutputStream} with the given {@link Compressor}.
089   *
090   * @param out        the location for the final output stream
091   * @param compressor compressor to use
092   * @return a stream the user can write uncompressed data to have it compressed
093   * @throws IOException
094   */
095  @Override
096  public CompressionOutputStream createOutputStream(OutputStream out,
097                                                    Compressor compressor)
098      throws IOException {
099    if (!isNativeCodeLoaded()) {
100      throw new RuntimeException("native lz4 library not available");
101    }
102    int bufferSize = conf.getInt(
103        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
104        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
105
106    int compressionOverhead = Math.max((int)(bufferSize * 0.01), 10);
107
108    return new BlockCompressorStream(out, compressor, bufferSize,
109        compressionOverhead);
110  }
111
112  /**
113   * Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
114   *
115   * @return the type of compressor needed by this codec.
116   */
117  @Override
118  public Class<? extends Compressor> getCompressorType() {
119    if (!isNativeCodeLoaded()) {
120      throw new RuntimeException("native lz4 library not available");
121    }
122
123    return Lz4Compressor.class;
124  }
125
126  /**
127   * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
128   *
129   * @return a new compressor for use by this codec
130   */
131  @Override
132  public Compressor createCompressor() {
133    if (!isNativeCodeLoaded()) {
134      throw new RuntimeException("native lz4 library not available");
135    }
136    int bufferSize = conf.getInt(
137        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
138        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
139    return new Lz4Compressor(bufferSize);
140  }
141
142  /**
143   * Create a {@link CompressionInputStream} that will read from the given
144   * input stream.
145   *
146   * @param in the stream to read compressed bytes from
147   * @return a stream to read uncompressed bytes from
148   * @throws IOException
149   */
150  @Override
151  public CompressionInputStream createInputStream(InputStream in)
152      throws IOException {
153    return createInputStream(in, createDecompressor());
154  }
155
156  /**
157   * Create a {@link CompressionInputStream} that will read from the given
158   * {@link InputStream} with the given {@link Decompressor}.
159   *
160   * @param in           the stream to read compressed bytes from
161   * @param decompressor decompressor to use
162   * @return a stream to read uncompressed bytes from
163   * @throws IOException
164   */
165  @Override
166  public CompressionInputStream createInputStream(InputStream in,
167                                                  Decompressor decompressor)
168      throws IOException {
169    if (!isNativeCodeLoaded()) {
170      throw new RuntimeException("native lz4 library not available");
171    }
172
173    return new BlockDecompressorStream(in, decompressor, conf.getInt(
174        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
175        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT));
176  }
177
178  /**
179   * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
180   *
181   * @return the type of decompressor needed by this codec.
182   */
183  @Override
184  public Class<? extends Decompressor> getDecompressorType() {
185    if (!isNativeCodeLoaded()) {
186      throw new RuntimeException("native lz4 library not available");
187    }
188
189    return Lz4Decompressor.class;
190  }
191
192  /**
193   * Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
194   *
195   * @return a new decompressor for use by this codec
196   */
197  @Override
198  public Decompressor createDecompressor() {
199    if (!isNativeCodeLoaded()) {
200      throw new RuntimeException("native lz4 library not available");
201    }
202    int bufferSize = conf.getInt(
203        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
204        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
205    return new Lz4Decompressor(bufferSize);
206  }
207
208  /**
209   * Get the default filename extension for this kind of compression.
210   *
211   * @return <code>.lz4</code>.
212   */
213  @Override
214  public String getDefaultExtension() {
215    return ".lz4";
216  }
217}