001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.io.compress;
020
021import java.io.IOException;
022import java.io.InputStream;
023import java.io.OutputStream;
024
025import org.apache.hadoop.conf.Configurable;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.io.compress.lz4.Lz4Compressor;
028import org.apache.hadoop.io.compress.lz4.Lz4Decompressor;
029import org.apache.hadoop.fs.CommonConfigurationKeys;
030import org.apache.hadoop.util.NativeCodeLoader;
031
032/**
033 * This class creates lz4 compressors/decompressors.
034 */
035public class Lz4Codec implements Configurable, CompressionCodec {
036
037  static {
038    NativeCodeLoader.isNativeCodeLoaded();
039  }
040
041  Configuration conf;
042
043  /**
044   * Set the configuration to be used by this object.
045   *
046   * @param conf the configuration object.
047   */
048  @Override
049  public void setConf(Configuration conf) {
050    this.conf = conf;
051  }
052
053  /**
054   * Return the configuration used by this object.
055   *
056   * @return the configuration object used by this objec.
057   */
058  @Override
059  public Configuration getConf() {
060    return conf;
061  }
062
063  /**
064   * Are the native lz4 libraries loaded & initialized?
065   *
066   * @return true if loaded & initialized, otherwise false
067   */
068  public static boolean isNativeCodeLoaded() {
069    return NativeCodeLoader.isNativeCodeLoaded();
070  }
071
072  public static String getLibraryName() {
073    return Lz4Compressor.getLibraryName();
074  }
075
076  /**
077   * Create a {@link CompressionOutputStream} that will write to the given
078   * {@link OutputStream}.
079   *
080   * @param out the location for the final output stream
081   * @return a stream the user can write uncompressed data to have it compressed
082   * @throws IOException
083   */
084  @Override
085  public CompressionOutputStream createOutputStream(OutputStream out)
086      throws IOException {
087    return createOutputStream(out, createCompressor());
088  }
089
090  /**
091   * Create a {@link CompressionOutputStream} that will write to the given
092   * {@link OutputStream} with the given {@link Compressor}.
093   *
094   * @param out        the location for the final output stream
095   * @param compressor compressor to use
096   * @return a stream the user can write uncompressed data to have it compressed
097   * @throws IOException
098   */
099  @Override
100  public CompressionOutputStream createOutputStream(OutputStream out,
101                                                    Compressor compressor)
102      throws IOException {
103    if (!isNativeCodeLoaded()) {
104      throw new RuntimeException("native lz4 library not available");
105    }
106    int bufferSize = conf.getInt(
107        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
108        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
109
110    int compressionOverhead = bufferSize/255 + 16;
111
112    return new BlockCompressorStream(out, compressor, bufferSize,
113        compressionOverhead);
114  }
115
116  /**
117   * Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
118   *
119   * @return the type of compressor needed by this codec.
120   */
121  @Override
122  public Class<? extends Compressor> getCompressorType() {
123    if (!isNativeCodeLoaded()) {
124      throw new RuntimeException("native lz4 library not available");
125    }
126
127    return Lz4Compressor.class;
128  }
129
130  /**
131   * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
132   *
133   * @return a new compressor for use by this codec
134   */
135  @Override
136  public Compressor createCompressor() {
137    if (!isNativeCodeLoaded()) {
138      throw new RuntimeException("native lz4 library not available");
139    }
140    int bufferSize = conf.getInt(
141        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
142        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
143    boolean useLz4HC = conf.getBoolean(
144        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_USELZ4HC_KEY,
145        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_USELZ4HC_DEFAULT);
146    return new Lz4Compressor(bufferSize, useLz4HC);
147  }
148
149  /**
150   * Create a {@link CompressionInputStream} that will read from the given
151   * input stream.
152   *
153   * @param in the stream to read compressed bytes from
154   * @return a stream to read uncompressed bytes from
155   * @throws IOException
156   */
157  @Override
158  public CompressionInputStream createInputStream(InputStream in)
159      throws IOException {
160    return createInputStream(in, createDecompressor());
161  }
162
163  /**
164   * Create a {@link CompressionInputStream} that will read from the given
165   * {@link InputStream} with the given {@link Decompressor}.
166   *
167   * @param in           the stream to read compressed bytes from
168   * @param decompressor decompressor to use
169   * @return a stream to read uncompressed bytes from
170   * @throws IOException
171   */
172  @Override
173  public CompressionInputStream createInputStream(InputStream in,
174                                                  Decompressor decompressor)
175      throws IOException {
176    if (!isNativeCodeLoaded()) {
177      throw new RuntimeException("native lz4 library not available");
178    }
179
180    return new BlockDecompressorStream(in, decompressor, conf.getInt(
181        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
182        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT));
183  }
184
185  /**
186   * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
187   *
188   * @return the type of decompressor needed by this codec.
189   */
190  @Override
191  public Class<? extends Decompressor> getDecompressorType() {
192    if (!isNativeCodeLoaded()) {
193      throw new RuntimeException("native lz4 library not available");
194    }
195
196    return Lz4Decompressor.class;
197  }
198
199  /**
200   * Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
201   *
202   * @return a new decompressor for use by this codec
203   */
204  @Override
205  public Decompressor createDecompressor() {
206    if (!isNativeCodeLoaded()) {
207      throw new RuntimeException("native lz4 library not available");
208    }
209    int bufferSize = conf.getInt(
210        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
211        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
212    return new Lz4Decompressor(bufferSize);
213  }
214
215  /**
216   * Get the default filename extension for this kind of compression.
217   *
218   * @return <code>.lz4</code>.
219   */
220  @Override
221  public String getDefaultExtension() {
222    return ".lz4";
223  }
224}