001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.io.compress;
020
021import java.io.IOException;
022import java.io.InputStream;
023import java.io.OutputStream;
024
025import org.apache.hadoop.conf.Configurable;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.io.compress.snappy.SnappyCompressor;
028import org.apache.hadoop.io.compress.snappy.SnappyDecompressor;
029import org.apache.hadoop.io.compress.snappy.SnappyDecompressor.SnappyDirectDecompressor;
030import org.apache.hadoop.fs.CommonConfigurationKeys;
031import org.apache.hadoop.util.NativeCodeLoader;
032
033/**
034 * This class creates snappy compressors/decompressors.
035 */
036public class SnappyCodec implements Configurable, CompressionCodec, DirectDecompressionCodec {
037  Configuration conf;
038
039  /**
040   * Set the configuration to be used by this object.
041   *
042   * @param conf the configuration object.
043   */
044  @Override
045  public void setConf(Configuration conf) {
046    this.conf = conf;
047  }
048
049  /**
050   * Return the configuration used by this object.
051   *
052   * @return the configuration object used by this objec.
053   */
054  @Override
055  public Configuration getConf() {
056    return conf;
057  }
058
059  /**
060   * Are the native snappy libraries loaded & initialized?
061   */
062  public static void checkNativeCodeLoaded() {
063      if (!NativeCodeLoader.buildSupportsSnappy()) {
064        throw new RuntimeException("native snappy library not available: " +
065            "this version of libhadoop was built without " +
066            "snappy support.");
067      }
068      if (!SnappyCompressor.isNativeCodeLoaded()) {
069        throw new RuntimeException("native snappy library not available: " +
070            "SnappyCompressor has not been loaded.");
071      }
072      if (!SnappyDecompressor.isNativeCodeLoaded()) {
073        throw new RuntimeException("native snappy library not available: " +
074            "SnappyDecompressor has not been loaded.");
075      }
076  }
077  
078  public static boolean isNativeCodeLoaded() {
079    return SnappyCompressor.isNativeCodeLoaded() && 
080        SnappyDecompressor.isNativeCodeLoaded();
081  }
082
083  public static String getLibraryName() {
084    return SnappyCompressor.getLibraryName();
085  }
086
087  /**
088   * Create a {@link CompressionOutputStream} that will write to the given
089   * {@link OutputStream}.
090   *
091   * @param out the location for the final output stream
092   * @return a stream the user can write uncompressed data to have it compressed
093   * @throws IOException
094   */
095  @Override
096  public CompressionOutputStream createOutputStream(OutputStream out)
097      throws IOException {
098    return CompressionCodec.Util.
099        createOutputStreamWithCodecPool(this, conf, out);
100  }
101
102  /**
103   * Create a {@link CompressionOutputStream} that will write to the given
104   * {@link OutputStream} with the given {@link Compressor}.
105   *
106   * @param out        the location for the final output stream
107   * @param compressor compressor to use
108   * @return a stream the user can write uncompressed data to have it compressed
109   * @throws IOException
110   */
111  @Override
112  public CompressionOutputStream createOutputStream(OutputStream out,
113                                                    Compressor compressor)
114      throws IOException {
115    checkNativeCodeLoaded();
116    int bufferSize = conf.getInt(
117        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
118        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
119
120    int compressionOverhead = (bufferSize / 6) + 32;
121
122    return new BlockCompressorStream(out, compressor, bufferSize,
123        compressionOverhead);
124  }
125
126  /**
127   * Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
128   *
129   * @return the type of compressor needed by this codec.
130   */
131  @Override
132  public Class<? extends Compressor> getCompressorType() {
133    checkNativeCodeLoaded();
134    return SnappyCompressor.class;
135  }
136
137  /**
138   * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
139   *
140   * @return a new compressor for use by this codec
141   */
142  @Override
143  public Compressor createCompressor() {
144    checkNativeCodeLoaded();
145    int bufferSize = conf.getInt(
146        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
147        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
148    return new SnappyCompressor(bufferSize);
149  }
150
151  /**
152   * Create a {@link CompressionInputStream} that will read from the given
153   * input stream.
154   *
155   * @param in the stream to read compressed bytes from
156   * @return a stream to read uncompressed bytes from
157   * @throws IOException
158   */
159  @Override
160  public CompressionInputStream createInputStream(InputStream in)
161      throws IOException {
162    return CompressionCodec.Util.
163        createInputStreamWithCodecPool(this, conf, in);
164  }
165
166  /**
167   * Create a {@link CompressionInputStream} that will read from the given
168   * {@link InputStream} with the given {@link Decompressor}.
169   *
170   * @param in           the stream to read compressed bytes from
171   * @param decompressor decompressor to use
172   * @return a stream to read uncompressed bytes from
173   * @throws IOException
174   */
175  @Override
176  public CompressionInputStream createInputStream(InputStream in,
177                                                  Decompressor decompressor)
178      throws IOException {
179    checkNativeCodeLoaded();
180    return new BlockDecompressorStream(in, decompressor, conf.getInt(
181        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
182        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT));
183  }
184
185  /**
186   * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
187   *
188   * @return the type of decompressor needed by this codec.
189   */
190  @Override
191  public Class<? extends Decompressor> getDecompressorType() {
192    checkNativeCodeLoaded();
193    return SnappyDecompressor.class;
194  }
195
196  /**
197   * Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
198   *
199   * @return a new decompressor for use by this codec
200   */
201  @Override
202  public Decompressor createDecompressor() {
203    checkNativeCodeLoaded();
204    int bufferSize = conf.getInt(
205        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
206        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
207    return new SnappyDecompressor(bufferSize);
208  }
209  
210  /**
211   * {@inheritDoc}
212   */
213  @Override
214  public DirectDecompressor createDirectDecompressor() {
215    return isNativeCodeLoaded() ? new SnappyDirectDecompressor() : null;
216  }
217
218  /**
219   * Get the default filename extension for this kind of compression.
220   *
221   * @return <code>.snappy</code>.
222   */
223  @Override
224  public String getDefaultExtension() {
225    return ".snappy";
226  }
227}