001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.io.compress;
020
021import java.io.IOException;
022import java.io.InputStream;
023import java.io.OutputStream;
024
025import org.apache.hadoop.conf.Configurable;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.io.compress.snappy.SnappyCompressor;
028import org.apache.hadoop.io.compress.snappy.SnappyDecompressor;
029import org.apache.hadoop.io.compress.snappy.SnappyDecompressor.SnappyDirectDecompressor;
030import org.apache.hadoop.fs.CommonConfigurationKeys;
031import org.apache.hadoop.util.NativeCodeLoader;
032
033/**
034 * This class creates snappy compressors/decompressors.
035 */
036public class SnappyCodec implements Configurable, CompressionCodec, DirectDecompressionCodec {
037  Configuration conf;
038
039  /**
040   * Set the configuration to be used by this object.
041   *
042   * @param conf the configuration object.
043   */
044  @Override
045  public void setConf(Configuration conf) {
046    this.conf = conf;
047  }
048
049  /**
050   * Return the configuration used by this object.
051   *
052   * @return the configuration object used by this objec.
053   */
054  @Override
055  public Configuration getConf() {
056    return conf;
057  }
058
059  /**
060   * Are the native snappy libraries loaded & initialized?
061   */
062  public static void checkNativeCodeLoaded() {
063      if (!NativeCodeLoader.isNativeCodeLoaded() ||
064          !NativeCodeLoader.buildSupportsSnappy()) {
065        throw new RuntimeException("native snappy library not available: " +
066            "this version of libhadoop was built without " +
067            "snappy support.");
068      }
069      if (!SnappyCompressor.isNativeCodeLoaded()) {
070        throw new RuntimeException("native snappy library not available: " +
071            "SnappyCompressor has not been loaded.");
072      }
073      if (!SnappyDecompressor.isNativeCodeLoaded()) {
074        throw new RuntimeException("native snappy library not available: " +
075            "SnappyDecompressor has not been loaded.");
076      }
077  }
078  
079  public static boolean isNativeCodeLoaded() {
080    return SnappyCompressor.isNativeCodeLoaded() && 
081        SnappyDecompressor.isNativeCodeLoaded();
082  }
083
084  public static String getLibraryName() {
085    return SnappyCompressor.getLibraryName();
086  }
087
088  /**
089   * Create a {@link CompressionOutputStream} that will write to the given
090   * {@link OutputStream}.
091   *
092   * @param out the location for the final output stream
093   * @return a stream the user can write uncompressed data to have it compressed
094   * @throws IOException
095   */
096  @Override
097  public CompressionOutputStream createOutputStream(OutputStream out)
098      throws IOException {
099    return CompressionCodec.Util.
100        createOutputStreamWithCodecPool(this, conf, out);
101  }
102
103  /**
104   * Create a {@link CompressionOutputStream} that will write to the given
105   * {@link OutputStream} with the given {@link Compressor}.
106   *
107   * @param out        the location for the final output stream
108   * @param compressor compressor to use
109   * @return a stream the user can write uncompressed data to have it compressed
110   * @throws IOException
111   */
112  @Override
113  public CompressionOutputStream createOutputStream(OutputStream out,
114                                                    Compressor compressor)
115      throws IOException {
116    checkNativeCodeLoaded();
117    int bufferSize = conf.getInt(
118        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
119        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
120
121    int compressionOverhead = (bufferSize / 6) + 32;
122
123    return new BlockCompressorStream(out, compressor, bufferSize,
124        compressionOverhead);
125  }
126
127  /**
128   * Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
129   *
130   * @return the type of compressor needed by this codec.
131   */
132  @Override
133  public Class<? extends Compressor> getCompressorType() {
134    checkNativeCodeLoaded();
135    return SnappyCompressor.class;
136  }
137
138  /**
139   * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
140   *
141   * @return a new compressor for use by this codec
142   */
143  @Override
144  public Compressor createCompressor() {
145    checkNativeCodeLoaded();
146    int bufferSize = conf.getInt(
147        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
148        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
149    return new SnappyCompressor(bufferSize);
150  }
151
152  /**
153   * Create a {@link CompressionInputStream} that will read from the given
154   * input stream.
155   *
156   * @param in the stream to read compressed bytes from
157   * @return a stream to read uncompressed bytes from
158   * @throws IOException
159   */
160  @Override
161  public CompressionInputStream createInputStream(InputStream in)
162      throws IOException {
163    return CompressionCodec.Util.
164        createInputStreamWithCodecPool(this, conf, in);
165  }
166
167  /**
168   * Create a {@link CompressionInputStream} that will read from the given
169   * {@link InputStream} with the given {@link Decompressor}.
170   *
171   * @param in           the stream to read compressed bytes from
172   * @param decompressor decompressor to use
173   * @return a stream to read uncompressed bytes from
174   * @throws IOException
175   */
176  @Override
177  public CompressionInputStream createInputStream(InputStream in,
178                                                  Decompressor decompressor)
179      throws IOException {
180    checkNativeCodeLoaded();
181    return new BlockDecompressorStream(in, decompressor, conf.getInt(
182        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
183        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT));
184  }
185
186  /**
187   * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
188   *
189   * @return the type of decompressor needed by this codec.
190   */
191  @Override
192  public Class<? extends Decompressor> getDecompressorType() {
193    checkNativeCodeLoaded();
194    return SnappyDecompressor.class;
195  }
196
197  /**
198   * Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
199   *
200   * @return a new decompressor for use by this codec
201   */
202  @Override
203  public Decompressor createDecompressor() {
204    checkNativeCodeLoaded();
205    int bufferSize = conf.getInt(
206        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
207        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
208    return new SnappyDecompressor(bufferSize);
209  }
210  
211  /**
212   * {@inheritDoc}
213   */
214  @Override
215  public DirectDecompressor createDirectDecompressor() {
216    return isNativeCodeLoaded() ? new SnappyDirectDecompressor() : null;
217  }
218
219  /**
220   * Get the default filename extension for this kind of compression.
221   *
222   * @return <code>.snappy</code>.
223   */
224  @Override
225  public String getDefaultExtension() {
226    return ".snappy";
227  }
228}