001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.io.compress;
020
021import java.io.IOException;
022import java.io.InputStream;
023import java.io.OutputStream;
024
025import org.apache.hadoop.conf.Configurable;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.io.compress.snappy.SnappyCompressor;
028import org.apache.hadoop.io.compress.snappy.SnappyDecompressor;
029import org.apache.hadoop.io.compress.snappy.SnappyDecompressor.SnappyDirectDecompressor;
030import org.apache.hadoop.fs.CommonConfigurationKeys;
031import org.apache.hadoop.util.NativeCodeLoader;
032
033/**
034 * This class creates snappy compressors/decompressors.
035 */
036public class SnappyCodec implements Configurable, CompressionCodec, DirectDecompressionCodec {
037  Configuration conf;
038
039  /**
040   * Set the configuration to be used by this object.
041   *
042   * @param conf the configuration object.
043   */
044  @Override
045  public void setConf(Configuration conf) {
046    this.conf = conf;
047  }
048
049  /**
050   * Return the configuration used by this object.
051   *
052   * @return the configuration object used by this objec.
053   */
054  @Override
055  public Configuration getConf() {
056    return conf;
057  }
058
059  /**
060   * Are the native snappy libraries loaded & initialized?
061   */
062  public static void checkNativeCodeLoaded() {
063      if (!NativeCodeLoader.buildSupportsSnappy()) {
064        throw new RuntimeException("native snappy library not available: " +
065            "this version of libhadoop was built without " +
066            "snappy support.");
067      }
068      if (!SnappyCompressor.isNativeCodeLoaded()) {
069        throw new RuntimeException("native snappy library not available: " +
070            "SnappyCompressor has not been loaded.");
071      }
072      if (!SnappyDecompressor.isNativeCodeLoaded()) {
073        throw new RuntimeException("native snappy library not available: " +
074            "SnappyDecompressor has not been loaded.");
075      }
076  }
077  
078  public static boolean isNativeCodeLoaded() {
079    return SnappyCompressor.isNativeCodeLoaded() && 
080        SnappyDecompressor.isNativeCodeLoaded();
081  }
082
083  public static String getLibraryName() {
084    return SnappyCompressor.getLibraryName();
085  }
086
087  /**
088   * Create a {@link CompressionOutputStream} that will write to the given
089   * {@link OutputStream}.
090   *
091   * @param out the location for the final output stream
092   * @return a stream the user can write uncompressed data to have it compressed
093   * @throws IOException
094   */
095  @Override
096  public CompressionOutputStream createOutputStream(OutputStream out)
097      throws IOException {
098    return createOutputStream(out, createCompressor());
099  }
100
101  /**
102   * Create a {@link CompressionOutputStream} that will write to the given
103   * {@link OutputStream} with the given {@link Compressor}.
104   *
105   * @param out        the location for the final output stream
106   * @param compressor compressor to use
107   * @return a stream the user can write uncompressed data to have it compressed
108   * @throws IOException
109   */
110  @Override
111  public CompressionOutputStream createOutputStream(OutputStream out,
112                                                    Compressor compressor)
113      throws IOException {
114    checkNativeCodeLoaded();
115    int bufferSize = conf.getInt(
116        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
117        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
118
119    int compressionOverhead = (bufferSize / 6) + 32;
120
121    return new BlockCompressorStream(out, compressor, bufferSize,
122        compressionOverhead);
123  }
124
125  /**
126   * Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
127   *
128   * @return the type of compressor needed by this codec.
129   */
130  @Override
131  public Class<? extends Compressor> getCompressorType() {
132    checkNativeCodeLoaded();
133    return SnappyCompressor.class;
134  }
135
136  /**
137   * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
138   *
139   * @return a new compressor for use by this codec
140   */
141  @Override
142  public Compressor createCompressor() {
143    checkNativeCodeLoaded();
144    int bufferSize = conf.getInt(
145        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
146        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
147    return new SnappyCompressor(bufferSize);
148  }
149
150  /**
151   * Create a {@link CompressionInputStream} that will read from the given
152   * input stream.
153   *
154   * @param in the stream to read compressed bytes from
155   * @return a stream to read uncompressed bytes from
156   * @throws IOException
157   */
158  @Override
159  public CompressionInputStream createInputStream(InputStream in)
160      throws IOException {
161    return createInputStream(in, createDecompressor());
162  }
163
164  /**
165   * Create a {@link CompressionInputStream} that will read from the given
166   * {@link InputStream} with the given {@link Decompressor}.
167   *
168   * @param in           the stream to read compressed bytes from
169   * @param decompressor decompressor to use
170   * @return a stream to read uncompressed bytes from
171   * @throws IOException
172   */
173  @Override
174  public CompressionInputStream createInputStream(InputStream in,
175                                                  Decompressor decompressor)
176      throws IOException {
177    checkNativeCodeLoaded();
178    return new BlockDecompressorStream(in, decompressor, conf.getInt(
179        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
180        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT));
181  }
182
183  /**
184   * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
185   *
186   * @return the type of decompressor needed by this codec.
187   */
188  @Override
189  public Class<? extends Decompressor> getDecompressorType() {
190    checkNativeCodeLoaded();
191    return SnappyDecompressor.class;
192  }
193
194  /**
195   * Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
196   *
197   * @return a new decompressor for use by this codec
198   */
199  @Override
200  public Decompressor createDecompressor() {
201    checkNativeCodeLoaded();
202    int bufferSize = conf.getInt(
203        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
204        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
205    return new SnappyDecompressor(bufferSize);
206  }
207  
208  /**
209   * {@inheritDoc}
210   */
211  @Override
212  public DirectDecompressor createDirectDecompressor() {
213    return isNativeCodeLoaded() ? new SnappyDirectDecompressor() : null;
214  }
215
216  /**
217   * Get the default filename extension for this kind of compression.
218   *
219   * @return <code>.snappy</code>.
220   */
221  @Override
222  public String getDefaultExtension() {
223    return ".snappy";
224  }
225}