001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.io.compress;
020
021import java.io.IOException;
022import java.io.InputStream;
023import java.io.OutputStream;
024
025import org.apache.hadoop.conf.Configurable;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.io.compress.snappy.SnappyCompressor;
028import org.apache.hadoop.io.compress.snappy.SnappyDecompressor;
029import org.apache.hadoop.fs.CommonConfigurationKeys;
030import org.apache.hadoop.util.NativeCodeLoader;
031
032/**
033 * This class creates snappy compressors/decompressors.
034 */
035public class SnappyCodec implements Configurable, CompressionCodec {
036  Configuration conf;
037
038  /**
039   * Set the configuration to be used by this object.
040   *
041   * @param conf the configuration object.
042   */
043  @Override
044  public void setConf(Configuration conf) {
045    this.conf = conf;
046  }
047
048  /**
049   * Return the configuration used by this object.
050   *
051   * @return the configuration object used by this objec.
052   */
053  @Override
054  public Configuration getConf() {
055    return conf;
056  }
057
058  /**
059   * Are the native snappy libraries loaded & initialized?
060   */
061  public static void checkNativeCodeLoaded() {
062      if (!NativeCodeLoader.buildSupportsSnappy()) {
063        throw new RuntimeException("native snappy library not available: " +
064            "this version of libhadoop was built without " +
065            "snappy support.");
066      }
067      if (!SnappyCompressor.isNativeCodeLoaded()) {
068        throw new RuntimeException("native snappy library not available: " +
069            "SnappyCompressor has not been loaded.");
070      }
071      if (!SnappyDecompressor.isNativeCodeLoaded()) {
072        throw new RuntimeException("native snappy library not available: " +
073            "SnappyDecompressor has not been loaded.");
074      }
075  }
076  
077  public static boolean isNativeCodeLoaded() {
078    return SnappyCompressor.isNativeCodeLoaded() && 
079        SnappyDecompressor.isNativeCodeLoaded();
080  }
081
082  /**
083   * Create a {@link CompressionOutputStream} that will write to the given
084   * {@link OutputStream}.
085   *
086   * @param out the location for the final output stream
087   * @return a stream the user can write uncompressed data to have it compressed
088   * @throws IOException
089   */
090  @Override
091  public CompressionOutputStream createOutputStream(OutputStream out)
092      throws IOException {
093    return createOutputStream(out, createCompressor());
094  }
095
096  /**
097   * Create a {@link CompressionOutputStream} that will write to the given
098   * {@link OutputStream} with the given {@link Compressor}.
099   *
100   * @param out        the location for the final output stream
101   * @param compressor compressor to use
102   * @return a stream the user can write uncompressed data to have it compressed
103   * @throws IOException
104   */
105  @Override
106  public CompressionOutputStream createOutputStream(OutputStream out,
107                                                    Compressor compressor)
108      throws IOException {
109    checkNativeCodeLoaded();
110    int bufferSize = conf.getInt(
111        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
112        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
113
114    int compressionOverhead = (bufferSize / 6) + 32;
115
116    return new BlockCompressorStream(out, compressor, bufferSize,
117        compressionOverhead);
118  }
119
120  /**
121   * Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
122   *
123   * @return the type of compressor needed by this codec.
124   */
125  @Override
126  public Class<? extends Compressor> getCompressorType() {
127    checkNativeCodeLoaded();
128    return SnappyCompressor.class;
129  }
130
131  /**
132   * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
133   *
134   * @return a new compressor for use by this codec
135   */
136  @Override
137  public Compressor createCompressor() {
138    checkNativeCodeLoaded();
139    int bufferSize = conf.getInt(
140        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
141        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
142    return new SnappyCompressor(bufferSize);
143  }
144
145  /**
146   * Create a {@link CompressionInputStream} that will read from the given
147   * input stream.
148   *
149   * @param in the stream to read compressed bytes from
150   * @return a stream to read uncompressed bytes from
151   * @throws IOException
152   */
153  @Override
154  public CompressionInputStream createInputStream(InputStream in)
155      throws IOException {
156    return createInputStream(in, createDecompressor());
157  }
158
159  /**
160   * Create a {@link CompressionInputStream} that will read from the given
161   * {@link InputStream} with the given {@link Decompressor}.
162   *
163   * @param in           the stream to read compressed bytes from
164   * @param decompressor decompressor to use
165   * @return a stream to read uncompressed bytes from
166   * @throws IOException
167   */
168  @Override
169  public CompressionInputStream createInputStream(InputStream in,
170                                                  Decompressor decompressor)
171      throws IOException {
172    checkNativeCodeLoaded();
173    return new BlockDecompressorStream(in, decompressor, conf.getInt(
174        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
175        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT));
176  }
177
178  /**
179   * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
180   *
181   * @return the type of decompressor needed by this codec.
182   */
183  @Override
184  public Class<? extends Decompressor> getDecompressorType() {
185    checkNativeCodeLoaded();
186    return SnappyDecompressor.class;
187  }
188
189  /**
190   * Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
191   *
192   * @return a new decompressor for use by this codec
193   */
194  @Override
195  public Decompressor createDecompressor() {
196    checkNativeCodeLoaded();
197    int bufferSize = conf.getInt(
198        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
199        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
200    return new SnappyDecompressor(bufferSize);
201  }
202
203  /**
204   * Get the default filename extension for this kind of compression.
205   *
206   * @return <code>.snappy</code>.
207   */
208  @Override
209  public String getDefaultExtension() {
210    return ".snappy";
211  }
212}