001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.io.compress;
020
021import java.io.IOException;
022import java.io.InputStream;
023import java.io.OutputStream;
024
025import org.apache.hadoop.conf.Configurable;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.io.compress.snappy.LoadSnappy;
028import org.apache.hadoop.io.compress.snappy.SnappyCompressor;
029import org.apache.hadoop.io.compress.snappy.SnappyDecompressor;
030import org.apache.hadoop.fs.CommonConfigurationKeys;
031import org.apache.hadoop.util.NativeCodeLoader;
032
033/**
034 * This class creates snappy compressors/decompressors.
035 */
036public class SnappyCodec implements Configurable, CompressionCodec {
037
038  static {
039    LoadSnappy.isLoaded();
040  }
041
042  Configuration conf;
043
044  /**
045   * Set the configuration to be used by this object.
046   *
047   * @param conf the configuration object.
048   */
049  @Override
050  public void setConf(Configuration conf) {
051    this.conf = conf;
052  }
053
054  /**
055   * Return the configuration used by this object.
056   *
057   * @return the configuration object used by this objec.
058   */
059  @Override
060  public Configuration getConf() {
061    return conf;
062  }
063
064  /**
065   * Are the native snappy libraries loaded & initialized?
066   *
067   * @return true if loaded & initialized, otherwise false
068   */
069  public static boolean isNativeCodeLoaded() {
070    return LoadSnappy.isLoaded() && NativeCodeLoader.isNativeCodeLoaded();
071  }
072
073  /**
074   * Create a {@link CompressionOutputStream} that will write to the given
075   * {@link OutputStream}.
076   *
077   * @param out the location for the final output stream
078   * @return a stream the user can write uncompressed data to have it compressed
079   * @throws IOException
080   */
081  @Override
082  public CompressionOutputStream createOutputStream(OutputStream out)
083      throws IOException {
084    return createOutputStream(out, createCompressor());
085  }
086
087  /**
088   * Create a {@link CompressionOutputStream} that will write to the given
089   * {@link OutputStream} with the given {@link Compressor}.
090   *
091   * @param out        the location for the final output stream
092   * @param compressor compressor to use
093   * @return a stream the user can write uncompressed data to have it compressed
094   * @throws IOException
095   */
096  @Override
097  public CompressionOutputStream createOutputStream(OutputStream out,
098                                                    Compressor compressor)
099      throws IOException {
100    if (!isNativeCodeLoaded()) {
101      throw new RuntimeException("native snappy library not available");
102    }
103    int bufferSize = conf.getInt(
104        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
105        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
106
107    int compressionOverhead = (bufferSize / 6) + 32;
108
109    return new BlockCompressorStream(out, compressor, bufferSize,
110        compressionOverhead);
111  }
112
113  /**
114   * Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
115   *
116   * @return the type of compressor needed by this codec.
117   */
118  @Override
119  public Class<? extends Compressor> getCompressorType() {
120    if (!isNativeCodeLoaded()) {
121      throw new RuntimeException("native snappy library not available");
122    }
123
124    return SnappyCompressor.class;
125  }
126
127  /**
128   * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
129   *
130   * @return a new compressor for use by this codec
131   */
132  @Override
133  public Compressor createCompressor() {
134    if (!isNativeCodeLoaded()) {
135      throw new RuntimeException("native snappy library not available");
136    }
137    int bufferSize = conf.getInt(
138        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
139        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
140    return new SnappyCompressor(bufferSize);
141  }
142
143  /**
144   * Create a {@link CompressionInputStream} that will read from the given
145   * input stream.
146   *
147   * @param in the stream to read compressed bytes from
148   * @return a stream to read uncompressed bytes from
149   * @throws IOException
150   */
151  @Override
152  public CompressionInputStream createInputStream(InputStream in)
153      throws IOException {
154    return createInputStream(in, createDecompressor());
155  }
156
157  /**
158   * Create a {@link CompressionInputStream} that will read from the given
159   * {@link InputStream} with the given {@link Decompressor}.
160   *
161   * @param in           the stream to read compressed bytes from
162   * @param decompressor decompressor to use
163   * @return a stream to read uncompressed bytes from
164   * @throws IOException
165   */
166  @Override
167  public CompressionInputStream createInputStream(InputStream in,
168                                                  Decompressor decompressor)
169      throws IOException {
170    if (!isNativeCodeLoaded()) {
171      throw new RuntimeException("native snappy library not available");
172    }
173
174    return new BlockDecompressorStream(in, decompressor, conf.getInt(
175        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
176        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT));
177  }
178
179  /**
180   * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
181   *
182   * @return the type of decompressor needed by this codec.
183   */
184  @Override
185  public Class<? extends Decompressor> getDecompressorType() {
186    if (!isNativeCodeLoaded()) {
187      throw new RuntimeException("native snappy library not available");
188    }
189
190    return SnappyDecompressor.class;
191  }
192
193  /**
194   * Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
195   *
196   * @return a new decompressor for use by this codec
197   */
198  @Override
199  public Decompressor createDecompressor() {
200    if (!isNativeCodeLoaded()) {
201      throw new RuntimeException("native snappy library not available");
202    }
203    int bufferSize = conf.getInt(
204        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
205        CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
206    return new SnappyDecompressor(bufferSize);
207  }
208
209  /**
210   * Get the default filename extension for this kind of compression.
211   *
212   * @return <code>.snappy</code>.
213   */
214  @Override
215  public String getDefaultExtension() {
216    return ".snappy";
217  }
218}