001 /*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.io.compress;
020
021 import java.io.IOException;
022 import java.io.InputStream;
023 import java.io.OutputStream;
024
025 import org.apache.hadoop.conf.Configurable;
026 import org.apache.hadoop.conf.Configuration;
027 import org.apache.hadoop.io.compress.snappy.SnappyCompressor;
028 import org.apache.hadoop.io.compress.snappy.SnappyDecompressor;
029 import org.apache.hadoop.fs.CommonConfigurationKeys;
030 import org.apache.hadoop.util.NativeCodeLoader;
031
032 /**
033 * This class creates snappy compressors/decompressors.
034 */
035 public class SnappyCodec implements Configurable, CompressionCodec {
036 Configuration conf;
037
038 /**
039 * Set the configuration to be used by this object.
040 *
041 * @param conf the configuration object.
042 */
043 @Override
044 public void setConf(Configuration conf) {
045 this.conf = conf;
046 }
047
048 /**
049 * Return the configuration used by this object.
050 *
051 * @return the configuration object used by this objec.
052 */
053 @Override
054 public Configuration getConf() {
055 return conf;
056 }
057
058 /**
059 * Are the native snappy libraries loaded & initialized?
060 */
061 public static void checkNativeCodeLoaded() {
062 if (!NativeCodeLoader.buildSupportsSnappy()) {
063 throw new RuntimeException("native snappy library not available: " +
064 "this version of libhadoop was built without " +
065 "snappy support.");
066 }
067 if (!SnappyCompressor.isNativeCodeLoaded()) {
068 throw new RuntimeException("native snappy library not available: " +
069 "SnappyCompressor has not been loaded.");
070 }
071 if (!SnappyDecompressor.isNativeCodeLoaded()) {
072 throw new RuntimeException("native snappy library not available: " +
073 "SnappyDecompressor has not been loaded.");
074 }
075 }
076
077 public static boolean isNativeCodeLoaded() {
078 return SnappyCompressor.isNativeCodeLoaded() &&
079 SnappyDecompressor.isNativeCodeLoaded();
080 }
081
082 public static String getLibraryName() {
083 return SnappyCompressor.getLibraryName();
084 }
085
086 /**
087 * Create a {@link CompressionOutputStream} that will write to the given
088 * {@link OutputStream}.
089 *
090 * @param out the location for the final output stream
091 * @return a stream the user can write uncompressed data to have it compressed
092 * @throws IOException
093 */
094 @Override
095 public CompressionOutputStream createOutputStream(OutputStream out)
096 throws IOException {
097 return createOutputStream(out, createCompressor());
098 }
099
100 /**
101 * Create a {@link CompressionOutputStream} that will write to the given
102 * {@link OutputStream} with the given {@link Compressor}.
103 *
104 * @param out the location for the final output stream
105 * @param compressor compressor to use
106 * @return a stream the user can write uncompressed data to have it compressed
107 * @throws IOException
108 */
109 @Override
110 public CompressionOutputStream createOutputStream(OutputStream out,
111 Compressor compressor)
112 throws IOException {
113 checkNativeCodeLoaded();
114 int bufferSize = conf.getInt(
115 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
116 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
117
118 int compressionOverhead = (bufferSize / 6) + 32;
119
120 return new BlockCompressorStream(out, compressor, bufferSize,
121 compressionOverhead);
122 }
123
124 /**
125 * Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
126 *
127 * @return the type of compressor needed by this codec.
128 */
129 @Override
130 public Class<? extends Compressor> getCompressorType() {
131 checkNativeCodeLoaded();
132 return SnappyCompressor.class;
133 }
134
135 /**
136 * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
137 *
138 * @return a new compressor for use by this codec
139 */
140 @Override
141 public Compressor createCompressor() {
142 checkNativeCodeLoaded();
143 int bufferSize = conf.getInt(
144 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
145 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
146 return new SnappyCompressor(bufferSize);
147 }
148
149 /**
150 * Create a {@link CompressionInputStream} that will read from the given
151 * input stream.
152 *
153 * @param in the stream to read compressed bytes from
154 * @return a stream to read uncompressed bytes from
155 * @throws IOException
156 */
157 @Override
158 public CompressionInputStream createInputStream(InputStream in)
159 throws IOException {
160 return createInputStream(in, createDecompressor());
161 }
162
163 /**
164 * Create a {@link CompressionInputStream} that will read from the given
165 * {@link InputStream} with the given {@link Decompressor}.
166 *
167 * @param in the stream to read compressed bytes from
168 * @param decompressor decompressor to use
169 * @return a stream to read uncompressed bytes from
170 * @throws IOException
171 */
172 @Override
173 public CompressionInputStream createInputStream(InputStream in,
174 Decompressor decompressor)
175 throws IOException {
176 checkNativeCodeLoaded();
177 return new BlockDecompressorStream(in, decompressor, conf.getInt(
178 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
179 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT));
180 }
181
182 /**
183 * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
184 *
185 * @return the type of decompressor needed by this codec.
186 */
187 @Override
188 public Class<? extends Decompressor> getDecompressorType() {
189 checkNativeCodeLoaded();
190 return SnappyDecompressor.class;
191 }
192
193 /**
194 * Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
195 *
196 * @return a new decompressor for use by this codec
197 */
198 @Override
199 public Decompressor createDecompressor() {
200 checkNativeCodeLoaded();
201 int bufferSize = conf.getInt(
202 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
203 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
204 return new SnappyDecompressor(bufferSize);
205 }
206
207 /**
208 * Get the default filename extension for this kind of compression.
209 *
210 * @return <code>.snappy</code>.
211 */
212 @Override
213 public String getDefaultExtension() {
214 return ".snappy";
215 }
216 }