001 /*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.io.compress;
020
021 import java.io.IOException;
022 import java.io.InputStream;
023 import java.io.OutputStream;
024
025 import org.apache.hadoop.conf.Configurable;
026 import org.apache.hadoop.conf.Configuration;
027 import org.apache.hadoop.io.compress.snappy.SnappyCompressor;
028 import org.apache.hadoop.io.compress.snappy.SnappyDecompressor;
029 import org.apache.hadoop.io.compress.snappy.SnappyDecompressor.SnappyDirectDecompressor;
030 import org.apache.hadoop.fs.CommonConfigurationKeys;
031 import org.apache.hadoop.util.NativeCodeLoader;
032
033 /**
034 * This class creates snappy compressors/decompressors.
035 */
036 public class SnappyCodec implements Configurable, CompressionCodec, DirectDecompressionCodec {
037 Configuration conf;
038
039 /**
040 * Set the configuration to be used by this object.
041 *
042 * @param conf the configuration object.
043 */
044 @Override
045 public void setConf(Configuration conf) {
046 this.conf = conf;
047 }
048
049 /**
050 * Return the configuration used by this object.
051 *
052 * @return the configuration object used by this objec.
053 */
054 @Override
055 public Configuration getConf() {
056 return conf;
057 }
058
059 /**
060 * Are the native snappy libraries loaded & initialized?
061 */
062 public static void checkNativeCodeLoaded() {
063 if (!NativeCodeLoader.buildSupportsSnappy()) {
064 throw new RuntimeException("native snappy library not available: " +
065 "this version of libhadoop was built without " +
066 "snappy support.");
067 }
068 if (!SnappyCompressor.isNativeCodeLoaded()) {
069 throw new RuntimeException("native snappy library not available: " +
070 "SnappyCompressor has not been loaded.");
071 }
072 if (!SnappyDecompressor.isNativeCodeLoaded()) {
073 throw new RuntimeException("native snappy library not available: " +
074 "SnappyDecompressor has not been loaded.");
075 }
076 }
077
078 public static boolean isNativeCodeLoaded() {
079 return SnappyCompressor.isNativeCodeLoaded() &&
080 SnappyDecompressor.isNativeCodeLoaded();
081 }
082
083 public static String getLibraryName() {
084 return SnappyCompressor.getLibraryName();
085 }
086
087 /**
088 * Create a {@link CompressionOutputStream} that will write to the given
089 * {@link OutputStream}.
090 *
091 * @param out the location for the final output stream
092 * @return a stream the user can write uncompressed data to have it compressed
093 * @throws IOException
094 */
095 @Override
096 public CompressionOutputStream createOutputStream(OutputStream out)
097 throws IOException {
098 return CompressionCodec.Util.
099 createOutputStreamWithCodecPool(this, conf, out);
100 }
101
102 /**
103 * Create a {@link CompressionOutputStream} that will write to the given
104 * {@link OutputStream} with the given {@link Compressor}.
105 *
106 * @param out the location for the final output stream
107 * @param compressor compressor to use
108 * @return a stream the user can write uncompressed data to have it compressed
109 * @throws IOException
110 */
111 @Override
112 public CompressionOutputStream createOutputStream(OutputStream out,
113 Compressor compressor)
114 throws IOException {
115 checkNativeCodeLoaded();
116 int bufferSize = conf.getInt(
117 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
118 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
119
120 int compressionOverhead = (bufferSize / 6) + 32;
121
122 return new BlockCompressorStream(out, compressor, bufferSize,
123 compressionOverhead);
124 }
125
126 /**
127 * Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
128 *
129 * @return the type of compressor needed by this codec.
130 */
131 @Override
132 public Class<? extends Compressor> getCompressorType() {
133 checkNativeCodeLoaded();
134 return SnappyCompressor.class;
135 }
136
137 /**
138 * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
139 *
140 * @return a new compressor for use by this codec
141 */
142 @Override
143 public Compressor createCompressor() {
144 checkNativeCodeLoaded();
145 int bufferSize = conf.getInt(
146 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
147 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
148 return new SnappyCompressor(bufferSize);
149 }
150
151 /**
152 * Create a {@link CompressionInputStream} that will read from the given
153 * input stream.
154 *
155 * @param in the stream to read compressed bytes from
156 * @return a stream to read uncompressed bytes from
157 * @throws IOException
158 */
159 @Override
160 public CompressionInputStream createInputStream(InputStream in)
161 throws IOException {
162 return CompressionCodec.Util.
163 createInputStreamWithCodecPool(this, conf, in);
164 }
165
166 /**
167 * Create a {@link CompressionInputStream} that will read from the given
168 * {@link InputStream} with the given {@link Decompressor}.
169 *
170 * @param in the stream to read compressed bytes from
171 * @param decompressor decompressor to use
172 * @return a stream to read uncompressed bytes from
173 * @throws IOException
174 */
175 @Override
176 public CompressionInputStream createInputStream(InputStream in,
177 Decompressor decompressor)
178 throws IOException {
179 checkNativeCodeLoaded();
180 return new BlockDecompressorStream(in, decompressor, conf.getInt(
181 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
182 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT));
183 }
184
185 /**
186 * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
187 *
188 * @return the type of decompressor needed by this codec.
189 */
190 @Override
191 public Class<? extends Decompressor> getDecompressorType() {
192 checkNativeCodeLoaded();
193 return SnappyDecompressor.class;
194 }
195
196 /**
197 * Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
198 *
199 * @return a new decompressor for use by this codec
200 */
201 @Override
202 public Decompressor createDecompressor() {
203 checkNativeCodeLoaded();
204 int bufferSize = conf.getInt(
205 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
206 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
207 return new SnappyDecompressor(bufferSize);
208 }
209
210 /**
211 * {@inheritDoc}
212 */
213 @Override
214 public DirectDecompressor createDirectDecompressor() {
215 return isNativeCodeLoaded() ? new SnappyDirectDecompressor() : null;
216 }
217
218 /**
219 * Get the default filename extension for this kind of compression.
220 *
221 * @return <code>.snappy</code>.
222 */
223 @Override
224 public String getDefaultExtension() {
225 return ".snappy";
226 }
227 }