001 /*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.io.compress;
020
021 import java.io.IOException;
022 import java.io.InputStream;
023 import java.io.OutputStream;
024
025 import org.apache.hadoop.conf.Configurable;
026 import org.apache.hadoop.conf.Configuration;
027 import org.apache.hadoop.io.compress.snappy.LoadSnappy;
028 import org.apache.hadoop.io.compress.snappy.SnappyCompressor;
029 import org.apache.hadoop.io.compress.snappy.SnappyDecompressor;
030 import org.apache.hadoop.fs.CommonConfigurationKeys;
031 import org.apache.hadoop.util.NativeCodeLoader;
032
033 /**
034 * This class creates snappy compressors/decompressors.
035 */
036 public class SnappyCodec implements Configurable, CompressionCodec {
037
038 static {
039 LoadSnappy.isLoaded();
040 }
041
042 Configuration conf;
043
044 /**
045 * Set the configuration to be used by this object.
046 *
047 * @param conf the configuration object.
048 */
049 @Override
050 public void setConf(Configuration conf) {
051 this.conf = conf;
052 }
053
054 /**
055 * Return the configuration used by this object.
056 *
057 * @return the configuration object used by this objec.
058 */
059 @Override
060 public Configuration getConf() {
061 return conf;
062 }
063
064 /**
065 * Are the native snappy libraries loaded & initialized?
066 *
067 * @return true if loaded & initialized, otherwise false
068 */
069 public static boolean isNativeCodeLoaded() {
070 return LoadSnappy.isLoaded() && NativeCodeLoader.isNativeCodeLoaded();
071 }
072
073 /**
074 * Create a {@link CompressionOutputStream} that will write to the given
075 * {@link OutputStream}.
076 *
077 * @param out the location for the final output stream
078 * @return a stream the user can write uncompressed data to have it compressed
079 * @throws IOException
080 */
081 @Override
082 public CompressionOutputStream createOutputStream(OutputStream out)
083 throws IOException {
084 return createOutputStream(out, createCompressor());
085 }
086
087 /**
088 * Create a {@link CompressionOutputStream} that will write to the given
089 * {@link OutputStream} with the given {@link Compressor}.
090 *
091 * @param out the location for the final output stream
092 * @param compressor compressor to use
093 * @return a stream the user can write uncompressed data to have it compressed
094 * @throws IOException
095 */
096 @Override
097 public CompressionOutputStream createOutputStream(OutputStream out,
098 Compressor compressor)
099 throws IOException {
100 if (!isNativeCodeLoaded()) {
101 throw new RuntimeException("native snappy library not available");
102 }
103 int bufferSize = conf.getInt(
104 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
105 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
106
107 int compressionOverhead = (bufferSize / 6) + 32;
108
109 return new BlockCompressorStream(out, compressor, bufferSize,
110 compressionOverhead);
111 }
112
113 /**
114 * Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
115 *
116 * @return the type of compressor needed by this codec.
117 */
118 @Override
119 public Class<? extends Compressor> getCompressorType() {
120 if (!isNativeCodeLoaded()) {
121 throw new RuntimeException("native snappy library not available");
122 }
123
124 return SnappyCompressor.class;
125 }
126
127 /**
128 * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
129 *
130 * @return a new compressor for use by this codec
131 */
132 @Override
133 public Compressor createCompressor() {
134 if (!isNativeCodeLoaded()) {
135 throw new RuntimeException("native snappy library not available");
136 }
137 int bufferSize = conf.getInt(
138 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
139 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
140 return new SnappyCompressor(bufferSize);
141 }
142
143 /**
144 * Create a {@link CompressionInputStream} that will read from the given
145 * input stream.
146 *
147 * @param in the stream to read compressed bytes from
148 * @return a stream to read uncompressed bytes from
149 * @throws IOException
150 */
151 @Override
152 public CompressionInputStream createInputStream(InputStream in)
153 throws IOException {
154 return createInputStream(in, createDecompressor());
155 }
156
157 /**
158 * Create a {@link CompressionInputStream} that will read from the given
159 * {@link InputStream} with the given {@link Decompressor}.
160 *
161 * @param in the stream to read compressed bytes from
162 * @param decompressor decompressor to use
163 * @return a stream to read uncompressed bytes from
164 * @throws IOException
165 */
166 @Override
167 public CompressionInputStream createInputStream(InputStream in,
168 Decompressor decompressor)
169 throws IOException {
170 if (!isNativeCodeLoaded()) {
171 throw new RuntimeException("native snappy library not available");
172 }
173
174 return new BlockDecompressorStream(in, decompressor, conf.getInt(
175 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
176 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT));
177 }
178
179 /**
180 * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
181 *
182 * @return the type of decompressor needed by this codec.
183 */
184 @Override
185 public Class<? extends Decompressor> getDecompressorType() {
186 if (!isNativeCodeLoaded()) {
187 throw new RuntimeException("native snappy library not available");
188 }
189
190 return SnappyDecompressor.class;
191 }
192
193 /**
194 * Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
195 *
196 * @return a new decompressor for use by this codec
197 */
198 @Override
199 public Decompressor createDecompressor() {
200 if (!isNativeCodeLoaded()) {
201 throw new RuntimeException("native snappy library not available");
202 }
203 int bufferSize = conf.getInt(
204 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
205 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
206 return new SnappyDecompressor(bufferSize);
207 }
208
209 /**
210 * Get the default filename extension for this kind of compression.
211 *
212 * @return <code>.snappy</code>.
213 */
214 @Override
215 public String getDefaultExtension() {
216 return ".snappy";
217 }
218 }