001 /*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.io.compress;
020
021 import java.io.IOException;
022 import java.io.InputStream;
023 import java.io.OutputStream;
024
025 import org.apache.hadoop.conf.Configurable;
026 import org.apache.hadoop.conf.Configuration;
027 import org.apache.hadoop.io.compress.lz4.Lz4Compressor;
028 import org.apache.hadoop.io.compress.lz4.Lz4Decompressor;
029 import org.apache.hadoop.fs.CommonConfigurationKeys;
030 import org.apache.hadoop.util.NativeCodeLoader;
031
032 /**
033 * This class creates lz4 compressors/decompressors.
034 */
035 public class Lz4Codec implements Configurable, CompressionCodec {
036
037 static {
038 NativeCodeLoader.isNativeCodeLoaded();
039 }
040
041 Configuration conf;
042
043 /**
044 * Set the configuration to be used by this object.
045 *
046 * @param conf the configuration object.
047 */
048 @Override
049 public void setConf(Configuration conf) {
050 this.conf = conf;
051 }
052
053 /**
054 * Return the configuration used by this object.
055 *
056 * @return the configuration object used by this objec.
057 */
058 @Override
059 public Configuration getConf() {
060 return conf;
061 }
062
063 /**
064 * Are the native lz4 libraries loaded & initialized?
065 *
066 * @return true if loaded & initialized, otherwise false
067 */
068 public static boolean isNativeCodeLoaded() {
069 return NativeCodeLoader.isNativeCodeLoaded();
070 }
071
072 /**
073 * Create a {@link CompressionOutputStream} that will write to the given
074 * {@link OutputStream}.
075 *
076 * @param out the location for the final output stream
077 * @return a stream the user can write uncompressed data to have it compressed
078 * @throws IOException
079 */
080 @Override
081 public CompressionOutputStream createOutputStream(OutputStream out)
082 throws IOException {
083 return createOutputStream(out, createCompressor());
084 }
085
086 /**
087 * Create a {@link CompressionOutputStream} that will write to the given
088 * {@link OutputStream} with the given {@link Compressor}.
089 *
090 * @param out the location for the final output stream
091 * @param compressor compressor to use
092 * @return a stream the user can write uncompressed data to have it compressed
093 * @throws IOException
094 */
095 @Override
096 public CompressionOutputStream createOutputStream(OutputStream out,
097 Compressor compressor)
098 throws IOException {
099 if (!isNativeCodeLoaded()) {
100 throw new RuntimeException("native lz4 library not available");
101 }
102 int bufferSize = conf.getInt(
103 CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
104 CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
105
106 int compressionOverhead = Math.max((int)(bufferSize * 0.01), 10);
107
108 return new BlockCompressorStream(out, compressor, bufferSize,
109 compressionOverhead);
110 }
111
112 /**
113 * Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
114 *
115 * @return the type of compressor needed by this codec.
116 */
117 @Override
118 public Class<? extends Compressor> getCompressorType() {
119 if (!isNativeCodeLoaded()) {
120 throw new RuntimeException("native lz4 library not available");
121 }
122
123 return Lz4Compressor.class;
124 }
125
126 /**
127 * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
128 *
129 * @return a new compressor for use by this codec
130 */
131 @Override
132 public Compressor createCompressor() {
133 if (!isNativeCodeLoaded()) {
134 throw new RuntimeException("native lz4 library not available");
135 }
136 int bufferSize = conf.getInt(
137 CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
138 CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
139 return new Lz4Compressor(bufferSize);
140 }
141
142 /**
143 * Create a {@link CompressionInputStream} that will read from the given
144 * input stream.
145 *
146 * @param in the stream to read compressed bytes from
147 * @return a stream to read uncompressed bytes from
148 * @throws IOException
149 */
150 @Override
151 public CompressionInputStream createInputStream(InputStream in)
152 throws IOException {
153 return createInputStream(in, createDecompressor());
154 }
155
156 /**
157 * Create a {@link CompressionInputStream} that will read from the given
158 * {@link InputStream} with the given {@link Decompressor}.
159 *
160 * @param in the stream to read compressed bytes from
161 * @param decompressor decompressor to use
162 * @return a stream to read uncompressed bytes from
163 * @throws IOException
164 */
165 @Override
166 public CompressionInputStream createInputStream(InputStream in,
167 Decompressor decompressor)
168 throws IOException {
169 if (!isNativeCodeLoaded()) {
170 throw new RuntimeException("native lz4 library not available");
171 }
172
173 return new BlockDecompressorStream(in, decompressor, conf.getInt(
174 CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
175 CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT));
176 }
177
178 /**
179 * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
180 *
181 * @return the type of decompressor needed by this codec.
182 */
183 @Override
184 public Class<? extends Decompressor> getDecompressorType() {
185 if (!isNativeCodeLoaded()) {
186 throw new RuntimeException("native lz4 library not available");
187 }
188
189 return Lz4Decompressor.class;
190 }
191
192 /**
193 * Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
194 *
195 * @return a new decompressor for use by this codec
196 */
197 @Override
198 public Decompressor createDecompressor() {
199 if (!isNativeCodeLoaded()) {
200 throw new RuntimeException("native lz4 library not available");
201 }
202 int bufferSize = conf.getInt(
203 CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
204 CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
205 return new Lz4Decompressor(bufferSize);
206 }
207
208 /**
209 * Get the default filename extension for this kind of compression.
210 *
211 * @return <code>.lz4</code>.
212 */
213 @Override
214 public String getDefaultExtension() {
215 return ".lz4";
216 }
217 }