001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.io.compress; 020 021import java.io.IOException; 022import java.io.InputStream; 023import java.io.OutputStream; 024 025import org.apache.hadoop.conf.Configurable; 026import org.apache.hadoop.conf.Configuration; 027import org.apache.hadoop.io.compress.snappy.SnappyCompressor; 028import org.apache.hadoop.io.compress.snappy.SnappyDecompressor; 029import org.apache.hadoop.io.compress.snappy.SnappyDecompressor.SnappyDirectDecompressor; 030import org.apache.hadoop.fs.CommonConfigurationKeys; 031import org.apache.hadoop.util.NativeCodeLoader; 032 033/** 034 * This class creates snappy compressors/decompressors. 035 */ 036public class SnappyCodec implements Configurable, CompressionCodec, DirectDecompressionCodec { 037 Configuration conf; 038 039 /** 040 * Set the configuration to be used by this object. 041 * 042 * @param conf the configuration object. 043 */ 044 @Override 045 public void setConf(Configuration conf) { 046 this.conf = conf; 047 } 048 049 /** 050 * Return the configuration used by this object. 051 * 052 * @return the configuration object used by this objec. 053 */ 054 @Override 055 public Configuration getConf() { 056 return conf; 057 } 058 059 /** 060 * Are the native snappy libraries loaded & initialized? 061 */ 062 public static void checkNativeCodeLoaded() { 063 if (!NativeCodeLoader.isNativeCodeLoaded() || 064 !NativeCodeLoader.buildSupportsSnappy()) { 065 throw new RuntimeException("native snappy library not available: " + 066 "this version of libhadoop was built without " + 067 "snappy support."); 068 } 069 if (!SnappyCompressor.isNativeCodeLoaded()) { 070 throw new RuntimeException("native snappy library not available: " + 071 "SnappyCompressor has not been loaded."); 072 } 073 if (!SnappyDecompressor.isNativeCodeLoaded()) { 074 throw new RuntimeException("native snappy library not available: " + 075 "SnappyDecompressor has not been loaded."); 076 } 077 } 078 079 public static boolean isNativeCodeLoaded() { 080 return SnappyCompressor.isNativeCodeLoaded() && 081 SnappyDecompressor.isNativeCodeLoaded(); 082 } 083 084 public static String getLibraryName() { 085 return SnappyCompressor.getLibraryName(); 086 } 087 088 /** 089 * Create a {@link CompressionOutputStream} that will write to the given 090 * {@link OutputStream}. 091 * 092 * @param out the location for the final output stream 093 * @return a stream the user can write uncompressed data to have it compressed 094 * @throws IOException 095 */ 096 @Override 097 public CompressionOutputStream createOutputStream(OutputStream out) 098 throws IOException { 099 return CompressionCodec.Util. 100 createOutputStreamWithCodecPool(this, conf, out); 101 } 102 103 /** 104 * Create a {@link CompressionOutputStream} that will write to the given 105 * {@link OutputStream} with the given {@link Compressor}. 106 * 107 * @param out the location for the final output stream 108 * @param compressor compressor to use 109 * @return a stream the user can write uncompressed data to have it compressed 110 * @throws IOException 111 */ 112 @Override 113 public CompressionOutputStream createOutputStream(OutputStream out, 114 Compressor compressor) 115 throws IOException { 116 checkNativeCodeLoaded(); 117 int bufferSize = conf.getInt( 118 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, 119 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); 120 121 int compressionOverhead = (bufferSize / 6) + 32; 122 123 return new BlockCompressorStream(out, compressor, bufferSize, 124 compressionOverhead); 125 } 126 127 /** 128 * Get the type of {@link Compressor} needed by this {@link CompressionCodec}. 129 * 130 * @return the type of compressor needed by this codec. 131 */ 132 @Override 133 public Class<? extends Compressor> getCompressorType() { 134 checkNativeCodeLoaded(); 135 return SnappyCompressor.class; 136 } 137 138 /** 139 * Create a new {@link Compressor} for use by this {@link CompressionCodec}. 140 * 141 * @return a new compressor for use by this codec 142 */ 143 @Override 144 public Compressor createCompressor() { 145 checkNativeCodeLoaded(); 146 int bufferSize = conf.getInt( 147 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, 148 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); 149 return new SnappyCompressor(bufferSize); 150 } 151 152 /** 153 * Create a {@link CompressionInputStream} that will read from the given 154 * input stream. 155 * 156 * @param in the stream to read compressed bytes from 157 * @return a stream to read uncompressed bytes from 158 * @throws IOException 159 */ 160 @Override 161 public CompressionInputStream createInputStream(InputStream in) 162 throws IOException { 163 return CompressionCodec.Util. 164 createInputStreamWithCodecPool(this, conf, in); 165 } 166 167 /** 168 * Create a {@link CompressionInputStream} that will read from the given 169 * {@link InputStream} with the given {@link Decompressor}. 170 * 171 * @param in the stream to read compressed bytes from 172 * @param decompressor decompressor to use 173 * @return a stream to read uncompressed bytes from 174 * @throws IOException 175 */ 176 @Override 177 public CompressionInputStream createInputStream(InputStream in, 178 Decompressor decompressor) 179 throws IOException { 180 checkNativeCodeLoaded(); 181 return new BlockDecompressorStream(in, decompressor, conf.getInt( 182 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, 183 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT)); 184 } 185 186 /** 187 * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}. 188 * 189 * @return the type of decompressor needed by this codec. 190 */ 191 @Override 192 public Class<? extends Decompressor> getDecompressorType() { 193 checkNativeCodeLoaded(); 194 return SnappyDecompressor.class; 195 } 196 197 /** 198 * Create a new {@link Decompressor} for use by this {@link CompressionCodec}. 199 * 200 * @return a new decompressor for use by this codec 201 */ 202 @Override 203 public Decompressor createDecompressor() { 204 checkNativeCodeLoaded(); 205 int bufferSize = conf.getInt( 206 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, 207 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); 208 return new SnappyDecompressor(bufferSize); 209 } 210 211 /** 212 * {@inheritDoc} 213 */ 214 @Override 215 public DirectDecompressor createDirectDecompressor() { 216 return isNativeCodeLoaded() ? new SnappyDirectDecompressor() : null; 217 } 218 219 /** 220 * Get the default filename extension for this kind of compression. 221 * 222 * @return <code>.snappy</code>. 223 */ 224 @Override 225 public String getDefaultExtension() { 226 return ".snappy"; 227 } 228}