001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.io.compress; 020 021import java.io.IOException; 022import java.io.InputStream; 023import java.io.OutputStream; 024 025import org.apache.hadoop.conf.Configurable; 026import org.apache.hadoop.conf.Configuration; 027import org.apache.hadoop.io.compress.snappy.SnappyCompressor; 028import org.apache.hadoop.io.compress.snappy.SnappyDecompressor; 029import org.apache.hadoop.io.compress.snappy.SnappyDecompressor.SnappyDirectDecompressor; 030import org.apache.hadoop.fs.CommonConfigurationKeys; 031import org.apache.hadoop.util.NativeCodeLoader; 032 033/** 034 * This class creates snappy compressors/decompressors. 035 */ 036public class SnappyCodec implements Configurable, CompressionCodec, DirectDecompressionCodec { 037 Configuration conf; 038 039 /** 040 * Set the configuration to be used by this object. 041 * 042 * @param conf the configuration object. 043 */ 044 @Override 045 public void setConf(Configuration conf) { 046 this.conf = conf; 047 } 048 049 /** 050 * Return the configuration used by this object. 051 * 052 * @return the configuration object used by this objec. 053 */ 054 @Override 055 public Configuration getConf() { 056 return conf; 057 } 058 059 /** 060 * Are the native snappy libraries loaded & initialized? 061 */ 062 public static void checkNativeCodeLoaded() { 063 if (!NativeCodeLoader.buildSupportsSnappy()) { 064 throw new RuntimeException("native snappy library not available: " + 065 "this version of libhadoop was built without " + 066 "snappy support."); 067 } 068 if (!SnappyCompressor.isNativeCodeLoaded()) { 069 throw new RuntimeException("native snappy library not available: " + 070 "SnappyCompressor has not been loaded."); 071 } 072 if (!SnappyDecompressor.isNativeCodeLoaded()) { 073 throw new RuntimeException("native snappy library not available: " + 074 "SnappyDecompressor has not been loaded."); 075 } 076 } 077 078 public static boolean isNativeCodeLoaded() { 079 return SnappyCompressor.isNativeCodeLoaded() && 080 SnappyDecompressor.isNativeCodeLoaded(); 081 } 082 083 public static String getLibraryName() { 084 return SnappyCompressor.getLibraryName(); 085 } 086 087 /** 088 * Create a {@link CompressionOutputStream} that will write to the given 089 * {@link OutputStream}. 090 * 091 * @param out the location for the final output stream 092 * @return a stream the user can write uncompressed data to have it compressed 093 * @throws IOException 094 */ 095 @Override 096 public CompressionOutputStream createOutputStream(OutputStream out) 097 throws IOException { 098 return CompressionCodec.Util. 099 createOutputStreamWithCodecPool(this, conf, out); 100 } 101 102 /** 103 * Create a {@link CompressionOutputStream} that will write to the given 104 * {@link OutputStream} with the given {@link Compressor}. 105 * 106 * @param out the location for the final output stream 107 * @param compressor compressor to use 108 * @return a stream the user can write uncompressed data to have it compressed 109 * @throws IOException 110 */ 111 @Override 112 public CompressionOutputStream createOutputStream(OutputStream out, 113 Compressor compressor) 114 throws IOException { 115 checkNativeCodeLoaded(); 116 int bufferSize = conf.getInt( 117 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, 118 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); 119 120 int compressionOverhead = (bufferSize / 6) + 32; 121 122 return new BlockCompressorStream(out, compressor, bufferSize, 123 compressionOverhead); 124 } 125 126 /** 127 * Get the type of {@link Compressor} needed by this {@link CompressionCodec}. 128 * 129 * @return the type of compressor needed by this codec. 130 */ 131 @Override 132 public Class<? extends Compressor> getCompressorType() { 133 checkNativeCodeLoaded(); 134 return SnappyCompressor.class; 135 } 136 137 /** 138 * Create a new {@link Compressor} for use by this {@link CompressionCodec}. 139 * 140 * @return a new compressor for use by this codec 141 */ 142 @Override 143 public Compressor createCompressor() { 144 checkNativeCodeLoaded(); 145 int bufferSize = conf.getInt( 146 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, 147 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); 148 return new SnappyCompressor(bufferSize); 149 } 150 151 /** 152 * Create a {@link CompressionInputStream} that will read from the given 153 * input stream. 154 * 155 * @param in the stream to read compressed bytes from 156 * @return a stream to read uncompressed bytes from 157 * @throws IOException 158 */ 159 @Override 160 public CompressionInputStream createInputStream(InputStream in) 161 throws IOException { 162 return CompressionCodec.Util. 163 createInputStreamWithCodecPool(this, conf, in); 164 } 165 166 /** 167 * Create a {@link CompressionInputStream} that will read from the given 168 * {@link InputStream} with the given {@link Decompressor}. 169 * 170 * @param in the stream to read compressed bytes from 171 * @param decompressor decompressor to use 172 * @return a stream to read uncompressed bytes from 173 * @throws IOException 174 */ 175 @Override 176 public CompressionInputStream createInputStream(InputStream in, 177 Decompressor decompressor) 178 throws IOException { 179 checkNativeCodeLoaded(); 180 return new BlockDecompressorStream(in, decompressor, conf.getInt( 181 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, 182 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT)); 183 } 184 185 /** 186 * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}. 187 * 188 * @return the type of decompressor needed by this codec. 189 */ 190 @Override 191 public Class<? extends Decompressor> getDecompressorType() { 192 checkNativeCodeLoaded(); 193 return SnappyDecompressor.class; 194 } 195 196 /** 197 * Create a new {@link Decompressor} for use by this {@link CompressionCodec}. 198 * 199 * @return a new decompressor for use by this codec 200 */ 201 @Override 202 public Decompressor createDecompressor() { 203 checkNativeCodeLoaded(); 204 int bufferSize = conf.getInt( 205 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, 206 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); 207 return new SnappyDecompressor(bufferSize); 208 } 209 210 /** 211 * {@inheritDoc} 212 */ 213 @Override 214 public DirectDecompressor createDirectDecompressor() { 215 return isNativeCodeLoaded() ? new SnappyDirectDecompressor() : null; 216 } 217 218 /** 219 * Get the default filename extension for this kind of compression. 220 * 221 * @return <code>.snappy</code>. 222 */ 223 @Override 224 public String getDefaultExtension() { 225 return ".snappy"; 226 } 227}