001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.io.compress; 020 021import java.io.IOException; 022import java.io.InputStream; 023import java.io.OutputStream; 024 025import org.apache.hadoop.conf.Configurable; 026import org.apache.hadoop.conf.Configuration; 027import org.apache.hadoop.io.compress.snappy.SnappyCompressor; 028import org.apache.hadoop.io.compress.snappy.SnappyDecompressor; 029import org.apache.hadoop.io.compress.snappy.SnappyDecompressor.SnappyDirectDecompressor; 030import org.apache.hadoop.fs.CommonConfigurationKeys; 031import org.apache.hadoop.util.NativeCodeLoader; 032 033/** 034 * This class creates snappy compressors/decompressors. 035 */ 036public class SnappyCodec implements Configurable, CompressionCodec, DirectDecompressionCodec { 037 Configuration conf; 038 039 /** 040 * Set the configuration to be used by this object. 041 * 042 * @param conf the configuration object. 043 */ 044 @Override 045 public void setConf(Configuration conf) { 046 this.conf = conf; 047 } 048 049 /** 050 * Return the configuration used by this object. 051 * 052 * @return the configuration object used by this objec. 053 */ 054 @Override 055 public Configuration getConf() { 056 return conf; 057 } 058 059 /** 060 * Are the native snappy libraries loaded & initialized? 061 */ 062 public static void checkNativeCodeLoaded() { 063 if (!NativeCodeLoader.buildSupportsSnappy()) { 064 throw new RuntimeException("native snappy library not available: " + 065 "this version of libhadoop was built without " + 066 "snappy support."); 067 } 068 if (!SnappyCompressor.isNativeCodeLoaded()) { 069 throw new RuntimeException("native snappy library not available: " + 070 "SnappyCompressor has not been loaded."); 071 } 072 if (!SnappyDecompressor.isNativeCodeLoaded()) { 073 throw new RuntimeException("native snappy library not available: " + 074 "SnappyDecompressor has not been loaded."); 075 } 076 } 077 078 public static boolean isNativeCodeLoaded() { 079 return SnappyCompressor.isNativeCodeLoaded() && 080 SnappyDecompressor.isNativeCodeLoaded(); 081 } 082 083 public static String getLibraryName() { 084 return SnappyCompressor.getLibraryName(); 085 } 086 087 /** 088 * Create a {@link CompressionOutputStream} that will write to the given 089 * {@link OutputStream}. 090 * 091 * @param out the location for the final output stream 092 * @return a stream the user can write uncompressed data to have it compressed 093 * @throws IOException 094 */ 095 @Override 096 public CompressionOutputStream createOutputStream(OutputStream out) 097 throws IOException { 098 return createOutputStream(out, createCompressor()); 099 } 100 101 /** 102 * Create a {@link CompressionOutputStream} that will write to the given 103 * {@link OutputStream} with the given {@link Compressor}. 104 * 105 * @param out the location for the final output stream 106 * @param compressor compressor to use 107 * @return a stream the user can write uncompressed data to have it compressed 108 * @throws IOException 109 */ 110 @Override 111 public CompressionOutputStream createOutputStream(OutputStream out, 112 Compressor compressor) 113 throws IOException { 114 checkNativeCodeLoaded(); 115 int bufferSize = conf.getInt( 116 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, 117 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); 118 119 int compressionOverhead = (bufferSize / 6) + 32; 120 121 return new BlockCompressorStream(out, compressor, bufferSize, 122 compressionOverhead); 123 } 124 125 /** 126 * Get the type of {@link Compressor} needed by this {@link CompressionCodec}. 127 * 128 * @return the type of compressor needed by this codec. 129 */ 130 @Override 131 public Class<? extends Compressor> getCompressorType() { 132 checkNativeCodeLoaded(); 133 return SnappyCompressor.class; 134 } 135 136 /** 137 * Create a new {@link Compressor} for use by this {@link CompressionCodec}. 138 * 139 * @return a new compressor for use by this codec 140 */ 141 @Override 142 public Compressor createCompressor() { 143 checkNativeCodeLoaded(); 144 int bufferSize = conf.getInt( 145 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, 146 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); 147 return new SnappyCompressor(bufferSize); 148 } 149 150 /** 151 * Create a {@link CompressionInputStream} that will read from the given 152 * input stream. 153 * 154 * @param in the stream to read compressed bytes from 155 * @return a stream to read uncompressed bytes from 156 * @throws IOException 157 */ 158 @Override 159 public CompressionInputStream createInputStream(InputStream in) 160 throws IOException { 161 return createInputStream(in, createDecompressor()); 162 } 163 164 /** 165 * Create a {@link CompressionInputStream} that will read from the given 166 * {@link InputStream} with the given {@link Decompressor}. 167 * 168 * @param in the stream to read compressed bytes from 169 * @param decompressor decompressor to use 170 * @return a stream to read uncompressed bytes from 171 * @throws IOException 172 */ 173 @Override 174 public CompressionInputStream createInputStream(InputStream in, 175 Decompressor decompressor) 176 throws IOException { 177 checkNativeCodeLoaded(); 178 return new BlockDecompressorStream(in, decompressor, conf.getInt( 179 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, 180 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT)); 181 } 182 183 /** 184 * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}. 185 * 186 * @return the type of decompressor needed by this codec. 187 */ 188 @Override 189 public Class<? extends Decompressor> getDecompressorType() { 190 checkNativeCodeLoaded(); 191 return SnappyDecompressor.class; 192 } 193 194 /** 195 * Create a new {@link Decompressor} for use by this {@link CompressionCodec}. 196 * 197 * @return a new decompressor for use by this codec 198 */ 199 @Override 200 public Decompressor createDecompressor() { 201 checkNativeCodeLoaded(); 202 int bufferSize = conf.getInt( 203 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, 204 CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); 205 return new SnappyDecompressor(bufferSize); 206 } 207 208 /** 209 * {@inheritDoc} 210 */ 211 @Override 212 public DirectDecompressor createDirectDecompressor() { 213 return isNativeCodeLoaded() ? new SnappyDirectDecompressor() : null; 214 } 215 216 /** 217 * Get the default filename extension for this kind of compression. 218 * 219 * @return <code>.snappy</code>. 220 */ 221 @Override 222 public String getDefaultExtension() { 223 return ".snappy"; 224 } 225}