001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, 013 * software distributed under the License is distributed on an 014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 015 * KIND, either express or implied. See the License for the 016 * specific language governing permissions and limitations 017 * under the License. 018 */ 019package org.apache.commons.compress.compressors.lz4; 020 021import java.io.IOException; 022import java.io.OutputStream; 023import java.util.Arrays; 024import java.util.Deque; 025import java.util.Iterator; 026import java.util.LinkedList; 027 028import org.apache.commons.compress.compressors.CompressorOutputStream; 029import org.apache.commons.compress.compressors.lz77support.LZ77Compressor; 030import org.apache.commons.compress.compressors.lz77support.Parameters; 031import org.apache.commons.compress.utils.ByteUtils; 032 033/** 034 * CompressorOutputStream for the LZ4 block format. 035 * 036 * @see <a href="http://lz4.github.io/lz4/lz4_Block_format.html">LZ4 Block Format Description</a> 037 * @since 1.14 038 * @NotThreadSafe 039 */ 040public class BlockLZ4CompressorOutputStream extends CompressorOutputStream { 041 042 private static final int MIN_BACK_REFERENCE_LENGTH = 4; 043 private static final int MIN_OFFSET_OF_LAST_BACK_REFERENCE = 12; 044 045 /* 046 047 The LZ4 block format has a few properties that make it less 048 straight-forward than one would hope: 049 050 * literal blocks and back-references must come in pairs (except 051 for the very last literal block), so consecutive literal 052 blocks created by the compressor must be merged into a single 053 block. 054 055 * the start of a literal/back-reference pair contains the length 056 of the back-reference (at least some part of it) so we can't 057 start writing the literal before we know how long the next 058 back-reference is going to be. 059 060 * there are special rules for the final blocks 061 062 > There are specific parsing rules to respect in order to remain 063 > compatible with assumptions made by the decoder : 064 > 065 > 1. The last 5 bytes are always literals 066 > 067 > 2. The last match must start at least 12 bytes before end of 068 > block. Consequently, a block with less than 13 bytes cannot be 069 > compressed. 070 071 which means any back-reference may need to get rewritten as a 072 literal block unless we know the next block is at least of 073 length 5 and the sum of this block's length and offset and the 074 next block's length is at least twelve. 075 076 */ 077 078 private final LZ77Compressor compressor; 079 private final OutputStream os; 080 081 // used in one-arg write method 082 private final byte[] oneByte = new byte[1]; 083 084 private boolean finished = false; 085 086 private Deque<Pair> pairs = new LinkedList<>(); 087 // keeps track of the last window-size bytes (64k) in order to be 088 // able to expand back-references when needed 089 private Deque<byte[]> expandedBlocks = new LinkedList<>(); 090 091 /** 092 * Creates a new LZ4 output stream. 093 * 094 * @param os 095 * An OutputStream to read compressed data from 096 * 097 * @throws IOException if reading fails 098 */ 099 public BlockLZ4CompressorOutputStream(final OutputStream os) throws IOException { 100 this(os, createParameterBuilder().build()); 101 } 102 103 /** 104 * Creates a new LZ4 output stream. 105 * 106 * @param os 107 * An OutputStream to read compressed data from 108 * @param params 109 * The parameters to use for LZ77 compression. 110 * 111 * @throws IOException if reading fails 112 */ 113 public BlockLZ4CompressorOutputStream(final OutputStream os, Parameters params) throws IOException { 114 this.os = os; 115 compressor = new LZ77Compressor(params, 116 new LZ77Compressor.Callback() { 117 @Override 118 public void accept(LZ77Compressor.Block block) throws IOException { 119 switch (block.getType()) { 120 case LITERAL: 121 addLiteralBlock((LZ77Compressor.LiteralBlock) block); 122 break; 123 case BACK_REFERENCE: 124 addBackReference((LZ77Compressor.BackReference) block); 125 break; 126 case EOD: 127 writeFinalLiteralBlock(); 128 break; 129 } 130 } 131 }); 132 } 133 134 @Override 135 public void write(int b) throws IOException { 136 oneByte[0] = (byte) (b & 0xff); 137 write(oneByte); 138 } 139 140 @Override 141 public void write(byte[] data, int off, int len) throws IOException { 142 compressor.compress(data, off, len); 143 } 144 145 @Override 146 public void close() throws IOException { 147 finish(); 148 os.close(); 149 } 150 151 /** 152 * Compresses all remaining data and writes it to the stream, 153 * doesn't close the underlying stream. 154 * @throws IOException if an error occurs 155 */ 156 public void finish() throws IOException { 157 if (!finished) { 158 compressor.finish(); 159 finished = true; 160 } 161 } 162 163 /** 164 * Adds some initial data to fill the window with. 165 * 166 * @param data the data to fill the window with. 167 * @param off offset of real data into the array 168 * @param len amount of data 169 * @throws IllegalStateException if the stream has already started to write data 170 * @see LZ77Compressor#prefill 171 */ 172 public void prefill(byte[] data, int off, int len) { 173 if (len > 0) { 174 byte[] b = Arrays.copyOfRange(data, off, off + len); 175 compressor.prefill(b); 176 recordLiteral(b); 177 } 178 } 179 180 private void addLiteralBlock(LZ77Compressor.LiteralBlock block) throws IOException { 181 Pair last = writeBlocksAndReturnUnfinishedPair(block.getLength()); 182 recordLiteral(last.addLiteral(block)); 183 clearUnusedBlocksAndPairs(); 184 } 185 186 private void addBackReference(LZ77Compressor.BackReference block) throws IOException { 187 Pair last = writeBlocksAndReturnUnfinishedPair(block.getLength()); 188 last.setBackReference(block); 189 recordBackReference(block); 190 clearUnusedBlocksAndPairs(); 191 } 192 193 private Pair writeBlocksAndReturnUnfinishedPair(int length) throws IOException { 194 writeWritablePairs(length); 195 Pair last = pairs.peekLast(); 196 if (last == null || last.hasBackReference()) { 197 last = new Pair(); 198 pairs.addLast(last); 199 } 200 return last; 201 } 202 203 private void recordLiteral(byte[] b) { 204 expandedBlocks.addFirst(b); 205 } 206 207 private void clearUnusedBlocksAndPairs() { 208 clearUnusedBlocks(); 209 clearUnusedPairs(); 210 } 211 212 private void clearUnusedBlocks() { 213 int blockLengths = 0; 214 int blocksToKeep = 0; 215 for (byte[] b : expandedBlocks) { 216 blocksToKeep++; 217 blockLengths += b.length; 218 if (blockLengths >= BlockLZ4CompressorInputStream.WINDOW_SIZE) { 219 break; 220 } 221 } 222 final int size = expandedBlocks.size(); 223 for (int i = blocksToKeep; i < size; i++) { 224 expandedBlocks.removeLast(); 225 } 226 } 227 228 private void recordBackReference(LZ77Compressor.BackReference block) { 229 expandedBlocks.addFirst(expand(block.getOffset(), block.getLength())); 230 } 231 232 private byte[] expand(final int offset, final int length) { 233 byte[] expanded = new byte[length]; 234 if (offset == 1) { // surprisingly common special case 235 byte[] block = expandedBlocks.peekFirst(); 236 byte b = block[block.length - 1]; 237 if (b != 0) { // the fresh array contains 0s anyway 238 Arrays.fill(expanded, b); 239 } 240 } else { 241 expandFromList(expanded, offset, length); 242 } 243 return expanded; 244 } 245 246 private void expandFromList(final byte[] expanded, int offset, int length) { 247 int offsetRemaining = offset; 248 int lengthRemaining = length; 249 int writeOffset = 0; 250 while (lengthRemaining > 0) { 251 // find block that contains offsetRemaining 252 byte[] block = null; 253 int copyLen, copyOffset; 254 if (offsetRemaining > 0) { 255 int blockOffset = 0; 256 for (byte[] b : expandedBlocks) { 257 if (b.length + blockOffset >= offsetRemaining) { 258 block = b; 259 break; 260 } 261 blockOffset += b.length; 262 } 263 if (block == null) { 264 // should not be possible 265 throw new IllegalStateException("failed to find a block containing offset " + offset); 266 } 267 copyOffset = blockOffset + block.length - offsetRemaining; 268 copyLen = Math.min(lengthRemaining, block.length - copyOffset); 269 } else { 270 // offsetRemaining is negative or 0 and points into the expanded bytes 271 block = expanded; 272 copyOffset = -offsetRemaining; 273 copyLen = Math.min(lengthRemaining, writeOffset + offsetRemaining); 274 } 275 System.arraycopy(block, copyOffset, expanded, writeOffset, copyLen); 276 offsetRemaining -= copyLen; 277 lengthRemaining -= copyLen; 278 writeOffset += copyLen; 279 } 280 } 281 282 private void clearUnusedPairs() { 283 int pairLengths = 0; 284 int pairsToKeep = 0; 285 for (Iterator<Pair> it = pairs.descendingIterator(); it.hasNext(); ) { 286 Pair p = it.next(); 287 pairsToKeep++; 288 pairLengths += p.length(); 289 if (pairLengths >= BlockLZ4CompressorInputStream.WINDOW_SIZE) { 290 break; 291 } 292 } 293 final int size = pairs.size(); 294 for (int i = pairsToKeep; i < size; i++) { 295 Pair p = pairs.peekFirst(); 296 if (p.hasBeenWritten()) { 297 pairs.removeFirst(); 298 } else { 299 break; 300 } 301 } 302 } 303 304 private void writeFinalLiteralBlock() throws IOException { 305 rewriteLastPairs(); 306 for (Pair p : pairs) { 307 if (!p.hasBeenWritten()) { 308 p.writeTo(os); 309 } 310 } 311 pairs.clear(); 312 } 313 314 private void writeWritablePairs(int lengthOfBlocksAfterLastPair) throws IOException { 315 int unwrittenLength = lengthOfBlocksAfterLastPair; 316 for (Iterator<Pair> it = pairs.descendingIterator(); it.hasNext(); ) { 317 Pair p = it.next(); 318 if (p.hasBeenWritten()) { 319 break; 320 } 321 unwrittenLength += p.length(); 322 } 323 for (Pair p : pairs) { 324 if (p.hasBeenWritten()) { 325 continue; 326 } 327 unwrittenLength -= p.length(); 328 if (p.canBeWritten(unwrittenLength)) { 329 p.writeTo(os); 330 } else { 331 break; 332 } 333 } 334 } 335 336 private void rewriteLastPairs() { 337 LinkedList<Pair> lastPairs = new LinkedList<>(); 338 LinkedList<Integer> pairLength = new LinkedList<>(); 339 int offset = 0; 340 for (Iterator<Pair> it = pairs.descendingIterator(); it.hasNext(); ) { 341 Pair p = it.next(); 342 if (p.hasBeenWritten()) { 343 break; 344 } 345 int len = p.length(); 346 pairLength.addFirst(len); 347 lastPairs.addFirst(p); 348 offset += len; 349 if (offset >= MIN_OFFSET_OF_LAST_BACK_REFERENCE) { 350 break; 351 } 352 } 353 for (Pair p : lastPairs) { 354 pairs.remove(p); 355 } 356 // lastPairs may contain between one and four Pairs: 357 // * the last pair may be a one byte literal 358 // * all other Pairs contain a back-reference which must be four bytes long at minimum 359 // we could merge them all into a single literal block but 360 // this may harm compression. For example compressing 361 // "bla.tar" from our tests yields a last block containing a 362 // back-reference of length > 2k and we'd end up with a last 363 // literal of that size rather than a 2k back-reference and a 364 // 12 byte literal at the end. 365 366 // Instead we merge all but the first of lastPairs into a new 367 // literal-only Pair "replacement" and look at the 368 // back-reference in the first of lastPairs and see if we can 369 // split it. We can split it if it is longer than 16 - 370 // replacement.length (i.e. the minimal length of four is kept 371 // while making sure the last literal is at least twelve bytes 372 // long). If we can't split it, we expand the first of the pairs 373 // as well. 374 375 // this is not optimal, we could get better compression 376 // results with more complex approaches as the last literal 377 // only needs to be five bytes long if the previous 378 // back-reference has an offset big enough 379 380 final int lastPairsSize = lastPairs.size(); 381 int toExpand = 0; 382 for (int i = 1; i < lastPairsSize; i++) { 383 toExpand += pairLength.get(i); 384 } 385 Pair replacement = new Pair(); 386 if (toExpand > 0) { 387 replacement.prependLiteral(expand(toExpand, toExpand)); 388 } 389 Pair splitCandidate = lastPairs.get(0); 390 int stillNeeded = MIN_OFFSET_OF_LAST_BACK_REFERENCE - toExpand; 391 int brLen = splitCandidate.hasBackReference() ? splitCandidate.backReferenceLength() : 0; 392 if (splitCandidate.hasBackReference() && brLen >= MIN_BACK_REFERENCE_LENGTH + stillNeeded) { 393 replacement.prependLiteral(expand(toExpand + stillNeeded, stillNeeded)); 394 pairs.add(splitCandidate.splitWithNewBackReferenceLengthOf(brLen - stillNeeded)); 395 } else { 396 if (splitCandidate.hasBackReference()) { 397 replacement.prependLiteral(expand(toExpand + brLen, brLen)); 398 } 399 splitCandidate.prependTo(replacement); 400 } 401 pairs.add(replacement); 402 } 403 404 /** 405 * Returns a builder correctly configured for the LZ4 algorithm. 406 * @return a builder correctly configured for the LZ4 algorithm 407 */ 408 public static Parameters.Builder createParameterBuilder() { 409 int maxLen = BlockLZ4CompressorInputStream.WINDOW_SIZE - 1; 410 return Parameters.builder(BlockLZ4CompressorInputStream.WINDOW_SIZE) 411 .withMinBackReferenceLength(MIN_BACK_REFERENCE_LENGTH) 412 .withMaxBackReferenceLength(maxLen) 413 .withMaxOffset(maxLen) 414 .withMaxLiteralLength(maxLen); 415 } 416 417 final static class Pair { 418 private final Deque<byte[]> literals = new LinkedList<>(); 419 private int brOffset, brLength; 420 private boolean written; 421 422 private void prependLiteral(byte[] data) { 423 literals.addFirst(data); 424 } 425 byte[] addLiteral(LZ77Compressor.LiteralBlock block) { 426 byte[] copy = Arrays.copyOfRange(block.getData(), block.getOffset(), 427 block.getOffset() + block.getLength()); 428 literals.add(copy); 429 return copy; 430 } 431 void setBackReference(LZ77Compressor.BackReference block) { 432 if (hasBackReference()) { 433 throw new IllegalStateException(); 434 } 435 brOffset = block.getOffset(); 436 brLength = block.getLength(); 437 } 438 boolean hasBackReference() { 439 return brOffset > 0; 440 } 441 boolean canBeWritten(int lengthOfBlocksAfterThisPair) { 442 return hasBackReference() 443 && lengthOfBlocksAfterThisPair >= MIN_OFFSET_OF_LAST_BACK_REFERENCE + MIN_BACK_REFERENCE_LENGTH; 444 } 445 int length() { 446 return literalLength() + brLength; 447 } 448 private boolean hasBeenWritten() { 449 return written; 450 } 451 void writeTo(OutputStream out) throws IOException { 452 int litLength = literalLength(); 453 out.write(lengths(litLength, brLength)); 454 if (litLength >= BlockLZ4CompressorInputStream.BACK_REFERENCE_SIZE_MASK) { 455 writeLength(litLength - BlockLZ4CompressorInputStream.BACK_REFERENCE_SIZE_MASK, out); 456 } 457 for (byte[] b : literals) { 458 out.write(b); 459 } 460 if (hasBackReference()) { 461 ByteUtils.toLittleEndian(out, brOffset, 2); 462 if (brLength - MIN_BACK_REFERENCE_LENGTH >= BlockLZ4CompressorInputStream.BACK_REFERENCE_SIZE_MASK) { 463 writeLength(brLength - MIN_BACK_REFERENCE_LENGTH 464 - BlockLZ4CompressorInputStream.BACK_REFERENCE_SIZE_MASK, out); 465 } 466 } 467 written = true; 468 } 469 private int literalLength() { 470 int length = 0; 471 for (byte[] b : literals) { 472 length += b.length; 473 } 474 return length; 475 } 476 private static int lengths(int litLength, int brLength) { 477 int l = litLength < 15 ? litLength : 15; 478 int br = brLength < 4 ? 0 : (brLength < 19 ? brLength - 4 : 15); 479 return (l << BlockLZ4CompressorInputStream.SIZE_BITS) | br; 480 } 481 private static void writeLength(int length, OutputStream out) throws IOException { 482 while (length >= 255) { 483 out.write(255); 484 length -= 255; 485 } 486 out.write(length); 487 } 488 private int backReferenceLength() { 489 return brLength; 490 } 491 private void prependTo(Pair other) { 492 Iterator<byte[]> listBackwards = literals.descendingIterator(); 493 while (listBackwards.hasNext()) { 494 other.prependLiteral(listBackwards.next()); 495 } 496 } 497 private Pair splitWithNewBackReferenceLengthOf(int newBackReferenceLength) { 498 Pair p = new Pair(); 499 p.literals.addAll(literals); 500 p.brOffset = brOffset; 501 p.brLength = newBackReferenceLength; 502 return p; 503 } 504 } 505}