001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 * 017 */ 018package org.apache.commons.compress.archivers.zip; 019 020 021import org.apache.commons.compress.parallel.FileBasedScatterGatherBackingStore; 022import org.apache.commons.compress.parallel.ScatterGatherBackingStore; 023import org.apache.commons.compress.utils.BoundedInputStream; 024 025import java.io.Closeable; 026import java.io.File; 027import java.io.FileNotFoundException; 028import java.io.IOException; 029import java.io.InputStream; 030import java.util.Iterator; 031import java.util.Queue; 032import java.util.concurrent.ConcurrentLinkedQueue; 033import java.util.concurrent.atomic.AtomicBoolean; 034import java.util.zip.Deflater; 035 036/** 037 * A zip output stream that is optimized for multi-threaded scatter/gather construction of zip files. 038 * <p> 039 * The internal data format of the entries used by this class are entirely private to this class 040 * and are not part of any public api whatsoever. 041 * </p> 042 * <p>It is possible to extend this class to support different kinds of backing storage, the default 043 * implementation only supports file-based backing. 044 * </p> 045 * Thread safety: This class supports multiple threads. But the "writeTo" method must be called 046 * by the thread that originally created the {@link ZipArchiveEntry}. 047 * 048 * @since 1.10 049 */ 050public class ScatterZipOutputStream implements Closeable { 051 private final Queue<CompressedEntry> items = new ConcurrentLinkedQueue<>(); 052 private final ScatterGatherBackingStore backingStore; 053 private final StreamCompressor streamCompressor; 054 private AtomicBoolean isClosed = new AtomicBoolean(); 055 private ZipEntryWriter zipEntryWriter = null; 056 057 private static class CompressedEntry { 058 final ZipArchiveEntryRequest zipArchiveEntryRequest; 059 final long crc; 060 final long compressedSize; 061 final long size; 062 063 public CompressedEntry(final ZipArchiveEntryRequest zipArchiveEntryRequest, final long crc, final long compressedSize, final long size) { 064 this.zipArchiveEntryRequest = zipArchiveEntryRequest; 065 this.crc = crc; 066 this.compressedSize = compressedSize; 067 this.size = size; 068 } 069 070 /** 071 * Update the original {@link ZipArchiveEntry} with sizes/crc 072 * Do not use this methods from threads that did not create the instance itself ! 073 * @return the zipArchiveEntry that is basis for this request 074 */ 075 076 public ZipArchiveEntry transferToArchiveEntry(){ 077 final ZipArchiveEntry entry = zipArchiveEntryRequest.getZipArchiveEntry(); 078 entry.setCompressedSize(compressedSize); 079 entry.setSize(size); 080 entry.setCrc(crc); 081 entry.setMethod(zipArchiveEntryRequest.getMethod()); 082 return entry; 083 } 084 } 085 086 public ScatterZipOutputStream(final ScatterGatherBackingStore backingStore, 087 final StreamCompressor streamCompressor) { 088 this.backingStore = backingStore; 089 this.streamCompressor = streamCompressor; 090 } 091 092 /** 093 * Add an archive entry to this scatter stream. 094 * 095 * @param zipArchiveEntryRequest The entry to write. 096 * @throws IOException If writing fails 097 */ 098 public void addArchiveEntry(final ZipArchiveEntryRequest zipArchiveEntryRequest) throws IOException { 099 try (final InputStream payloadStream = zipArchiveEntryRequest.getPayloadStream()) { 100 streamCompressor.deflate(payloadStream, zipArchiveEntryRequest.getMethod()); 101 } 102 items.add(new CompressedEntry(zipArchiveEntryRequest, streamCompressor.getCrc32(), 103 streamCompressor.getBytesWrittenForLastEntry(), streamCompressor.getBytesRead())); 104 } 105 106 /** 107 * Write the contents of this scatter stream to a target archive. 108 * 109 * @param target The archive to receive the contents of this {@link ScatterZipOutputStream}. 110 * @throws IOException If writing fails 111 * @see #zipEntryWriter() 112 */ 113 public void writeTo(final ZipArchiveOutputStream target) throws IOException { 114 backingStore.closeForWriting(); 115 try (final InputStream data = backingStore.getInputStream()) { 116 for (final CompressedEntry compressedEntry : items) { 117 try (final BoundedInputStream rawStream = new BoundedInputStream(data, 118 compressedEntry.compressedSize)) { 119 target.addRawArchiveEntry(compressedEntry.transferToArchiveEntry(), rawStream); 120 } 121 } 122 } 123 } 124 125 public static class ZipEntryWriter implements Closeable { 126 private final Iterator<CompressedEntry> itemsIterator; 127 private final InputStream itemsIteratorData; 128 129 public ZipEntryWriter(ScatterZipOutputStream scatter) throws IOException { 130 scatter.backingStore.closeForWriting(); 131 itemsIterator = scatter.items.iterator(); 132 itemsIteratorData = scatter.backingStore.getInputStream(); 133 } 134 135 @Override 136 public void close() throws IOException { 137 if (itemsIteratorData != null) { 138 itemsIteratorData.close(); 139 } 140 } 141 142 public void writeNextZipEntry(final ZipArchiveOutputStream target) throws IOException { 143 CompressedEntry compressedEntry = itemsIterator.next(); 144 try (final BoundedInputStream rawStream = new BoundedInputStream(itemsIteratorData, compressedEntry.compressedSize)) { 145 target.addRawArchiveEntry(compressedEntry.transferToArchiveEntry(), rawStream); 146 } 147 } 148 } 149 150 /** 151 * Get a zip entry writer for this scatter stream. 152 * @throws IOException If getting scatter stream input stream 153 * @return the ZipEntryWriter created on first call of the method 154 */ 155 public ZipEntryWriter zipEntryWriter() throws IOException { 156 if (zipEntryWriter == null) { 157 zipEntryWriter = new ZipEntryWriter(this); 158 } 159 return zipEntryWriter; 160 } 161 162 /** 163 * Closes this stream, freeing all resources involved in the creation of this stream. 164 * @throws IOException If closing fails 165 */ 166 @Override 167 public void close() throws IOException { 168 if (!isClosed.compareAndSet(false, true)) { 169 return; 170 } 171 try { 172 if (zipEntryWriter != null) { 173 zipEntryWriter.close(); 174 } 175 backingStore.close(); 176 } finally { 177 streamCompressor.close(); 178 } 179 } 180 181 /** 182 * Create a {@link ScatterZipOutputStream} with default compression level that is backed by a file 183 * 184 * @param file The file to offload compressed data into. 185 * @return A ScatterZipOutputStream that is ready for use. 186 * @throws FileNotFoundException if the file cannot be found 187 */ 188 public static ScatterZipOutputStream fileBased(final File file) throws FileNotFoundException { 189 return fileBased(file, Deflater.DEFAULT_COMPRESSION); 190 } 191 192 /** 193 * Create a {@link ScatterZipOutputStream} that is backed by a file 194 * 195 * @param file The file to offload compressed data into. 196 * @param compressionLevel The compression level to use, @see #Deflater 197 * @return A ScatterZipOutputStream that is ready for use. 198 * @throws FileNotFoundException if the file cannot be found 199 */ 200 public static ScatterZipOutputStream fileBased(final File file, final int compressionLevel) throws FileNotFoundException { 201 final ScatterGatherBackingStore bs = new FileBasedScatterGatherBackingStore(file); 202 // lifecycle is bound to the ScatterZipOutputStream returned 203 final StreamCompressor sc = StreamCompressor.create(compressionLevel, bs); //NOSONAR 204 return new ScatterZipOutputStream(bs, sc); 205 } 206}