Source code

001/*
002 *  Licensed to the Apache Software Foundation (ASF) under one or more
003 *  contributor license agreements.  See the NOTICE file distributed with
004 *  this work for additional information regarding copyright ownership.
005 *  The ASF licenses this file to You under the Apache License, Version 2.0
006 *  (the "License"); you may not use this file except in compliance with
007 *  the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 *  Unless required by applicable law or agreed to in writing, software
012 *  distributed under the License is distributed on an "AS IS" BASIS,
013 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 *  See the License for the specific language governing permissions and
015 *  limitations under the License.
016 *
017 */
018package org.apache.commons.compress.archivers.zip;
019
020
021import java.io.Closeable;
022import java.io.File;
023import java.io.FileNotFoundException;
024import java.io.IOException;
025import java.io.InputStream;
026import java.nio.file.Path;
027import java.util.Iterator;
028import java.util.Queue;
029import java.util.concurrent.ConcurrentLinkedQueue;
030import java.util.concurrent.atomic.AtomicBoolean;
031import java.util.zip.Deflater;
032
033import org.apache.commons.compress.parallel.FileBasedScatterGatherBackingStore;
034import org.apache.commons.compress.parallel.ScatterGatherBackingStore;
035import org.apache.commons.compress.utils.BoundedInputStream;
036
037/**
038 * A ZIP output stream that is optimized for multi-threaded scatter/gather construction of ZIP files.
039 * <p>
040 * The internal data format of the entries used by this class are entirely private to this class and are not part of any public api whatsoever.
041 * </p>
042 * <p>
043 * It is possible to extend this class to support different kinds of backing storage, the default implementation only supports file-based backing.
044 * </p>
045 * <p>
046 * Thread safety: This class supports multiple threads. But the "writeTo" method must be called by the thread that originally created the
047 * {@link ZipArchiveEntry}.
048 * </p>
049 *
050 * @since 1.10
051 */
052public class ScatterZipOutputStream implements Closeable {
053
054    private static class CompressedEntry {
055        final ZipArchiveEntryRequest zipArchiveEntryRequest;
056        final long crc;
057        final long compressedSize;
058        final long size;
059
060        public CompressedEntry(final ZipArchiveEntryRequest zipArchiveEntryRequest, final long crc, final long compressedSize, final long size) {
061            this.zipArchiveEntryRequest = zipArchiveEntryRequest;
062            this.crc = crc;
063            this.compressedSize = compressedSize;
064            this.size = size;
065        }
066
067        /**
068         * Updates the original {@link ZipArchiveEntry} with sizes/crc
069         * Do not use this methods from threads that did not create the instance itself !
070         * @return the zipArchiveEntry that is basis for this request
071         */
072
073        public ZipArchiveEntry transferToArchiveEntry() {
074            final ZipArchiveEntry entry = zipArchiveEntryRequest.getZipArchiveEntry();
075            entry.setCompressedSize(compressedSize);
076            entry.setSize(size);
077            entry.setCrc(crc);
078            entry.setMethod(zipArchiveEntryRequest.getMethod());
079            return entry;
080        }
081    }
082
083    public static class ZipEntryWriter implements Closeable {
084        private final Iterator<CompressedEntry> itemsIterator;
085        private final InputStream itemsIteratorData;
086
087        public ZipEntryWriter(final ScatterZipOutputStream scatter) throws IOException {
088            scatter.backingStore.closeForWriting();
089            itemsIterator = scatter.items.iterator();
090            itemsIteratorData = scatter.backingStore.getInputStream();
091        }
092
093        @Override
094        public void close() throws IOException {
095            if (itemsIteratorData != null) {
096                itemsIteratorData.close();
097            }
098        }
099
100        public void writeNextZipEntry(final ZipArchiveOutputStream target) throws IOException {
101            final CompressedEntry compressedEntry = itemsIterator.next();
102            try (final BoundedInputStream rawStream = new BoundedInputStream(itemsIteratorData, compressedEntry.compressedSize)) {
103                target.addRawArchiveEntry(compressedEntry.transferToArchiveEntry(), rawStream);
104            }
105        }
106    }
107
108    /**
109     * Creates a {@link ScatterZipOutputStream} with default compression level that is backed by a file
110     *
111     * @param file The file to offload compressed data into.
112     * @return A ScatterZipOutputStream that is ready for use.
113     * @throws FileNotFoundException if the file cannot be found
114     */
115    public static ScatterZipOutputStream fileBased(final File file) throws FileNotFoundException {
116        return pathBased(file.toPath(), Deflater.DEFAULT_COMPRESSION);
117    }
118
119    /**
120     * Creates a {@link ScatterZipOutputStream} that is backed by a file
121     *
122     * @param file             The file to offload compressed data into.
123     * @param compressionLevel The compression level to use, @see #Deflater
124     * @return A  ScatterZipOutputStream that is ready for use.
125     * @throws FileNotFoundException if the file cannot be found
126     */
127    public static ScatterZipOutputStream fileBased(final File file, final int compressionLevel) throws FileNotFoundException {
128        return pathBased(file.toPath(), compressionLevel);
129    }
130
131    /**
132     * Creates a {@link ScatterZipOutputStream} with default compression level that is backed by a file
133     * @param path The path to offload compressed data into.
134     * @return A ScatterZipOutputStream that is ready for use.
135     * @throws FileNotFoundException if the path cannot be found
136     * @since 1.22
137     */
138    public static ScatterZipOutputStream pathBased(final Path path) throws FileNotFoundException {
139        return pathBased(path, Deflater.DEFAULT_COMPRESSION);
140    }
141
142    /**
143     * Creates a {@link ScatterZipOutputStream} that is backed by a file
144     * @param path The path to offload compressed data into.
145     * @param compressionLevel The compression level to use, @see #Deflater
146     * @return A ScatterZipOutputStream that is ready for use.
147     * @throws FileNotFoundException if the path cannot be found
148     * @since 1.22
149     */
150    public static ScatterZipOutputStream pathBased(final Path path, final int compressionLevel) throws FileNotFoundException {
151        final ScatterGatherBackingStore bs = new FileBasedScatterGatherBackingStore(path);
152        // lifecycle is bound to the ScatterZipOutputStream returned
153        final StreamCompressor sc = StreamCompressor.create(compressionLevel, bs); //NOSONAR
154        return new ScatterZipOutputStream(bs, sc);
155    }
156
157    private final Queue<CompressedEntry> items = new ConcurrentLinkedQueue<>();
158
159    private final ScatterGatherBackingStore backingStore;
160
161    private final StreamCompressor streamCompressor;
162
163    private final AtomicBoolean isClosed = new AtomicBoolean();
164
165    private ZipEntryWriter zipEntryWriter;
166
167    public ScatterZipOutputStream(final ScatterGatherBackingStore backingStore,
168                                  final StreamCompressor streamCompressor) {
169        this.backingStore = backingStore;
170        this.streamCompressor = streamCompressor;
171    }
172
173    /**
174     * Adds an archive entry to this scatter stream.
175     *
176     * @param zipArchiveEntryRequest The entry to write.
177     * @throws IOException    If writing fails
178     */
179    public void addArchiveEntry(final ZipArchiveEntryRequest zipArchiveEntryRequest) throws IOException {
180        try (final InputStream payloadStream = zipArchiveEntryRequest.getPayloadStream()) {
181            streamCompressor.deflate(payloadStream, zipArchiveEntryRequest.getMethod());
182        }
183        items.add(new CompressedEntry(zipArchiveEntryRequest, streamCompressor.getCrc32(),
184                                      streamCompressor.getBytesWrittenForLastEntry(), streamCompressor.getBytesRead()));
185    }
186
187    /**
188     * Closes this stream, freeing all resources involved in the creation of this stream.
189     * @throws IOException If closing fails
190     */
191    @Override
192    public void close() throws IOException {
193        if (!isClosed.compareAndSet(false, true)) {
194            return;
195        }
196        try {
197            if (zipEntryWriter != null) {
198                zipEntryWriter.close();
199            }
200            backingStore.close();
201        } finally {
202            streamCompressor.close();
203        }
204    }
205
206    /**
207     * Writes the contents of this scatter stream to a target archive.
208     *
209     * @param target The archive to receive the contents of this {@link ScatterZipOutputStream}.
210     * @throws IOException If writing fails
211     * @see #zipEntryWriter()
212     */
213    public void writeTo(final ZipArchiveOutputStream target) throws IOException {
214        backingStore.closeForWriting();
215        try (final InputStream data = backingStore.getInputStream()) {
216            for (final CompressedEntry compressedEntry : items) {
217                try (final BoundedInputStream rawStream = new BoundedInputStream(data,
218                        compressedEntry.compressedSize)) {
219                    target.addRawArchiveEntry(compressedEntry.transferToArchiveEntry(), rawStream);
220                }
221            }
222        }
223    }
224
225    /**
226     * Gets a ZIP entry writer for this scatter stream.
227     * @throws IOException If getting scatter stream input stream
228     * @return the ZipEntryWriter created on first call of the method
229     */
230    public ZipEntryWriter zipEntryWriter() throws IOException {
231        if (zipEntryWriter == null) {
232            zipEntryWriter = new ZipEntryWriter(this);
233        }
234        return zipEntryWriter;
235    }
236}