001package com.box.sdk;
002
003import com.box.sdk.http.HttpMethod;
004import com.eclipsesource.json.Json;
005import com.eclipsesource.json.JsonObject;
006import java.io.IOException;
007import java.io.InputStream;
008import java.net.URL;
009import java.security.DigestInputStream;
010import java.security.MessageDigest;
011import java.security.NoSuchAlgorithmException;
012import java.util.ArrayList;
013import java.util.List;
014import java.util.Map;
015import java.util.concurrent.Executors;
016import java.util.concurrent.ThreadPoolExecutor;
017import java.util.concurrent.TimeUnit;
018
019/**
020 * Utility class for uploading large files.
021 */
022public final class LargeFileUpload {
023
024    private static final String DIGEST_HEADER_PREFIX_SHA = "sha=";
025    private static final String DIGEST_ALGORITHM_SHA1 = "SHA1";
026
027    private static final String OFFSET_QUERY_STRING = "offset";
028    private static final String LIMIT_QUERY_STRING = "limit";
029    private static final int DEFAULT_CONNECTIONS = 3;
030    private static final int DEFAULT_TIMEOUT = 1;
031    private static final TimeUnit DEFAULT_TIMEUNIT = TimeUnit.HOURS;
032    private static final int THREAD_POOL_WAIT_TIME_IN_MILLIS = 1000;
033    private final ThreadPoolExecutor executorService;
034    private final long timeout;
035    private final TimeUnit timeUnit;
036    private int connections;
037
038    /**
039     * Creates a LargeFileUpload object.
040     *
041     * @param nParallelConnections number of parallel http connections to use
042     * @param timeOut              time to wait before killing the job
043     * @param unit                 time unit for the time wait value
044     */
045    public LargeFileUpload(int nParallelConnections, long timeOut, TimeUnit unit) {
046        this.executorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(nParallelConnections);
047        this.timeout = timeOut;
048        this.timeUnit = unit;
049    }
050
051    /**
052     * Creates a LargeFileUpload object with a default number of parallel conections and timeout.
053     */
054    public LargeFileUpload() {
055        this.executorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(LargeFileUpload.DEFAULT_CONNECTIONS);
056        this.timeout = LargeFileUpload.DEFAULT_TIMEOUT;
057        this.timeUnit = LargeFileUpload.DEFAULT_TIMEUNIT;
058    }
059
060    private static byte[] getBytesFromStream(InputStream stream, int numBytes) {
061
062        int bytesNeeded = numBytes;
063        int offset = 0;
064        byte[] bytes = new byte[numBytes];
065
066        while (bytesNeeded > 0) {
067
068            int bytesRead;
069            try {
070                bytesRead = stream.read(bytes, offset, bytesNeeded);
071            } catch (IOException ioe) {
072                throw new BoxAPIException("Reading data from stream failed.", ioe);
073            }
074
075            if (bytesRead == -1) {
076                throw new BoxAPIException("Stream ended while upload was progressing");
077            }
078
079            bytesNeeded = bytesNeeded - bytesRead;
080            offset = offset + bytesRead;
081        }
082
083        return bytes;
084    }
085
086    private BoxFileUploadSession.Info createUploadSession(BoxAPIConnection boxApi, String folderId,
087                                                          URL url, String fileName, long fileSize) {
088
089        BoxJSONRequest request = new BoxJSONRequest(boxApi, url, HttpMethod.POST);
090
091        //Create the JSON body of the request
092        JsonObject body = new JsonObject();
093        body.add("folder_id", folderId);
094        body.add("file_name", fileName);
095        body.add("file_size", fileSize);
096        request.setBody(body.toString());
097
098        BoxJSONResponse response = (BoxJSONResponse) request.send();
099        JsonObject jsonObject = Json.parse(response.getJSON()).asObject();
100
101        String sessionId = jsonObject.get("id").asString();
102        BoxFileUploadSession session = new BoxFileUploadSession(boxApi, sessionId);
103
104        return session.new Info(jsonObject);
105    }
106
107    /**
108     * Uploads a new large file.
109     *
110     * @param boxApi   the API connection to be used by the upload session.
111     * @param folderId the id of the folder in which the file will be uploaded.
112     * @param stream   the input stream that feeds the content of the file.
113     * @param url      the upload session URL.
114     * @param fileName the name of the file to be created.
115     * @param fileSize the total size of the file.
116     * @return the created file instance.
117     * @throws InterruptedException when a thread gets interupted.
118     * @throws IOException          when reading a stream throws exception.
119     */
120    public BoxFile.Info upload(BoxAPIConnection boxApi, String folderId, InputStream stream, URL url,
121                               String fileName, long fileSize) throws InterruptedException, IOException {
122        //Create a upload session
123        BoxFileUploadSession.Info session = this.createUploadSession(boxApi, folderId, url, fileName, fileSize);
124        return this.uploadHelper(session, stream, fileSize, null);
125    }
126
127    /**
128     * Uploads a new large file and sets file attributes.
129     *
130     * @param boxApi         the API connection to be used by the upload session.
131     * @param folderId       the id of the folder in which the file will be uploaded.
132     * @param stream         the input stream that feeds the content of the file.
133     * @param url            the upload session URL.
134     * @param fileName       the name of the file to be created.
135     * @param fileSize       the total size of the file.
136     * @param fileAttributes file attributes to set
137     * @return the created file instance.
138     * @throws InterruptedException when a thread gets interupted.
139     * @throws IOException          when reading a stream throws exception.
140     */
141    public BoxFile.Info upload(BoxAPIConnection boxApi, String folderId, InputStream stream, URL url,
142                               String fileName, long fileSize, Map<String, String> fileAttributes)
143        throws InterruptedException, IOException {
144        //Create a upload session
145        BoxFileUploadSession.Info session = this.createUploadSession(boxApi, folderId, url, fileName, fileSize);
146        return this.uploadHelper(session, stream, fileSize, fileAttributes);
147    }
148
149    /**
150     * Creates a new version of a large file.
151     *
152     * @param boxApi   the API connection to be used by the upload session.
153     * @param stream   the input stream that feeds the content of the file.
154     * @param url      the upload session URL.
155     * @param fileSize the total size of the file.
156     * @return the file instance that also contains the version information.
157     * @throws InterruptedException when a thread gets interupted.
158     * @throws IOException          when reading a stream throws exception.
159     */
160    public BoxFile.Info upload(BoxAPIConnection boxApi, InputStream stream, URL url, long fileSize)
161        throws InterruptedException, IOException {
162        //creates a upload session
163        BoxFileUploadSession.Info session = this.createUploadSession(boxApi, url, fileSize);
164        return this.uploadHelper(session, stream, fileSize, null);
165    }
166
167    /**
168     * Creates a new version of a large file and sets file attributes.
169     *
170     * @param boxApi         the API connection to be used by the upload session.
171     * @param stream         the input stream that feeds the content of the file.
172     * @param url            the upload session URL.
173     * @param fileSize       the total size of the file.
174     * @param fileAttributes file attributes to set.
175     * @return the file instance that also contains the version information.
176     * @throws InterruptedException when a thread gets interupted.
177     * @throws IOException          when reading a stream throws exception.
178     */
179    public BoxFile.Info upload(BoxAPIConnection boxApi, InputStream stream, URL url, long fileSize,
180                               Map<String, String> fileAttributes)
181        throws InterruptedException, IOException {
182        //creates a upload session
183        BoxFileUploadSession.Info session = this.createUploadSession(boxApi, url, fileSize);
184        return this.uploadHelper(session, stream, fileSize, fileAttributes);
185    }
186
187    private BoxFile.Info uploadHelper(BoxFileUploadSession.Info session, InputStream stream, long fileSize,
188                                      Map<String, String> fileAttributes)
189        throws InterruptedException {
190        //Upload parts using the upload session
191        MessageDigest digest;
192        try {
193            digest = MessageDigest.getInstance(DIGEST_ALGORITHM_SHA1);
194        } catch (NoSuchAlgorithmException ae) {
195            throw new BoxAPIException("Digest algorithm not found", ae);
196        }
197        DigestInputStream dis = new DigestInputStream(stream, digest);
198        List<BoxFileUploadSessionPart> parts = this.uploadParts(session, dis, fileSize);
199
200        //Creates the file hash
201        byte[] digestBytes = digest.digest();
202        String digestStr = Base64.encode(digestBytes);
203
204        //Commit the upload session. If there is a failure, abort the commit.
205        try {
206            return session.getResource().commit(digestStr, parts, fileAttributes, null, null);
207        } catch (Exception e) {
208            session.getResource().abort();
209            throw new BoxAPIException("Unable to commit the upload session", e);
210        }
211    }
212
213    private BoxFileUploadSession.Info createUploadSession(BoxAPIConnection boxApi, URL url, long fileSize) {
214        BoxJSONRequest request = new BoxJSONRequest(boxApi, url, HttpMethod.POST);
215
216        //Creates the body of the request
217        JsonObject body = new JsonObject();
218        body.add("file_size", fileSize);
219        request.setBody(body.toString());
220
221        BoxJSONResponse response = (BoxJSONResponse) request.send();
222        JsonObject jsonObject = Json.parse(response.getJSON()).asObject();
223
224        String sessionId = jsonObject.get("id").asString();
225        BoxFileUploadSession session = new BoxFileUploadSession(boxApi, sessionId);
226
227        return session.new Info(jsonObject);
228    }
229
230    /*
231     * Upload parts of the file. The part size is retrieved from the upload session.
232     */
233    private List<BoxFileUploadSessionPart> uploadParts(BoxFileUploadSession.Info session, InputStream stream,
234                                                       long fileSize) throws InterruptedException {
235        List<BoxFileUploadSessionPart> parts = new ArrayList<>();
236
237        int partSize = session.getPartSize();
238        long offset = 0;
239        long processed = 0;
240        int partPostion = 0;
241        //Set the Max Queue Size to 1.5x the number of processors
242        double maxQueueSizeDouble = Math.ceil(this.executorService.getMaximumPoolSize() * 1.5);
243        int maxQueueSize = Double.valueOf(maxQueueSizeDouble).intValue();
244        while (processed < fileSize) {
245            //Waiting for any thread to finish before
246            long timeoutForWaitingInMillis = TimeUnit.MILLISECONDS.convert(this.timeout, this.timeUnit);
247            if (this.executorService.getCorePoolSize() <= this.executorService.getActiveCount()) {
248                if (timeoutForWaitingInMillis > 0) {
249                    Thread.sleep(LargeFileUpload.THREAD_POOL_WAIT_TIME_IN_MILLIS);
250                    timeoutForWaitingInMillis -= THREAD_POOL_WAIT_TIME_IN_MILLIS;
251                } else {
252                    throw new BoxAPIException("Upload parts timedout");
253                }
254            }
255            if (this.executorService.getQueue().size() < maxQueueSize) {
256                long diff = fileSize - processed;
257                //The size last part of the file can be lesser than the part size.
258                if (diff < (long) partSize) {
259                    partSize = (int) diff;
260                }
261                parts.add(null);
262                byte[] bytes = getBytesFromStream(stream, partSize);
263                this.executorService.execute(
264                    new LargeFileUploadTask(session.getResource(), bytes, offset,
265                        partSize, fileSize, parts, partPostion)
266                );
267
268                //Increase the offset and proceesed bytes to calculate the Content-Range header.
269                processed += partSize;
270                offset += partSize;
271                partPostion++;
272            }
273        }
274        this.executorService.shutdown();
275        this.executorService.awaitTermination(this.timeout, this.timeUnit);
276        return parts;
277    }
278
279    /**
280     * Generates the Base64 encoded SHA-1 hash for content available in the stream.
281     * It can be used to calculate the hash of a file.
282     *
283     * @param stream the input stream of the file or data.
284     * @return the Base64 encoded hash string.
285     */
286    public String generateDigest(InputStream stream) {
287        MessageDigest digest;
288        try {
289            digest = MessageDigest.getInstance(DIGEST_ALGORITHM_SHA1);
290        } catch (NoSuchAlgorithmException ae) {
291            throw new BoxAPIException("Digest algorithm not found", ae);
292        }
293
294        //Calcuate the digest using the stream.
295        DigestInputStream dis = new DigestInputStream(stream, digest);
296        try {
297            int value = dis.read();
298            while (value != -1) {
299                value = dis.read();
300            }
301        } catch (IOException ioe) {
302            throw new BoxAPIException("Reading the stream failed.", ioe);
303        }
304
305        //Get the calculated digest for the stream
306        byte[] digestBytes = digest.digest();
307        return Base64.encode(digestBytes);
308    }
309}