001package com.box.sdk;
002
003import com.box.sdk.http.HttpMethod;
004import com.eclipsesource.json.JsonObject;
005import java.io.IOException;
006import java.io.InputStream;
007import java.net.URL;
008import java.security.DigestInputStream;
009import java.security.MessageDigest;
010import java.security.NoSuchAlgorithmException;
011import java.util.ArrayList;
012import java.util.List;
013import java.util.Map;
014import java.util.concurrent.Executors;
015import java.util.concurrent.ThreadPoolExecutor;
016import java.util.concurrent.TimeUnit;
017
018/**
019 * Utility class for uploading large files.
020 */
021public final class LargeFileUpload {
022
023    private static final String DIGEST_HEADER_PREFIX_SHA = "sha=";
024    private static final String DIGEST_ALGORITHM_SHA1 = "SHA1";
025
026    private static final String OFFSET_QUERY_STRING = "offset";
027    private static final String LIMIT_QUERY_STRING = "limit";
028    private static final int DEFAULT_CONNECTIONS = 3;
029    private static final int DEFAULT_TIMEOUT = 1;
030    private static final TimeUnit DEFAULT_TIMEUNIT = TimeUnit.HOURS;
031    private static final int THREAD_POOL_WAIT_TIME_IN_MILLIS = 1000;
032    private ThreadPoolExecutor executorService;
033    private long timeout;
034    private TimeUnit timeUnit;
035    private int connections;
036
037    /**
038     * Creates a LargeFileUpload object.
039     *
040     * @param nParallelConnections number of parallel http connections to use
041     * @param timeOut              time to wait before killing the job
042     * @param unit                 time unit for the time wait value
043     */
044    public LargeFileUpload(int nParallelConnections, long timeOut, TimeUnit unit) {
045        this.executorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(nParallelConnections);
046        this.timeout = timeOut;
047        this.timeUnit = unit;
048    }
049
050    /**
051     * Creates a LargeFileUpload object with a default number of parallel conections and timeout.
052     */
053    public LargeFileUpload() {
054        this.executorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(LargeFileUpload.DEFAULT_CONNECTIONS);
055        this.timeout = LargeFileUpload.DEFAULT_TIMEOUT;
056        this.timeUnit = LargeFileUpload.DEFAULT_TIMEUNIT;
057    }
058
059    private static byte[] getBytesFromStream(InputStream stream, int numBytes) {
060
061        int bytesNeeded = numBytes;
062        int offset = 0;
063        byte[] bytes = new byte[numBytes];
064
065        while (bytesNeeded > 0) {
066
067            int bytesRead = -1;
068            try {
069                bytesRead = stream.read(bytes, offset, bytesNeeded);
070            } catch (IOException ioe) {
071                throw new BoxAPIException("Reading data from stream failed.", ioe);
072            }
073
074            if (bytesRead == -1) {
075                throw new BoxAPIException("Stream ended while upload was progressing");
076            }
077
078            bytesNeeded = bytesNeeded - bytesRead;
079            offset = offset + bytesRead;
080        }
081
082        return bytes;
083    }
084
085    private BoxFileUploadSession.Info createUploadSession(BoxAPIConnection boxApi, String folderId,
086                                                          URL url, String fileName, long fileSize) {
087
088        BoxJSONRequest request = new BoxJSONRequest(boxApi, url, HttpMethod.POST);
089
090        //Create the JSON body of the request
091        JsonObject body = new JsonObject();
092        body.add("folder_id", folderId);
093        body.add("file_name", fileName);
094        body.add("file_size", fileSize);
095        request.setBody(body.toString());
096
097        BoxJSONResponse response = (BoxJSONResponse) request.send();
098        JsonObject jsonObject = JsonObject.readFrom(response.getJSON());
099
100        String sessionId = jsonObject.get("id").asString();
101        BoxFileUploadSession session = new BoxFileUploadSession(boxApi, sessionId);
102
103        return session.new Info(jsonObject);
104    }
105
106    /**
107     * Uploads a new large file.
108     *
109     * @param boxApi   the API connection to be used by the upload session.
110     * @param folderId the id of the folder in which the file will be uploaded.
111     * @param stream   the input stream that feeds the content of the file.
112     * @param url      the upload session URL.
113     * @param fileName the name of the file to be created.
114     * @param fileSize the total size of the file.
115     * @return the created file instance.
116     * @throws InterruptedException when a thread gets interupted.
117     * @throws IOException          when reading a stream throws exception.
118     */
119    public BoxFile.Info upload(BoxAPIConnection boxApi, String folderId, InputStream stream, URL url,
120                               String fileName, long fileSize) throws InterruptedException, IOException {
121        //Create a upload session
122        BoxFileUploadSession.Info session = this.createUploadSession(boxApi, folderId, url, fileName, fileSize);
123        return this.uploadHelper(session, stream, fileSize, null);
124    }
125
126    /**
127     * Uploads a new large file and sets file attributes.
128     *
129     * @param boxApi         the API connection to be used by the upload session.
130     * @param folderId       the id of the folder in which the file will be uploaded.
131     * @param stream         the input stream that feeds the content of the file.
132     * @param url            the upload session URL.
133     * @param fileName       the name of the file to be created.
134     * @param fileSize       the total size of the file.
135     * @param fileAttributes file attributes to set
136     * @return the created file instance.
137     * @throws InterruptedException when a thread gets interupted.
138     * @throws IOException          when reading a stream throws exception.
139     */
140    public BoxFile.Info upload(BoxAPIConnection boxApi, String folderId, InputStream stream, URL url,
141                               String fileName, long fileSize, Map<String, String> fileAttributes)
142        throws InterruptedException, IOException {
143        //Create a upload session
144        BoxFileUploadSession.Info session = this.createUploadSession(boxApi, folderId, url, fileName, fileSize);
145        return this.uploadHelper(session, stream, fileSize, fileAttributes);
146    }
147
148    /**
149     * Creates a new version of a large file.
150     *
151     * @param boxApi   the API connection to be used by the upload session.
152     * @param stream   the input stream that feeds the content of the file.
153     * @param url      the upload session URL.
154     * @param fileSize the total size of the file.
155     * @return the file instance that also contains the version information.
156     * @throws InterruptedException when a thread gets interupted.
157     * @throws IOException          when reading a stream throws exception.
158     */
159    public BoxFile.Info upload(BoxAPIConnection boxApi, InputStream stream, URL url, long fileSize)
160        throws InterruptedException, IOException {
161        //creates a upload session
162        BoxFileUploadSession.Info session = this.createUploadSession(boxApi, url, fileSize);
163        return this.uploadHelper(session, stream, fileSize, null);
164    }
165
166    /**
167     * Creates a new version of a large file and sets file attributes.
168     *
169     * @param boxApi         the API connection to be used by the upload session.
170     * @param stream         the input stream that feeds the content of the file.
171     * @param url            the upload session URL.
172     * @param fileSize       the total size of the file.
173     * @param fileAttributes file attributes to set.
174     * @return the file instance that also contains the version information.
175     * @throws InterruptedException when a thread gets interupted.
176     * @throws IOException          when reading a stream throws exception.
177     */
178    public BoxFile.Info upload(BoxAPIConnection boxApi, InputStream stream, URL url, long fileSize,
179                               Map<String, String> fileAttributes)
180        throws InterruptedException, IOException {
181        //creates a upload session
182        BoxFileUploadSession.Info session = this.createUploadSession(boxApi, url, fileSize);
183        return this.uploadHelper(session, stream, fileSize, fileAttributes);
184    }
185
186    private BoxFile.Info uploadHelper(BoxFileUploadSession.Info session, InputStream stream, long fileSize,
187                                      Map<String, String> fileAttributes)
188        throws InterruptedException, IOException {
189        //Upload parts using the upload session
190        MessageDigest digest = null;
191        try {
192            digest = MessageDigest.getInstance(DIGEST_ALGORITHM_SHA1);
193        } catch (NoSuchAlgorithmException ae) {
194            throw new BoxAPIException("Digest algorithm not found", ae);
195        }
196        DigestInputStream dis = new DigestInputStream(stream, digest);
197        List<BoxFileUploadSessionPart> parts = this.uploadParts(session, dis, fileSize);
198
199        //Creates the file hash
200        byte[] digestBytes = digest.digest();
201        String digestStr = Base64.encode(digestBytes);
202
203        //Commit the upload session. If there is a failure, abort the commit.
204        try {
205            return session.getResource().commit(digestStr, parts, fileAttributes, null, null);
206        } catch (Exception e) {
207            session.getResource().abort();
208            throw new BoxAPIException("Unable to commit the upload session", e);
209        }
210    }
211
212    private BoxFileUploadSession.Info createUploadSession(BoxAPIConnection boxApi, URL url, long fileSize) {
213        BoxJSONRequest request = new BoxJSONRequest(boxApi, url, HttpMethod.POST);
214
215        //Creates the body of the request
216        JsonObject body = new JsonObject();
217        body.add("file_size", fileSize);
218        request.setBody(body.toString());
219
220        BoxJSONResponse response = (BoxJSONResponse) request.send();
221        JsonObject jsonObject = JsonObject.readFrom(response.getJSON());
222
223        String sessionId = jsonObject.get("id").asString();
224        BoxFileUploadSession session = new BoxFileUploadSession(boxApi, sessionId);
225
226        return session.new Info(jsonObject);
227    }
228
229    /*
230     * Upload parts of the file. The part size is retrieved from the upload session.
231     */
232    private List<BoxFileUploadSessionPart> uploadParts(BoxFileUploadSession.Info session, InputStream stream,
233                                                       long fileSize) throws InterruptedException {
234        List<BoxFileUploadSessionPart> parts = new ArrayList<BoxFileUploadSessionPart>();
235
236        int partSize = session.getPartSize();
237        long offset = 0;
238        long processed = 0;
239        int partPostion = 0;
240        //Set the Max Queue Size to 1.5x the number of processors
241        double maxQueueSizeDouble = Math.ceil(this.executorService.getMaximumPoolSize() * 1.5);
242        int maxQueueSize = Double.valueOf(maxQueueSizeDouble).intValue();
243        while (processed < fileSize) {
244            //Waiting for any thread to finish before
245            long timeoutForWaitingInMillis = TimeUnit.MILLISECONDS.convert(this.timeout, this.timeUnit);
246            if (this.executorService.getCorePoolSize() <= this.executorService.getActiveCount()) {
247                if (timeoutForWaitingInMillis > 0) {
248                    Thread.sleep(LargeFileUpload.THREAD_POOL_WAIT_TIME_IN_MILLIS);
249                    timeoutForWaitingInMillis -= THREAD_POOL_WAIT_TIME_IN_MILLIS;
250                } else {
251                    throw new BoxAPIException("Upload parts timedout");
252                }
253            }
254            if (this.executorService.getQueue().size() < maxQueueSize) {
255                long diff = fileSize - (long) processed;
256                //The size last part of the file can be lesser than the part size.
257                if (diff < (long) partSize) {
258                    partSize = (int) diff;
259                }
260                parts.add(null);
261                byte[] bytes = getBytesFromStream(stream, partSize);
262                this.executorService.execute(
263                    new LargeFileUploadTask(session.getResource(), bytes, offset,
264                        partSize, fileSize, parts, partPostion)
265                );
266
267                //Increase the offset and proceesed bytes to calculate the Content-Range header.
268                processed += partSize;
269                offset += partSize;
270                partPostion++;
271            }
272        }
273        this.executorService.shutdown();
274        this.executorService.awaitTermination(this.timeout, this.timeUnit);
275        return parts;
276    }
277
278    /**
279     * Generates the Base64 encoded SHA-1 hash for content available in the stream.
280     * It can be used to calculate the hash of a file.
281     *
282     * @param stream the input stream of the file or data.
283     * @return the Base64 encoded hash string.
284     */
285    public String generateDigest(InputStream stream) {
286        MessageDigest digest = null;
287        try {
288            digest = MessageDigest.getInstance(DIGEST_ALGORITHM_SHA1);
289        } catch (NoSuchAlgorithmException ae) {
290            throw new BoxAPIException("Digest algorithm not found", ae);
291        }
292
293        //Calcuate the digest using the stream.
294        DigestInputStream dis = new DigestInputStream(stream, digest);
295        try {
296            int value = dis.read();
297            while (value != -1) {
298                value = dis.read();
299            }
300        } catch (IOException ioe) {
301            throw new BoxAPIException("Reading the stream failed.", ioe);
302        }
303
304        //Get the calculated digest for the stream
305        byte[] digestBytes = digest.digest();
306        return Base64.encode(digestBytes);
307    }
308}