001package com.box.sdk;
002
003import java.io.IOException;
004import java.io.InputStream;
005import java.net.URL;
006import java.security.DigestInputStream;
007import java.security.MessageDigest;
008import java.security.NoSuchAlgorithmException;
009import java.util.ArrayList;
010import java.util.List;
011import java.util.concurrent.Executors;
012import java.util.concurrent.ThreadPoolExecutor;
013import java.util.concurrent.TimeUnit;
014
015import com.box.sdk.http.HttpMethod;
016import com.eclipsesource.json.JsonObject;
017
018/**
019 * Utility class for uploading large files.
020 */
021public final class LargeFileUpload {
022
023    private static final String DIGEST_HEADER_PREFIX_SHA = "sha=";
024    private static final String DIGEST_ALGORITHM_SHA1 = "SHA1";
025
026    private static final String OFFSET_QUERY_STRING = "offset";
027    private static final String LIMIT_QUERY_STRING = "limit";
028    private static final int DEFAULT_CONNECTIONS = 3;
029    private static final int DEFAULT_TIMEOUT = 1;
030    private static final TimeUnit DEFAULT_TIMEUNIT = TimeUnit.HOURS;
031    private static final int THREAD_POOL_WAIT_TIME_IN_MILLIS = 1000;
032    private ThreadPoolExecutor executorService;
033    private long timeout;
034    private TimeUnit timeUnit;
035    private int connections;
036
037    /**
038     * Creates a LargeFileUpload object.
039     * @param nParallelConnections number of parallel http connections to use
040     * @param timeOut time to wait before killing the job
041     * @param unit time unit for the time wait value
042     */
043    public LargeFileUpload(int nParallelConnections, long timeOut, TimeUnit unit) {
044        this.executorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(nParallelConnections);
045        this.timeout = timeOut;
046        this.timeUnit = unit;
047    }
048
049    /**
050     * Creates a LargeFileUpload object with a default number of parallel conections and timeout.
051     */
052    public LargeFileUpload() {
053        this.executorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(LargeFileUpload.DEFAULT_CONNECTIONS);
054        this.timeout = LargeFileUpload.DEFAULT_TIMEOUT;
055        this.timeUnit = LargeFileUpload.DEFAULT_TIMEUNIT;
056    }
057
058    private BoxFileUploadSession.Info createUploadSession(BoxAPIConnection boxApi, String folderId,
059                                                         URL url, String fileName, long fileSize) {
060
061        BoxJSONRequest request = new BoxJSONRequest(boxApi, url, HttpMethod.POST);
062
063        //Create the JSON body of the request
064        JsonObject body = new JsonObject();
065        body.add("folder_id", folderId);
066        body.add("file_name", fileName);
067        body.add("file_size", fileSize);
068        request.setBody(body.toString());
069
070        BoxJSONResponse response = (BoxJSONResponse) request.send();
071        JsonObject jsonObject = JsonObject.readFrom(response.getJSON());
072
073        String sessionId = jsonObject.get("id").asString();
074        BoxFileUploadSession session = new BoxFileUploadSession(boxApi, sessionId);
075
076        return session.new Info(jsonObject);
077    }
078
079    /**
080     * Uploads a new large file.
081     * @param boxApi the API connection to be used by the upload session.
082     * @param folderId the id of the folder in which the file will be uploaded.
083     * @param stream the input stream that feeds the content of the file.
084     * @param url the upload session URL.
085     * @param fileName the name of the file to be created.
086     * @param fileSize the total size of the file.
087     * @return the created file instance.
088     * @throws InterruptedException when a thread gets interupted.
089     * @throws IOException when reading a stream throws exception.
090     */
091    public BoxFile.Info upload(BoxAPIConnection boxApi, String folderId, InputStream stream, URL url,
092                               String fileName, long fileSize) throws InterruptedException, IOException {
093        //Create a upload session
094        BoxFileUploadSession.Info session = this.createUploadSession(boxApi, folderId, url, fileName, fileSize);
095        return this.uploadHelper(session, stream, fileSize);
096    }
097
098    /**
099     * Creates a new version of a large file.
100     * @param boxApi the API connection to be used by the upload session.
101     * @param stream the input stream that feeds the content of the file.
102     * @param url the upload session URL.
103     * @param fileSize the total size of the file.
104     * @return the file instance that also contains the version information.
105     * @throws InterruptedException when a thread gets interupted.
106     * @throws IOException when reading a stream throws exception.
107     */
108    public BoxFile.Info upload(BoxAPIConnection boxApi, InputStream stream, URL url, long fileSize)
109        throws InterruptedException, IOException {
110        //creates a upload session
111        BoxFileUploadSession.Info session = this.createUploadSession(boxApi, url, fileSize);
112        return this.uploadHelper(session, stream, fileSize);
113    }
114
115    private BoxFile.Info uploadHelper(BoxFileUploadSession.Info session, InputStream stream, long fileSize)
116        throws InterruptedException, IOException {
117        //Upload parts using the upload session
118        MessageDigest digest = null;
119        try {
120            digest = MessageDigest.getInstance(DIGEST_ALGORITHM_SHA1);
121        } catch (NoSuchAlgorithmException ae) {
122            throw new BoxAPIException("Digest algorithm not found", ae);
123        }
124        DigestInputStream dis = new DigestInputStream(stream, digest);
125        List<BoxFileUploadSessionPart> parts = this.uploadParts(session, dis, fileSize);
126
127        //Creates the file hash
128        byte[] digestBytes = digest.digest();
129        String digestStr = Base64.encode(digestBytes);
130
131        //Commit the upload session. If there is a failure, abort the commit.
132        try {
133            return session.getResource().commit(digestStr, parts, null, null, null);
134        } catch (Exception e) {
135            session.getResource().abort();
136            throw new BoxAPIException("Unable to commit the upload session", e);
137        }
138    }
139
140    private BoxFileUploadSession.Info createUploadSession(BoxAPIConnection boxApi, URL url, long fileSize) {
141        BoxJSONRequest request = new BoxJSONRequest(boxApi, url, HttpMethod.POST);
142
143        //Creates the body of the request
144        JsonObject body = new JsonObject();
145        body.add("file_size", fileSize);
146        request.setBody(body.toString());
147
148        BoxJSONResponse response = (BoxJSONResponse) request.send();
149        JsonObject jsonObject = JsonObject.readFrom(response.getJSON());
150
151        String sessionId = jsonObject.get("id").asString();
152        BoxFileUploadSession session = new BoxFileUploadSession(boxApi, sessionId);
153
154        return session.new Info(jsonObject);
155    }
156
157    /*
158     * Upload parts of the file. The part size is retrieved from the upload session.
159     */
160    private List<BoxFileUploadSessionPart> uploadParts(BoxFileUploadSession.Info session, InputStream stream,
161                                                       long fileSize) throws InterruptedException {
162        List<BoxFileUploadSessionPart> parts = new ArrayList<BoxFileUploadSessionPart>();
163
164        int partSize = session.getPartSize();
165        long offset = 0;
166        int processed = 0;
167        int partPostion = 0;
168        while (processed < fileSize) {
169            //Waiting for any thread to finish before
170            long timeoutForWaitingInMillis = TimeUnit.MILLISECONDS.convert(this.timeout, this.timeUnit);
171            if (this.executorService.getCorePoolSize() <= this.executorService.getActiveCount()) {
172                if (timeoutForWaitingInMillis > 0) {
173                    Thread.sleep(LargeFileUpload.THREAD_POOL_WAIT_TIME_IN_MILLIS);
174                    timeoutForWaitingInMillis -= THREAD_POOL_WAIT_TIME_IN_MILLIS;
175                } else {
176                    throw new BoxAPIException("Upload parts timedout");
177                }
178            }
179            long diff = fileSize - (long) processed;
180            //The size last part of the file can be lesser than the part size.
181            if (diff < (long) partSize) {
182                partSize = (int) diff;
183            }
184            parts.add(null);
185            byte[] bytes = new byte[partSize];
186            try {
187                int readStatus = stream.read(bytes);
188                if (readStatus == -1) {
189                    throw new BoxAPIException("Stream ended while upload was progressing");
190                }
191            } catch (IOException ioe) {
192                throw new BoxAPIException("Reading data from stream failed.", ioe);
193            }
194            this.executorService.execute(
195                new LargeFileUploadTask(session.getResource(), bytes, offset, partSize, fileSize, parts, partPostion)
196            );
197
198            //Increase the offset and proceesed bytes to calculate the Content-Range header.
199            processed += partSize;
200            offset += partSize;
201            partPostion++;
202        }
203        this.executorService.shutdown();
204        this.executorService.awaitTermination(this.timeout, this.timeUnit);
205        return parts;
206    }
207
208    /**
209     * Generates the Base64 encoded SHA-1 hash for content available in the stream.
210     * It can be used to calculate the hash of a file.
211     * @param stream the input stream of the file or data.
212     * @return the Base64 encoded hash string.
213     */
214    public String generateDigest(InputStream stream) {
215        MessageDigest digest = null;
216        try {
217            digest = MessageDigest.getInstance(DIGEST_ALGORITHM_SHA1);
218        } catch (NoSuchAlgorithmException ae) {
219            throw new BoxAPIException("Digest algorithm not found", ae);
220        }
221
222        //Calcuate the digest using the stream.
223        DigestInputStream dis = new DigestInputStream(stream, digest);
224        try {
225            int value = dis.read();
226            while (value != -1) {
227                value = dis.read();
228            }
229        } catch (IOException ioe) {
230            throw new BoxAPIException("Reading the stream failed.", ioe);
231        }
232
233        //Get the calculated digest for the stream
234        byte[] digestBytes = digest.digest();
235        return Base64.encode(digestBytes);
236    }
237}