001package com.box.sdk; 002 003import java.io.IOException; 004import java.io.InputStream; 005import java.net.URL; 006import java.security.DigestInputStream; 007import java.security.MessageDigest; 008import java.security.NoSuchAlgorithmException; 009import java.util.ArrayList; 010import java.util.List; 011import java.util.concurrent.Executors; 012import java.util.concurrent.ThreadPoolExecutor; 013import java.util.concurrent.TimeUnit; 014 015import com.box.sdk.http.HttpMethod; 016import com.eclipsesource.json.JsonObject; 017 018/** 019 * Utility class for uploading large files. 020 */ 021public final class LargeFileUpload { 022 023 private static final String DIGEST_HEADER_PREFIX_SHA = "sha="; 024 private static final String DIGEST_ALGORITHM_SHA1 = "SHA1"; 025 026 private static final String OFFSET_QUERY_STRING = "offset"; 027 private static final String LIMIT_QUERY_STRING = "limit"; 028 private static final int DEFAULT_CONNECTIONS = 3; 029 private static final int DEFAULT_TIMEOUT = 1; 030 private static final TimeUnit DEFAULT_TIMEUNIT = TimeUnit.HOURS; 031 private static final int THREAD_POOL_WAIT_TIME_IN_MILLIS = 1000; 032 private ThreadPoolExecutor executorService; 033 private long timeout; 034 private TimeUnit timeUnit; 035 private int connections; 036 037 /** 038 * Creates a LargeFileUpload object. 039 * @param nParallelConnections number of parallel http connections to use 040 * @param timeOut time to wait before killing the job 041 * @param unit time unit for the time wait value 042 */ 043 public LargeFileUpload(int nParallelConnections, long timeOut, TimeUnit unit) { 044 this.executorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(nParallelConnections); 045 this.timeout = timeOut; 046 this.timeUnit = unit; 047 } 048 049 /** 050 * Creates a LargeFileUpload object with a default number of parallel conections and timeout. 051 */ 052 public LargeFileUpload() { 053 this.executorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(LargeFileUpload.DEFAULT_CONNECTIONS); 054 this.timeout = LargeFileUpload.DEFAULT_TIMEOUT; 055 this.timeUnit = LargeFileUpload.DEFAULT_TIMEUNIT; 056 } 057 058 private BoxFileUploadSession.Info createUploadSession(BoxAPIConnection boxApi, String folderId, 059 URL url, String fileName, long fileSize) { 060 061 BoxJSONRequest request = new BoxJSONRequest(boxApi, url, HttpMethod.POST); 062 063 //Create the JSON body of the request 064 JsonObject body = new JsonObject(); 065 body.add("folder_id", folderId); 066 body.add("file_name", fileName); 067 body.add("file_size", fileSize); 068 request.setBody(body.toString()); 069 070 BoxJSONResponse response = (BoxJSONResponse) request.send(); 071 JsonObject jsonObject = JsonObject.readFrom(response.getJSON()); 072 073 String sessionId = jsonObject.get("id").asString(); 074 BoxFileUploadSession session = new BoxFileUploadSession(boxApi, sessionId); 075 076 return session.new Info(jsonObject); 077 } 078 079 /** 080 * Uploads a new large file. 081 * @param boxApi the API connection to be used by the upload session. 082 * @param folderId the id of the folder in which the file will be uploaded. 083 * @param stream the input stream that feeds the content of the file. 084 * @param url the upload session URL. 085 * @param fileName the name of the file to be created. 086 * @param fileSize the total size of the file. 087 * @return the created file instance. 088 * @throws InterruptedException when a thread gets interupted. 089 * @throws IOException when reading a stream throws exception. 090 */ 091 public BoxFile.Info upload(BoxAPIConnection boxApi, String folderId, InputStream stream, URL url, 092 String fileName, long fileSize) throws InterruptedException, IOException { 093 //Create a upload session 094 BoxFileUploadSession.Info session = this.createUploadSession(boxApi, folderId, url, fileName, fileSize); 095 return this.uploadHelper(session, stream, fileSize); 096 } 097 098 /** 099 * Creates a new version of a large file. 100 * @param boxApi the API connection to be used by the upload session. 101 * @param stream the input stream that feeds the content of the file. 102 * @param url the upload session URL. 103 * @param fileSize the total size of the file. 104 * @return the file instance that also contains the version information. 105 * @throws InterruptedException when a thread gets interupted. 106 * @throws IOException when reading a stream throws exception. 107 */ 108 public BoxFile.Info upload(BoxAPIConnection boxApi, InputStream stream, URL url, long fileSize) 109 throws InterruptedException, IOException { 110 //creates a upload session 111 BoxFileUploadSession.Info session = this.createUploadSession(boxApi, url, fileSize); 112 return this.uploadHelper(session, stream, fileSize); 113 } 114 115 private BoxFile.Info uploadHelper(BoxFileUploadSession.Info session, InputStream stream, long fileSize) 116 throws InterruptedException, IOException { 117 //Upload parts using the upload session 118 MessageDigest digest = null; 119 try { 120 digest = MessageDigest.getInstance(DIGEST_ALGORITHM_SHA1); 121 } catch (NoSuchAlgorithmException ae) { 122 throw new BoxAPIException("Digest algorithm not found", ae); 123 } 124 DigestInputStream dis = new DigestInputStream(stream, digest); 125 List<BoxFileUploadSessionPart> parts = this.uploadParts(session, dis, fileSize); 126 127 //Creates the file hash 128 byte[] digestBytes = digest.digest(); 129 String digestStr = Base64.encode(digestBytes); 130 131 //Commit the upload session. If there is a failure, abort the commit. 132 try { 133 return session.getResource().commit(digestStr, parts, null, null, null); 134 } catch (Exception e) { 135 session.getResource().abort(); 136 throw new BoxAPIException("Unable to commit the upload session", e); 137 } 138 } 139 140 private BoxFileUploadSession.Info createUploadSession(BoxAPIConnection boxApi, URL url, long fileSize) { 141 BoxJSONRequest request = new BoxJSONRequest(boxApi, url, HttpMethod.POST); 142 143 //Creates the body of the request 144 JsonObject body = new JsonObject(); 145 body.add("file_size", fileSize); 146 request.setBody(body.toString()); 147 148 BoxJSONResponse response = (BoxJSONResponse) request.send(); 149 JsonObject jsonObject = JsonObject.readFrom(response.getJSON()); 150 151 String sessionId = jsonObject.get("id").asString(); 152 BoxFileUploadSession session = new BoxFileUploadSession(boxApi, sessionId); 153 154 return session.new Info(jsonObject); 155 } 156 157 /* 158 * Upload parts of the file. The part size is retrieved from the upload session. 159 */ 160 private List<BoxFileUploadSessionPart> uploadParts(BoxFileUploadSession.Info session, InputStream stream, 161 long fileSize) throws InterruptedException { 162 List<BoxFileUploadSessionPart> parts = new ArrayList<BoxFileUploadSessionPart>(); 163 164 int partSize = session.getPartSize(); 165 long offset = 0; 166 int processed = 0; 167 int partPostion = 0; 168 while (processed < fileSize) { 169 //Waiting for any thread to finish before 170 long timeoutForWaitingInMillis = TimeUnit.MILLISECONDS.convert(this.timeout, this.timeUnit); 171 if (this.executorService.getCorePoolSize() <= this.executorService.getActiveCount()) { 172 if (timeoutForWaitingInMillis > 0) { 173 Thread.sleep(LargeFileUpload.THREAD_POOL_WAIT_TIME_IN_MILLIS); 174 timeoutForWaitingInMillis -= THREAD_POOL_WAIT_TIME_IN_MILLIS; 175 } else { 176 throw new BoxAPIException("Upload parts timedout"); 177 } 178 } 179 long diff = fileSize - (long) processed; 180 //The size last part of the file can be lesser than the part size. 181 if (diff < (long) partSize) { 182 partSize = (int) diff; 183 } 184 parts.add(null); 185 byte[] bytes = new byte[partSize]; 186 try { 187 int readStatus = stream.read(bytes); 188 if (readStatus == -1) { 189 throw new BoxAPIException("Stream ended while upload was progressing"); 190 } 191 } catch (IOException ioe) { 192 throw new BoxAPIException("Reading data from stream failed.", ioe); 193 } 194 this.executorService.execute( 195 new LargeFileUploadTask(session.getResource(), bytes, offset, partSize, fileSize, parts, partPostion) 196 ); 197 198 //Increase the offset and proceesed bytes to calculate the Content-Range header. 199 processed += partSize; 200 offset += partSize; 201 partPostion++; 202 } 203 this.executorService.shutdown(); 204 this.executorService.awaitTermination(this.timeout, this.timeUnit); 205 return parts; 206 } 207 208 /** 209 * Generates the Base64 encoded SHA-1 hash for content available in the stream. 210 * It can be used to calculate the hash of a file. 211 * @param stream the input stream of the file or data. 212 * @return the Base64 encoded hash string. 213 */ 214 public String generateDigest(InputStream stream) { 215 MessageDigest digest = null; 216 try { 217 digest = MessageDigest.getInstance(DIGEST_ALGORITHM_SHA1); 218 } catch (NoSuchAlgorithmException ae) { 219 throw new BoxAPIException("Digest algorithm not found", ae); 220 } 221 222 //Calcuate the digest using the stream. 223 DigestInputStream dis = new DigestInputStream(stream, digest); 224 try { 225 int value = dis.read(); 226 while (value != -1) { 227 value = dis.read(); 228 } 229 } catch (IOException ioe) { 230 throw new BoxAPIException("Reading the stream failed.", ioe); 231 } 232 233 //Get the calculated digest for the stream 234 byte[] digestBytes = digest.digest(); 235 return Base64.encode(digestBytes); 236 } 237}