001package com.box.sdk; 002 003import com.box.sdk.http.HttpMethod; 004import com.eclipsesource.json.JsonObject; 005import java.io.IOException; 006import java.io.InputStream; 007import java.net.URL; 008import java.security.DigestInputStream; 009import java.security.MessageDigest; 010import java.security.NoSuchAlgorithmException; 011import java.util.ArrayList; 012import java.util.List; 013import java.util.Map; 014import java.util.concurrent.Executors; 015import java.util.concurrent.ThreadPoolExecutor; 016import java.util.concurrent.TimeUnit; 017 018/** 019 * Utility class for uploading large files. 020 */ 021public final class LargeFileUpload { 022 023 private static final String DIGEST_HEADER_PREFIX_SHA = "sha="; 024 private static final String DIGEST_ALGORITHM_SHA1 = "SHA1"; 025 026 private static final String OFFSET_QUERY_STRING = "offset"; 027 private static final String LIMIT_QUERY_STRING = "limit"; 028 private static final int DEFAULT_CONNECTIONS = 3; 029 private static final int DEFAULT_TIMEOUT = 1; 030 private static final TimeUnit DEFAULT_TIMEUNIT = TimeUnit.HOURS; 031 private static final int THREAD_POOL_WAIT_TIME_IN_MILLIS = 1000; 032 private ThreadPoolExecutor executorService; 033 private long timeout; 034 private TimeUnit timeUnit; 035 private int connections; 036 037 /** 038 * Creates a LargeFileUpload object. 039 * 040 * @param nParallelConnections number of parallel http connections to use 041 * @param timeOut time to wait before killing the job 042 * @param unit time unit for the time wait value 043 */ 044 public LargeFileUpload(int nParallelConnections, long timeOut, TimeUnit unit) { 045 this.executorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(nParallelConnections); 046 this.timeout = timeOut; 047 this.timeUnit = unit; 048 } 049 050 /** 051 * Creates a LargeFileUpload object with a default number of parallel conections and timeout. 052 */ 053 public LargeFileUpload() { 054 this.executorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(LargeFileUpload.DEFAULT_CONNECTIONS); 055 this.timeout = LargeFileUpload.DEFAULT_TIMEOUT; 056 this.timeUnit = LargeFileUpload.DEFAULT_TIMEUNIT; 057 } 058 059 private static byte[] getBytesFromStream(InputStream stream, int numBytes) { 060 061 int bytesNeeded = numBytes; 062 int offset = 0; 063 byte[] bytes = new byte[numBytes]; 064 065 while (bytesNeeded > 0) { 066 067 int bytesRead = -1; 068 try { 069 bytesRead = stream.read(bytes, offset, bytesNeeded); 070 } catch (IOException ioe) { 071 throw new BoxAPIException("Reading data from stream failed.", ioe); 072 } 073 074 if (bytesRead == -1) { 075 throw new BoxAPIException("Stream ended while upload was progressing"); 076 } 077 078 bytesNeeded = bytesNeeded - bytesRead; 079 offset = offset + bytesRead; 080 } 081 082 return bytes; 083 } 084 085 private BoxFileUploadSession.Info createUploadSession(BoxAPIConnection boxApi, String folderId, 086 URL url, String fileName, long fileSize) { 087 088 BoxJSONRequest request = new BoxJSONRequest(boxApi, url, HttpMethod.POST); 089 090 //Create the JSON body of the request 091 JsonObject body = new JsonObject(); 092 body.add("folder_id", folderId); 093 body.add("file_name", fileName); 094 body.add("file_size", fileSize); 095 request.setBody(body.toString()); 096 097 BoxJSONResponse response = (BoxJSONResponse) request.send(); 098 JsonObject jsonObject = JsonObject.readFrom(response.getJSON()); 099 100 String sessionId = jsonObject.get("id").asString(); 101 BoxFileUploadSession session = new BoxFileUploadSession(boxApi, sessionId); 102 103 return session.new Info(jsonObject); 104 } 105 106 /** 107 * Uploads a new large file. 108 * 109 * @param boxApi the API connection to be used by the upload session. 110 * @param folderId the id of the folder in which the file will be uploaded. 111 * @param stream the input stream that feeds the content of the file. 112 * @param url the upload session URL. 113 * @param fileName the name of the file to be created. 114 * @param fileSize the total size of the file. 115 * @return the created file instance. 116 * @throws InterruptedException when a thread gets interupted. 117 * @throws IOException when reading a stream throws exception. 118 */ 119 public BoxFile.Info upload(BoxAPIConnection boxApi, String folderId, InputStream stream, URL url, 120 String fileName, long fileSize) throws InterruptedException, IOException { 121 //Create a upload session 122 BoxFileUploadSession.Info session = this.createUploadSession(boxApi, folderId, url, fileName, fileSize); 123 return this.uploadHelper(session, stream, fileSize, null); 124 } 125 126 /** 127 * Uploads a new large file and sets file attributes. 128 * 129 * @param boxApi the API connection to be used by the upload session. 130 * @param folderId the id of the folder in which the file will be uploaded. 131 * @param stream the input stream that feeds the content of the file. 132 * @param url the upload session URL. 133 * @param fileName the name of the file to be created. 134 * @param fileSize the total size of the file. 135 * @param fileAttributes file attributes to set 136 * @return the created file instance. 137 * @throws InterruptedException when a thread gets interupted. 138 * @throws IOException when reading a stream throws exception. 139 */ 140 public BoxFile.Info upload(BoxAPIConnection boxApi, String folderId, InputStream stream, URL url, 141 String fileName, long fileSize, Map<String, String> fileAttributes) 142 throws InterruptedException, IOException { 143 //Create a upload session 144 BoxFileUploadSession.Info session = this.createUploadSession(boxApi, folderId, url, fileName, fileSize); 145 return this.uploadHelper(session, stream, fileSize, fileAttributes); 146 } 147 148 /** 149 * Creates a new version of a large file. 150 * 151 * @param boxApi the API connection to be used by the upload session. 152 * @param stream the input stream that feeds the content of the file. 153 * @param url the upload session URL. 154 * @param fileSize the total size of the file. 155 * @return the file instance that also contains the version information. 156 * @throws InterruptedException when a thread gets interupted. 157 * @throws IOException when reading a stream throws exception. 158 */ 159 public BoxFile.Info upload(BoxAPIConnection boxApi, InputStream stream, URL url, long fileSize) 160 throws InterruptedException, IOException { 161 //creates a upload session 162 BoxFileUploadSession.Info session = this.createUploadSession(boxApi, url, fileSize); 163 return this.uploadHelper(session, stream, fileSize, null); 164 } 165 166 /** 167 * Creates a new version of a large file and sets file attributes. 168 * 169 * @param boxApi the API connection to be used by the upload session. 170 * @param stream the input stream that feeds the content of the file. 171 * @param url the upload session URL. 172 * @param fileSize the total size of the file. 173 * @param fileAttributes file attributes to set. 174 * @return the file instance that also contains the version information. 175 * @throws InterruptedException when a thread gets interupted. 176 * @throws IOException when reading a stream throws exception. 177 */ 178 public BoxFile.Info upload(BoxAPIConnection boxApi, InputStream stream, URL url, long fileSize, 179 Map<String, String> fileAttributes) 180 throws InterruptedException, IOException { 181 //creates a upload session 182 BoxFileUploadSession.Info session = this.createUploadSession(boxApi, url, fileSize); 183 return this.uploadHelper(session, stream, fileSize, fileAttributes); 184 } 185 186 private BoxFile.Info uploadHelper(BoxFileUploadSession.Info session, InputStream stream, long fileSize, 187 Map<String, String> fileAttributes) 188 throws InterruptedException, IOException { 189 //Upload parts using the upload session 190 MessageDigest digest = null; 191 try { 192 digest = MessageDigest.getInstance(DIGEST_ALGORITHM_SHA1); 193 } catch (NoSuchAlgorithmException ae) { 194 throw new BoxAPIException("Digest algorithm not found", ae); 195 } 196 DigestInputStream dis = new DigestInputStream(stream, digest); 197 List<BoxFileUploadSessionPart> parts = this.uploadParts(session, dis, fileSize); 198 199 //Creates the file hash 200 byte[] digestBytes = digest.digest(); 201 String digestStr = Base64.encode(digestBytes); 202 203 //Commit the upload session. If there is a failure, abort the commit. 204 try { 205 return session.getResource().commit(digestStr, parts, fileAttributes, null, null); 206 } catch (Exception e) { 207 session.getResource().abort(); 208 throw new BoxAPIException("Unable to commit the upload session", e); 209 } 210 } 211 212 private BoxFileUploadSession.Info createUploadSession(BoxAPIConnection boxApi, URL url, long fileSize) { 213 BoxJSONRequest request = new BoxJSONRequest(boxApi, url, HttpMethod.POST); 214 215 //Creates the body of the request 216 JsonObject body = new JsonObject(); 217 body.add("file_size", fileSize); 218 request.setBody(body.toString()); 219 220 BoxJSONResponse response = (BoxJSONResponse) request.send(); 221 JsonObject jsonObject = JsonObject.readFrom(response.getJSON()); 222 223 String sessionId = jsonObject.get("id").asString(); 224 BoxFileUploadSession session = new BoxFileUploadSession(boxApi, sessionId); 225 226 return session.new Info(jsonObject); 227 } 228 229 /* 230 * Upload parts of the file. The part size is retrieved from the upload session. 231 */ 232 private List<BoxFileUploadSessionPart> uploadParts(BoxFileUploadSession.Info session, InputStream stream, 233 long fileSize) throws InterruptedException { 234 List<BoxFileUploadSessionPart> parts = new ArrayList<BoxFileUploadSessionPart>(); 235 236 int partSize = session.getPartSize(); 237 long offset = 0; 238 long processed = 0; 239 int partPostion = 0; 240 //Set the Max Queue Size to 1.5x the number of processors 241 double maxQueueSizeDouble = Math.ceil(this.executorService.getMaximumPoolSize() * 1.5); 242 int maxQueueSize = Double.valueOf(maxQueueSizeDouble).intValue(); 243 while (processed < fileSize) { 244 //Waiting for any thread to finish before 245 long timeoutForWaitingInMillis = TimeUnit.MILLISECONDS.convert(this.timeout, this.timeUnit); 246 if (this.executorService.getCorePoolSize() <= this.executorService.getActiveCount()) { 247 if (timeoutForWaitingInMillis > 0) { 248 Thread.sleep(LargeFileUpload.THREAD_POOL_WAIT_TIME_IN_MILLIS); 249 timeoutForWaitingInMillis -= THREAD_POOL_WAIT_TIME_IN_MILLIS; 250 } else { 251 throw new BoxAPIException("Upload parts timedout"); 252 } 253 } 254 if (this.executorService.getQueue().size() < maxQueueSize) { 255 long diff = fileSize - (long) processed; 256 //The size last part of the file can be lesser than the part size. 257 if (diff < (long) partSize) { 258 partSize = (int) diff; 259 } 260 parts.add(null); 261 byte[] bytes = getBytesFromStream(stream, partSize); 262 this.executorService.execute( 263 new LargeFileUploadTask(session.getResource(), bytes, offset, 264 partSize, fileSize, parts, partPostion) 265 ); 266 267 //Increase the offset and proceesed bytes to calculate the Content-Range header. 268 processed += partSize; 269 offset += partSize; 270 partPostion++; 271 } 272 } 273 this.executorService.shutdown(); 274 this.executorService.awaitTermination(this.timeout, this.timeUnit); 275 return parts; 276 } 277 278 /** 279 * Generates the Base64 encoded SHA-1 hash for content available in the stream. 280 * It can be used to calculate the hash of a file. 281 * 282 * @param stream the input stream of the file or data. 283 * @return the Base64 encoded hash string. 284 */ 285 public String generateDigest(InputStream stream) { 286 MessageDigest digest = null; 287 try { 288 digest = MessageDigest.getInstance(DIGEST_ALGORITHM_SHA1); 289 } catch (NoSuchAlgorithmException ae) { 290 throw new BoxAPIException("Digest algorithm not found", ae); 291 } 292 293 //Calcuate the digest using the stream. 294 DigestInputStream dis = new DigestInputStream(stream, digest); 295 try { 296 int value = dis.read(); 297 while (value != -1) { 298 value = dis.read(); 299 } 300 } catch (IOException ioe) { 301 throw new BoxAPIException("Reading the stream failed.", ioe); 302 } 303 304 //Get the calculated digest for the stream 305 byte[] digestBytes = digest.digest(); 306 return Base64.encode(digestBytes); 307 } 308}