001package com.box.sdk; 002 003import com.box.sdk.http.HttpMethod; 004import com.eclipsesource.json.Json; 005import com.eclipsesource.json.JsonObject; 006import java.io.IOException; 007import java.io.InputStream; 008import java.net.URL; 009import java.security.DigestInputStream; 010import java.security.MessageDigest; 011import java.security.NoSuchAlgorithmException; 012import java.util.ArrayList; 013import java.util.List; 014import java.util.Map; 015import java.util.concurrent.Executors; 016import java.util.concurrent.ThreadPoolExecutor; 017import java.util.concurrent.TimeUnit; 018 019/** 020 * Utility class for uploading large files. 021 */ 022public final class LargeFileUpload { 023 024 private static final String DIGEST_HEADER_PREFIX_SHA = "sha="; 025 private static final String DIGEST_ALGORITHM_SHA1 = "SHA1"; 026 027 private static final String OFFSET_QUERY_STRING = "offset"; 028 private static final String LIMIT_QUERY_STRING = "limit"; 029 private static final int DEFAULT_CONNECTIONS = 3; 030 private static final int DEFAULT_TIMEOUT = 1; 031 private static final TimeUnit DEFAULT_TIMEUNIT = TimeUnit.HOURS; 032 private static final int THREAD_POOL_WAIT_TIME_IN_MILLIS = 1000; 033 private final ThreadPoolExecutor executorService; 034 private final long timeout; 035 private final TimeUnit timeUnit; 036 private int connections; 037 038 /** 039 * Creates a LargeFileUpload object. 040 * 041 * @param nParallelConnections number of parallel http connections to use 042 * @param timeOut time to wait before killing the job 043 * @param unit time unit for the time wait value 044 */ 045 public LargeFileUpload(int nParallelConnections, long timeOut, TimeUnit unit) { 046 this.executorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(nParallelConnections); 047 this.timeout = timeOut; 048 this.timeUnit = unit; 049 } 050 051 /** 052 * Creates a LargeFileUpload object with a default number of parallel conections and timeout. 053 */ 054 public LargeFileUpload() { 055 this.executorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(LargeFileUpload.DEFAULT_CONNECTIONS); 056 this.timeout = LargeFileUpload.DEFAULT_TIMEOUT; 057 this.timeUnit = LargeFileUpload.DEFAULT_TIMEUNIT; 058 } 059 060 private static byte[] getBytesFromStream(InputStream stream, int numBytes) { 061 062 int bytesNeeded = numBytes; 063 int offset = 0; 064 byte[] bytes = new byte[numBytes]; 065 066 while (bytesNeeded > 0) { 067 068 int bytesRead; 069 try { 070 bytesRead = stream.read(bytes, offset, bytesNeeded); 071 } catch (IOException ioe) { 072 throw new BoxAPIException("Reading data from stream failed.", ioe); 073 } 074 075 if (bytesRead == -1) { 076 throw new BoxAPIException("Stream ended while upload was progressing"); 077 } 078 079 bytesNeeded = bytesNeeded - bytesRead; 080 offset = offset + bytesRead; 081 } 082 083 return bytes; 084 } 085 086 private BoxFileUploadSession.Info createUploadSession(BoxAPIConnection boxApi, String folderId, 087 URL url, String fileName, long fileSize) { 088 089 BoxJSONRequest request = new BoxJSONRequest(boxApi, url, HttpMethod.POST); 090 091 //Create the JSON body of the request 092 JsonObject body = new JsonObject(); 093 body.add("folder_id", folderId); 094 body.add("file_name", fileName); 095 body.add("file_size", fileSize); 096 request.setBody(body.toString()); 097 098 BoxJSONResponse response = (BoxJSONResponse) request.send(); 099 JsonObject jsonObject = Json.parse(response.getJSON()).asObject(); 100 101 String sessionId = jsonObject.get("id").asString(); 102 BoxFileUploadSession session = new BoxFileUploadSession(boxApi, sessionId); 103 104 return session.new Info(jsonObject); 105 } 106 107 /** 108 * Uploads a new large file. 109 * 110 * @param boxApi the API connection to be used by the upload session. 111 * @param folderId the id of the folder in which the file will be uploaded. 112 * @param stream the input stream that feeds the content of the file. 113 * @param url the upload session URL. 114 * @param fileName the name of the file to be created. 115 * @param fileSize the total size of the file. 116 * @return the created file instance. 117 * @throws InterruptedException when a thread gets interupted. 118 * @throws IOException when reading a stream throws exception. 119 */ 120 public BoxFile.Info upload(BoxAPIConnection boxApi, String folderId, InputStream stream, URL url, 121 String fileName, long fileSize) throws InterruptedException, IOException { 122 //Create a upload session 123 BoxFileUploadSession.Info session = this.createUploadSession(boxApi, folderId, url, fileName, fileSize); 124 return this.uploadHelper(session, stream, fileSize, null); 125 } 126 127 /** 128 * Uploads a new large file and sets file attributes. 129 * 130 * @param boxApi the API connection to be used by the upload session. 131 * @param folderId the id of the folder in which the file will be uploaded. 132 * @param stream the input stream that feeds the content of the file. 133 * @param url the upload session URL. 134 * @param fileName the name of the file to be created. 135 * @param fileSize the total size of the file. 136 * @param fileAttributes file attributes to set 137 * @return the created file instance. 138 * @throws InterruptedException when a thread gets interupted. 139 * @throws IOException when reading a stream throws exception. 140 */ 141 public BoxFile.Info upload(BoxAPIConnection boxApi, String folderId, InputStream stream, URL url, 142 String fileName, long fileSize, Map<String, String> fileAttributes) 143 throws InterruptedException, IOException { 144 //Create a upload session 145 BoxFileUploadSession.Info session = this.createUploadSession(boxApi, folderId, url, fileName, fileSize); 146 return this.uploadHelper(session, stream, fileSize, fileAttributes); 147 } 148 149 /** 150 * Creates a new version of a large file. 151 * 152 * @param boxApi the API connection to be used by the upload session. 153 * @param stream the input stream that feeds the content of the file. 154 * @param url the upload session URL. 155 * @param fileSize the total size of the file. 156 * @return the file instance that also contains the version information. 157 * @throws InterruptedException when a thread gets interupted. 158 * @throws IOException when reading a stream throws exception. 159 */ 160 public BoxFile.Info upload(BoxAPIConnection boxApi, InputStream stream, URL url, long fileSize) 161 throws InterruptedException, IOException { 162 //creates a upload session 163 BoxFileUploadSession.Info session = this.createUploadSession(boxApi, url, fileSize); 164 return this.uploadHelper(session, stream, fileSize, null); 165 } 166 167 /** 168 * Creates a new version of a large file and sets file attributes. 169 * 170 * @param boxApi the API connection to be used by the upload session. 171 * @param stream the input stream that feeds the content of the file. 172 * @param url the upload session URL. 173 * @param fileSize the total size of the file. 174 * @param fileAttributes file attributes to set. 175 * @return the file instance that also contains the version information. 176 * @throws InterruptedException when a thread gets interupted. 177 * @throws IOException when reading a stream throws exception. 178 */ 179 public BoxFile.Info upload(BoxAPIConnection boxApi, InputStream stream, URL url, long fileSize, 180 Map<String, String> fileAttributes) 181 throws InterruptedException, IOException { 182 //creates a upload session 183 BoxFileUploadSession.Info session = this.createUploadSession(boxApi, url, fileSize); 184 return this.uploadHelper(session, stream, fileSize, fileAttributes); 185 } 186 187 private BoxFile.Info uploadHelper(BoxFileUploadSession.Info session, InputStream stream, long fileSize, 188 Map<String, String> fileAttributes) 189 throws InterruptedException { 190 //Upload parts using the upload session 191 MessageDigest digest; 192 try { 193 digest = MessageDigest.getInstance(DIGEST_ALGORITHM_SHA1); 194 } catch (NoSuchAlgorithmException ae) { 195 throw new BoxAPIException("Digest algorithm not found", ae); 196 } 197 DigestInputStream dis = new DigestInputStream(stream, digest); 198 List<BoxFileUploadSessionPart> parts = this.uploadParts(session, dis, fileSize); 199 200 //Creates the file hash 201 byte[] digestBytes = digest.digest(); 202 String digestStr = Base64.encode(digestBytes); 203 204 //Commit the upload session. If there is a failure, abort the commit. 205 try { 206 return session.getResource().commit(digestStr, parts, fileAttributes, null, null); 207 } catch (Exception e) { 208 session.getResource().abort(); 209 throw new BoxAPIException("Unable to commit the upload session", e); 210 } 211 } 212 213 private BoxFileUploadSession.Info createUploadSession(BoxAPIConnection boxApi, URL url, long fileSize) { 214 BoxJSONRequest request = new BoxJSONRequest(boxApi, url, HttpMethod.POST); 215 216 //Creates the body of the request 217 JsonObject body = new JsonObject(); 218 body.add("file_size", fileSize); 219 request.setBody(body.toString()); 220 221 BoxJSONResponse response = (BoxJSONResponse) request.send(); 222 JsonObject jsonObject = Json.parse(response.getJSON()).asObject(); 223 224 String sessionId = jsonObject.get("id").asString(); 225 BoxFileUploadSession session = new BoxFileUploadSession(boxApi, sessionId); 226 227 return session.new Info(jsonObject); 228 } 229 230 /* 231 * Upload parts of the file. The part size is retrieved from the upload session. 232 */ 233 private List<BoxFileUploadSessionPart> uploadParts(BoxFileUploadSession.Info session, InputStream stream, 234 long fileSize) throws InterruptedException { 235 List<BoxFileUploadSessionPart> parts = new ArrayList<>(); 236 237 int partSize = session.getPartSize(); 238 long offset = 0; 239 long processed = 0; 240 int partPostion = 0; 241 //Set the Max Queue Size to 1.5x the number of processors 242 double maxQueueSizeDouble = Math.ceil(this.executorService.getMaximumPoolSize() * 1.5); 243 int maxQueueSize = Double.valueOf(maxQueueSizeDouble).intValue(); 244 while (processed < fileSize) { 245 //Waiting for any thread to finish before 246 long timeoutForWaitingInMillis = TimeUnit.MILLISECONDS.convert(this.timeout, this.timeUnit); 247 if (this.executorService.getCorePoolSize() <= this.executorService.getActiveCount()) { 248 if (timeoutForWaitingInMillis > 0) { 249 Thread.sleep(LargeFileUpload.THREAD_POOL_WAIT_TIME_IN_MILLIS); 250 timeoutForWaitingInMillis -= THREAD_POOL_WAIT_TIME_IN_MILLIS; 251 } else { 252 throw new BoxAPIException("Upload parts timedout"); 253 } 254 } 255 if (this.executorService.getQueue().size() < maxQueueSize) { 256 long diff = fileSize - processed; 257 //The size last part of the file can be lesser than the part size. 258 if (diff < (long) partSize) { 259 partSize = (int) diff; 260 } 261 parts.add(null); 262 byte[] bytes = getBytesFromStream(stream, partSize); 263 this.executorService.execute( 264 new LargeFileUploadTask(session.getResource(), bytes, offset, 265 partSize, fileSize, parts, partPostion) 266 ); 267 268 //Increase the offset and proceesed bytes to calculate the Content-Range header. 269 processed += partSize; 270 offset += partSize; 271 partPostion++; 272 } 273 } 274 this.executorService.shutdown(); 275 this.executorService.awaitTermination(this.timeout, this.timeUnit); 276 return parts; 277 } 278 279 /** 280 * Generates the Base64 encoded SHA-1 hash for content available in the stream. 281 * It can be used to calculate the hash of a file. 282 * 283 * @param stream the input stream of the file or data. 284 * @return the Base64 encoded hash string. 285 */ 286 public String generateDigest(InputStream stream) { 287 MessageDigest digest; 288 try { 289 digest = MessageDigest.getInstance(DIGEST_ALGORITHM_SHA1); 290 } catch (NoSuchAlgorithmException ae) { 291 throw new BoxAPIException("Digest algorithm not found", ae); 292 } 293 294 //Calcuate the digest using the stream. 295 DigestInputStream dis = new DigestInputStream(stream, digest); 296 try { 297 int value = dis.read(); 298 while (value != -1) { 299 value = dis.read(); 300 } 301 } catch (IOException ioe) { 302 throw new BoxAPIException("Reading the stream failed.", ioe); 303 } 304 305 //Get the calculated digest for the stream 306 byte[] digestBytes = digest.digest(); 307 return Base64.encode(digestBytes); 308 } 309}