Merge pull request #237 from tencentyun/dev/checkPointUpload

upload part preflight

Author: Dzkol

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.qcloud</groupId>
     <artifactId>cos_api</artifactId>
-    <version>5.6.244</version>
+    <version>5.6.244.2</version>
     <packaging>jar</packaging>
     <name>cos-java-sdk</name>
     <description>java sdk for qcloud cos</description>

src/main/java/com/qcloud/cos/COSClient.java

@@ -166,7 +166,7 @@ public class COSClient implements COS {
 
     private CosHttpClient cosHttpClient;
 
-    private ConcurrentHashMap<String, Long> preflightBuckets = new ConcurrentHashMap<>();
+    private static ConcurrentHashMap<String, Long> preflightBuckets = new ConcurrentHashMap<>();
 
     public COSClient(COSCredentials cred, ClientConfig clientConfig) {
         this(new COSStaticCredentialsProvider(cred), clientConfig);
@@ -1699,6 +1699,18 @@ public UploadPartResult uploadPart(UploadPartRequest uploadPartRequest)
         rejectNull(partSize, "The part size parameter must be specified when uploading a part");
         rejectNull(clientConfig.getRegion(),
                 "region is null, region in clientConfig must be specified when uploading a part");
+
+        try {
+            preflightObj(uploadPartRequest);
+        } catch (CosServiceException cse) {
+            String msg = String.format("fail to do the preflight request due to the service exception[statusCode:%s, requestId:%s], will not do the upload part request", cse.getStatusCode(), cse.getRequestId());
+            log.warn(msg);
+            throw cse;
+        } catch (CosClientException cce) {
+            log.warn("fail to do the preflight request due to the client exception, will not do the upload part request", cce);
+            throw cce;
+        }
+
         CosHttpRequest<UploadPartRequest> request =
                 createRequest(bucketName, key, uploadPartRequest, HttpMethodName.PUT);
         request.addParameter("uploadId", uploadId);
@@ -1762,6 +1774,21 @@ private UploadPartResult doUploadPart(final String bucketName, final String key,
             ObjectMetadata metadata = invoke(request, new CosMetadataResponseHandler());
             final String etag = metadata.getETag();
 
+            if (metadata.isNeedPreflight()) {
+                Long currentTime = System.currentTimeMillis();
+                if ((preflightBuckets.get(bucketName) == null) || ((currentTime - preflightBuckets.get(bucketName)) > clientConfig.getPreflightStatusUpdateInterval())) {
+                    String reqMsg = String.format("will update preflight status, bucket[%s]", bucketName);
+                    log.info(reqMsg);
+                    preflightBuckets.put(bucketName, currentTime);
+                }
+            } else {
+                Long currentTime = System.currentTimeMillis();
+                if ((preflightBuckets.get(bucketName) != null) && ((currentTime - preflightBuckets.get(bucketName)) > clientConfig.getPreflightStatusUpdateInterval())) {
+                    String reqMsg = String.format("will remove bucket[%s] from preflight lists", bucketName);
+                    log.info(reqMsg);
+                    preflightBuckets.remove(bucketName);
+                }
+            }
 
             if (md5DigestStream != null && !skipMd5CheckStrategy
                     .skipClientSideValidationPerUploadPartResponse(metadata)) {
@@ -5576,5 +5603,37 @@ private void preflightObj(PutObjectRequest putObjectRequest) throws CosClientExc
             invoke(request, voidCosResponseHandler);
         }
     }
+
+    private void preflightObj(UploadPartRequest uploadPartRequest) throws CosClientException, CosServiceException {
+        String bucketName = uploadPartRequest.getBucketName();
+        String key = uploadPartRequest.getKey();
+        rejectEmpty(bucketName,
+                "The bucket name parameter must be specified when doing preflight request");
+        rejectEmpty(key,
+                "The key parameter must be specified when doing preflight request");
+        if (clientConfig.isCheckPreflightStatus() && preflightBuckets.containsKey(bucketName)) {
+            String reqMsg = String.format("will do preflight request for upload part object[%s] to the bucket[%s]", key, bucketName);
+            log.debug(reqMsg);
+            uploadPartRequest.setHasDonePreflight(true);
+            CosServiceRequest serviceRequest = new CosServiceRequest();
+            Map<String, String> customHeaders = uploadPartRequest.getCustomRequestHeaders();
+            if (customHeaders != null) {
+                for (Map.Entry<String, String> e : customHeaders.entrySet()) {
+                    serviceRequest.putCustomRequestHeader(e.getKey(), e.getValue());
+                }
+            }
+            CosHttpRequest<CosServiceRequest> request = createRequest(bucketName, key, serviceRequest, HttpMethodName.HEAD);
+            if (uploadPartRequest.getFixedEndpointAddr() != null) {
+                request.setEndpoint(uploadPartRequest.getFixedEndpointAddr());
+            }
+            request.addParameter("preflight", null);
+            ObjectMetadata metadata = uploadPartRequest.getObjectMetadata();
+            if (metadata != null) {
+                populateRequestMetadata(request, metadata);
+            }
+            request.addHeader("x-cos-next-action", "UploadPart");
+            invoke(request, voidCosResponseHandler);
+        }
+    }
 }
 

src/main/java/com/qcloud/cos/model/UploadPartRequest.java

@@ -99,6 +99,7 @@ public class UploadPartRequest extends CosServiceRequest implements SSECustomerK
      */
     private SSECustomerKey sseCustomerKey;
 
+    private boolean hasDonePreflight = false;
 
     /**
      * Sets the stream containing the data to upload for the new part.
@@ -583,4 +584,12 @@ public int getTrafficLimit() {
     public void setTrafficLimit(int trafficLimit) {
         this.trafficLimit = trafficLimit;
     }
+
+    public boolean hasDonePreflight() {
+        return hasDonePreflight;
+    }
+
+    public void setHasDonePreflight(boolean hasDonePreflight) {
+        this.hasDonePreflight = hasDonePreflight;
+    }
 }

src/main/java/com/qcloud/cos/retry/PredefinedRetryPolicies.java

@@ -25,6 +25,7 @@
 import com.qcloud.cos.internal.CosServiceRequest;
 
 import com.qcloud.cos.model.PutObjectRequest;
+import com.qcloud.cos.model.UploadPartRequest;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.CircularRedirectException;
 import org.slf4j.Logger;
@@ -74,6 +75,13 @@ public <X extends CosServiceRequest> boolean shouldRetry(CosHttpRequest<X> reque
                     }
                 }
 
+                if (!isRetryAfterPreflight() && request.getOriginalRequest() != null && request.getOriginalRequest() instanceof UploadPartRequest && ((UploadPartRequest) request.getOriginalRequest()).hasDonePreflight()) {
+                    if (((CosServiceException) exception).getStatusCode() == 503 && Objects.equals(((CosServiceException) exception).getErrorCode(), "SlowDown")) {
+                        log.info("will not retry for 503 while upload part, because preflight request has been done");
+                        return false;
+                    }
+                }
+
                 return true;
             }