Feature/support storage signed put (#18)

* feat: Support Storage Put Signed URL (client upload)

* fix: put pre-signed url with option header

* add: pre-signed url add tagging header

* update: object_storage example with put object

* fix: alicloud option key_id and key_secret return

AliCloudStorageOption GetSecretID() and GetSecretKey()

---------

Co-authored-by: Xudong Liu <xudong.liu@ewp-group.com>

Author: sincerefly

cloud/examples/object_storage/object_storage.go

@@ -1,9 +1,14 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"io"
+	"log"
+	"net/http"
 	"time"
 
 	"github.com/byte-power/gorich/cloud"
@@ -29,11 +34,16 @@ func main() {
 	object_storage_examples("aws_bucket_name_xxx", optionForAWS)
 
 	optionForAliOSS := object_storage.AliCloudStorageOption{
-		CredentialType: "oidc_role_arn",
-		EndPoint:       "oss-cn-zhangjiakou.aliyuncs.com",
-		SessionName:    "test-rrsa-oidc-token",
+		//CredentialType: "oidc_role_arn",
+		//EndPoint:       "oss-cn-zhangjiakou.aliyuncs.com",
+		//SessionName:    "test-rrsa-oidc-token",
+
+		CredentialType:  "ak",
+		EndPoint:        "oss-cn-beijing.aliyuncs.com",
+		AccessKeyID:     "alicloud_access_key_id_xxx",
+		AccessKeySecret: "alicloud_access_key_secret_xxx",
 	}
-	object_storage_examples("my-bucket", optionForAliOSS)
+	object_storage_examples("alicloud_bucket_name_xxx", optionForAliOSS)
 }
 
 func object_storage_examples(bucketName string, option cloud.Option) {
@@ -186,4 +196,80 @@ func object_storage_examples(bucketName string, option cloud.Option) {
 			fmt.Printf("GetSignedURLForExistedKey %s %s\n", name, url)
 		}
 	}
+
+	// PutSignedURL examples
+	for name, item := range files {
+		// get pre-signed put url
+		opt := object_storage.PutHeaderOption{
+			ContentType: aws.String(item.ContentType),
+		}
+		url, err := service.PutSignedURL(name, 1*time.Hour, opt)
+		if err != nil {
+			fmt.Printf("GetSignedURL for object %s error %s\n", name, err)
+			return
+		}
+		fmt.Printf("GetSignedURL for put object  %s %s\n", name, url)
+
+		// put content to s3 with signed-url
+		if err := uploadContent(url, string(item.Body), item.ContentType); err != nil {
+			fmt.Printf("Error uploading content: %v\n", err)
+			return
+		}
+
+		// get pre-signed url for download and check
+		getSignedURL, err := service.GetSignedURL(name, 1*time.Hour)
+		if err != nil {
+			fmt.Printf("GetSignedURL for object %s error %s\n", name, err)
+			return
+		}
+		fmt.Printf("GetSignedURL for get object %s %s\n", name, getSignedURL)
+
+		// check content
+		content, err := accessContentBySignedURL(getSignedURL)
+		if err != nil {
+			log.Fatalf("Get content err: %v", err)
+		}
+		fmt.Println("Get content:", content)
+	}
+}
+
+// uploadContent uploads content to S3 using the provided presigned URL.
+func uploadContent(signedURL, content, contentType string) error {
+	req, err := http.NewRequest("PUT", signedURL, bytes.NewBuffer([]byte(content)))
+	if err != nil {
+		return fmt.Errorf("error creating PUT request: %w", err)
+	}
+	req.Header.Set("Content-Type", contentType)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("error executing PUT request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("upload failed with status code %d", resp.StatusCode)
+	}
+	fmt.Println("Content uploaded successfully")
+	return nil
+}
+
+// accessContentBySignedURL access content by Signed URL
+func accessContentBySignedURL(url string) (string, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", fmt.Errorf("access content failed: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("access content failed, status code: %d", resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("get content failed: %v", err)
+	}
+	return string(body), nil
 }

cloud/object_storage/alicloud_object_storage.go

@@ -17,27 +17,33 @@ import (
 var ossClientMap = make(map[string]*oss.Client)
 
 var (
-	ErrAliCloudStorageServiceCredentialTypeEmpty = errors.New("credential_type for alicloud storage service is empty")
-	ErrAliCloudStorageServiceEndPointEmpty       = errors.New("endpoint for alicloud storage service is empty")
-	ErrAliCloudStorageServiceSessionNameEmpty    = errors.New("session_name for alicloud storage service is empty")
+	ErrAliCloudStorageServiceCredentialTypeEmpty  = errors.New("credential_type for alicloud storage service is empty")
+	ErrAliCloudStorageServiceEndPointEmpty        = errors.New("endpoint for alicloud storage service is empty")
+	ErrAliCloudStorageServiceSessionNameEmpty     = errors.New("session_name for alicloud storage service is empty")
+	ErrAliCloudStorageServiceAccessKeyIDEmpty     = errors.New("access_key_id for alicloud storage service is empty")
+	ErrAliCloudStorageServiceAccessKeySecretEmpty = errors.New("access_key_secret for alicloud storage service is empty")
 )
 
 type AliCloudStorageOption struct {
-	CredentialType string // eg: "oidc_role_arn"
+	CredentialType string // eg: "oidc_role_arn" or "ak"
 	EndPoint       string // eg: "oss-cn-zhangjiakou.aliyuncs.com"
 	SessionName    string // eg: "test-rrsa-oidc-token"
+
+	// "ak" required
+	AccessKeyID     string
+	AccessKeySecret string
 }
 
 func (option AliCloudStorageOption) GetProvider() cloud.Provider {
 	return cloud.AliCloudStorageProvider
 }
 
 func (option AliCloudStorageOption) GetSecretID() string {
-	return ""
+	return option.AccessKeyID
 }
 
 func (option AliCloudStorageOption) GetSecretKey() string {
-	return ""
+	return option.AccessKeySecret
 }
 
 func (option AliCloudStorageOption) GetAssumeRoleArn() string {
@@ -79,8 +85,18 @@ func (option AliCloudStorageOption) check() error {
 	if option.EndPoint == "" {
 		return ErrAliCloudStorageServiceEndPointEmpty
 	}
-	if option.SessionName == "" {
-		return ErrAliCloudStorageServiceSessionNameEmpty
+
+	if option.CredentialType == "oidc_role_arn" {
+		if option.SessionName == "" {
+			return ErrAliCloudStorageServiceSessionNameEmpty
+		}
+	} else if option.CredentialType == "ak" {
+		if option.AccessKeyID == "" {
+			return ErrAliCloudStorageServiceAccessKeyIDEmpty
+		}
+		if option.AccessKeySecret == "" {
+			return ErrAliCloudStorageServiceAccessKeySecretEmpty
+		}
 	}
 	return nil
 }
@@ -109,17 +125,28 @@ func GetAliCloudObjectService(bucketName string, option cloud.Option) (ObjectSto
 		return &AliCloudObjectStorageService{client: client, bucketName: bucketName}, nil
 	}
 
-	cred, err := newOidcCredential(storageOption.CredentialType, storageOption.SessionName)
-	if err != nil {
-		return nil, err
-	}
-
-	provider := &aliCloudCredentialsProvider{
-		cred: cred,
-	}
-	client, err := oss.New(storageOption.EndPoint, "", "", oss.SetCredentialsProvider(provider))
-	if err != nil {
-		return nil, err
+	var client *oss.Client
+	if storageOption.CredentialType == "oidc_role_arn" {
+		cred, err := newOidcCredential(storageOption.CredentialType, storageOption.SessionName)
+		if err != nil {
+			return nil, err
+		}
+		provider := &aliCloudCredentialsProvider{
+			cred: cred,
+		}
+		ossClient, err := oss.New(storageOption.EndPoint, "", "", oss.SetCredentialsProvider(provider))
+		if err != nil {
+			return nil, err
+		}
+		client = ossClient
+	} else if storageOption.CredentialType == "ak" {
+		ossClient, err := oss.New(storageOption.EndPoint, storageOption.AccessKeyID, storageOption.AccessKeySecret)
+		if err != nil {
+			return nil, err
+		}
+		client = ossClient
+	} else {
+		return nil, fmt.Errorf("credential type '%s' unsupported", storageOption.CredentialType)
 	}
 
 	// cache client
@@ -319,3 +346,15 @@ func (service *AliCloudObjectStorageService) GetSignedURLForExistedKey(ctx conte
 	}
 	return service.GetSignedURL(key, duration)
 }
+
+func (service *AliCloudObjectStorageService) PutSignedURL(key string, duration time.Duration, option PutHeaderOption) (string, error) {
+	if key == "" {
+		return "", ErrObjectKeyEmpty
+	}
+	bucket, err := service.client.Bucket(service.bucketName)
+	if err != nil {
+		return "", err
+	}
+	options := option.ToAliCloudOptions()
+	return bucket.SignURL(key, oss.HTTPPut, int64(duration.Seconds()), options...)
+}

cloud/object_storage/aws_object_storage.go

@@ -204,6 +204,28 @@ func (service *AWSObjectStorageService) GetSignedURLForExistedKey(ctx context.Co
 	return service.GetSignedURL(key, duration)
 }
 
+func (service *AWSObjectStorageService) PutSignedURL(key string, duration time.Duration, option PutHeaderOption) (string, error) {
+	if key == "" {
+		return "", ErrObjectKeyEmpty
+	}
+
+	request, _ := service.client.PutObjectRequest(&s3.PutObjectInput{
+		Bucket:             &service.bucketName,
+		Key:                &key,
+		ContentDisposition: option.ContentDisposition,
+		ContentEncoding:    option.ContentEncoding,
+		ContentMD5:         option.ContentMD5,
+		ContentType:        option.ContentType,
+		ContentLength:      option.ContentLength,
+		Tagging:            option.Tagging,
+	})
+	url, err := request.Presign(duration)
+	if err != nil {
+		return "", err
+	}
+	return url, nil
+}
+
 func isNotFoundErrorForAWS(err error) bool {
 	awsErr, ok := err.(awserr.Error)
 	if !ok {

cloud/object_storage/object_storage.go

@@ -3,6 +3,11 @@ package object_storage
 import (
 	"context"
 	"errors"
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+	"github.com/tencentyun/cos-go-sdk-v5"
+	"net/http"
+	"net/url"
+	"strconv"
 	"time"
 
 	"github.com/byte-power/gorich/cloud"
@@ -24,6 +29,7 @@ type ObjectStorageService interface {
 	GetSignedURL(key string, duration time.Duration) (string, error)
 	// GetSignedURLForExistedKey generates signed url if key exists. If key does not exist, return error
 	GetSignedURLForExistedKey(ctx context.Context, key string, duration time.Duration) (string, error)
+	PutSignedURL(key string, duration time.Duration, option PutHeaderOption) (string, error)
 }
 
 type PutObjectInput struct {
@@ -75,3 +81,64 @@ func GetObjectStorageService(bucketName string, option cloud.Option) (ObjectStor
 	}
 	return nil, cloud.ErrUnsupportedCloudProvider
 }
+
+type PutHeaderOption struct {
+	ContentDisposition *string
+	ContentEncoding    *string
+	ContentMD5         *string
+	ContentType        *string
+	ContentLength      *int64
+	Tagging            *string
+}
+
+func (o *PutHeaderOption) ToAliCloudOptions() []oss.Option {
+
+	options := make([]oss.Option, 0)
+	if o.ContentDisposition != nil {
+		options = append(options, oss.ContentDisposition(*o.ContentDisposition))
+	}
+	if o.ContentEncoding != nil {
+		options = append(options, oss.ContentEncoding(*o.ContentEncoding))
+	}
+	if o.ContentMD5 != nil {
+		options = append(options, oss.ContentMD5(*o.ContentMD5))
+	}
+	if o.ContentType != nil {
+		options = append(options, oss.ContentType(*o.ContentType))
+	}
+	if o.ContentLength != nil {
+		options = append(options, oss.ContentLength(*o.ContentLength))
+	}
+	if o.Tagging != nil {
+		options = append(options, oss.SetHeader(oss.HTTPHeaderOssTagging, *o.Tagging))
+	}
+	return options
+}
+
+func (o *PutHeaderOption) ToTencentCloudOptions() *cos.PresignedURLOptions {
+
+	opt := &cos.PresignedURLOptions{
+		Query:  &url.Values{},
+		Header: &http.Header{},
+	}
+
+	if o.ContentDisposition != nil {
+		opt.Header.Add("Content-Disposition", *o.ContentDisposition)
+	}
+	if o.ContentEncoding != nil {
+		opt.Header.Add("Content-Encoding", *o.ContentEncoding)
+	}
+	if o.ContentMD5 != nil {
+		opt.Header.Add("Content-MD5", *o.ContentMD5)
+	}
+	if o.ContentType != nil {
+		opt.Header.Add("Content-Type", *o.ContentType)
+	}
+	if o.ContentLength != nil {
+		opt.Header.Add("Content-Length", strconv.FormatInt(*o.ContentLength, 10))
+	}
+	if o.Tagging != nil {
+		opt.Header.Add("x-cos-tagging", *o.Tagging)
+	}
+	return opt
+}

cloud/object_storage/tencentcloud_object_storage.go

@@ -257,3 +257,22 @@ func (service *TencentCloudObjectStorageService) GetSignedURLForExistedKey(ctx c
 	}
 	return service.GetSignedURL(key, duration)
 }
+
+func (service *TencentCloudObjectStorageService) PutSignedURL(key string, duration time.Duration, option PutHeaderOption) (string, error) {
+	if key == "" {
+		return "", ErrObjectKeyEmpty
+	}
+
+	options := option.ToTencentCloudOptions()
+
+	url, err := service.client.Object.GetPresignedURL(
+		context.Background(), http.MethodPut, key,
+		service.client.GetCredential().SecretID,
+		service.client.GetCredential().SecretKey,
+		duration, options,
+	)
+	if err != nil {
+		return "", err
+	}
+	return url.String(), nil
+}