Merge pull request #33024 from hashicorp/f-s3-us-east-1-regional

provider: Adds support for enabling regional endpoint for S3

Author: gdavison

.changelog/33024.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+provider: Adds the `s3_us_east_1_regional_endpoint` attribute to support using the regional S3 API endpoint in `us-east-1`.
+```

go.mod

@@ -66,8 +66,8 @@ require (
 	github.com/beevik/etree v1.2.0
 	github.com/google/go-cmp v0.5.9
 	github.com/hashicorp/aws-cloudformation-resource-schema-sdk-go v0.21.0
-	github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.33
-	github.com/hashicorp/aws-sdk-go-base/v2/awsv1shim/v2 v2.0.0-beta.34
+	github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.34
+	github.com/hashicorp/aws-sdk-go-base/v2/awsv1shim/v2 v2.0.0-beta.35
 	github.com/hashicorp/awspolicyequivalence v1.6.0
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320

go.sum

@@ -214,10 +214,10 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hashicorp/aws-cloudformation-resource-schema-sdk-go v0.21.0 h1:IUypt/TbXiJBkBbE3926CgnjD8IltAitdn7Yive61DY=
 github.com/hashicorp/aws-cloudformation-resource-schema-sdk-go v0.21.0/go.mod h1:cdTE6F2pCKQobug+RqRaQp7Kz9hIEqiSvpPmb6E5G1w=
-github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.33 h1:8ogYFA7VJSKIOvyODL2mJphWLYc87ZdoKR9Zg3UEAVM=
-github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.33/go.mod h1:D8McAXJE4Ki5B4MYCicebCGfH8f5s3hNJc8ubR8x//4=
-github.com/hashicorp/aws-sdk-go-base/v2/awsv1shim/v2 v2.0.0-beta.34 h1:t2/JBep3k7Maetnw6N5e9PKWVKPHpWEJHYz5avmeKl0=
-github.com/hashicorp/aws-sdk-go-base/v2/awsv1shim/v2 v2.0.0-beta.34/go.mod h1:yfSdY7j8d/ifBnvktBI58gZH+tg6fQLqKAtZtGYMVBo=
+github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.34 h1:WH0OOrhZe6wzOnA+ra0ZV0+5BWSElVriWmudH2S2cFw=
+github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.34/go.mod h1:cR5oVK+h10mSG4T9eHaBAYfacxUlYI5vNfJuIRMGfMA=
+github.com/hashicorp/aws-sdk-go-base/v2/awsv1shim/v2 v2.0.0-beta.35 h1:FLgIkz1RPYkYG62Q+u7M/JtU2tEKPUDMeDH+WtZ04ic=
+github.com/hashicorp/aws-sdk-go-base/v2/awsv1shim/v2 v2.0.0-beta.35/go.mod h1:AQknW73NE5hbAZn/ruNomae0OJUNf5xzsAi6yDndWgs=
 github.com/hashicorp/awspolicyequivalence v1.6.0 h1:7aadmkalbc5ewStC6g3rljx1iNvP4QyAhg2KsHx8bU8=
 github.com/hashicorp/awspolicyequivalence v1.6.0/go.mod h1:9IOaIHx+a7C0NfUNk1A93M7kHd5rJ19aoUx37LZGC14=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=

internal/conns/awsclient.go

@@ -33,14 +33,15 @@ type AWSClient struct {
 	Session                 *session_sdkv1.Session
 	TerraformVersion        string
 
-	awsConfig      *aws_sdkv2.Config
-	clients        map[string]any
-	conns          map[string]any
-	endpoints      map[string]string // From provider configuration.
-	httpClient     *http.Client
-	lock           sync.Mutex
-	s3UsePathStyle bool   // From provider configuration.
-	stsRegion      string // From provider configuration.
+	awsConfig                 *aws_sdkv2.Config
+	clients                   map[string]any
+	conns                     map[string]any
+	endpoints                 map[string]string // From provider configuration.
+	httpClient                *http.Client
+	lock                      sync.Mutex
+	s3UsePathStyle            bool                                      // From provider configuration.
+	s3UsEast1RegionalEndpoint endpoints_sdkv1.S3UsEast1RegionalEndpoint // From provider configuration.
+	stsRegion                 string                                    // From provider configuration.
 }
 
 // PartitionHostname returns a hostname with the provider domain suffix for the partition
@@ -145,6 +146,7 @@ func (client *AWSClient) apiClientConfig(servicePackageName string) map[string]a
 	switch servicePackageName {
 	case names.S3:
 		m["s3_use_path_style"] = client.s3UsePathStyle
+		m["s3_us_east_1_regional_endpoint"] = client.s3UsEast1RegionalEndpoint
 	case names.STS:
 		m["sts_region"] = client.stsRegion
 	}

internal/conns/config.go

@@ -13,6 +13,7 @@ import (
 	awsbase "github.com/hashicorp/aws-sdk-go-base/v2"
 	awsbasev1 "github.com/hashicorp/aws-sdk-go-base/v2/awsv1shim/v2"
 	basediag "github.com/hashicorp/aws-sdk-go-base/v2/diag"
+	"github.com/hashicorp/aws-sdk-go-base/v2/logging"
 	"github.com/hashicorp/terraform-plugin-log/tflog"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
 	"github.com/hashicorp/terraform-provider-aws/internal/errs"
@@ -41,6 +42,7 @@ type Config struct {
 	Region                         string
 	RetryMode                      aws_sdkv2.RetryMode
 	S3UsePathStyle                 bool
+	S3UsEast1RegionalEndpoint      endpoints_sdkv1.S3UsEast1RegionalEndpoint
 	SecretKey                      string
 	SharedConfigFiles              []string
 	SharedCredentialsFiles         []string
@@ -59,6 +61,8 @@ type Config struct {
 func (c *Config) ConfigureProvider(ctx context.Context, client *AWSClient) (*AWSClient, diag.Diagnostics) {
 	var diags diag.Diagnostics
 
+	ctx, logger := logging.NewTfLogger(ctx)
+
 	awsbaseConfig := awsbase.Config{
 		AccessKey:                     c.AccessKey,
 		APNInfo:                       StdUserAgentProducts(c.TerraformVersion),
@@ -70,6 +74,7 @@ func (c *Config) ConfigureProvider(ctx context.Context, client *AWSClient) (*AWS
 		Insecure:                      c.Insecure,
 		HTTPClient:                    client.HTTPClient(),
 		HTTPProxy:                     c.HTTPProxy,
+		Logger:                        logger,
 		MaxRetries:                    c.MaxRetries,
 		Profile:                       c.Profile,
 		Region:                        c.Region,
@@ -204,6 +209,7 @@ func (c *Config) ConfigureProvider(ctx context.Context, client *AWSClient) (*AWS
 	client.conns = make(map[string]any, 0)
 	client.endpoints = c.Endpoints
 	client.s3UsePathStyle = c.S3UsePathStyle
+	client.s3UsEast1RegionalEndpoint = c.S3UsEast1RegionalEndpoint
 	client.stsRegion = c.STSRegion
 
 	return client, diags

internal/provider/fwprovider/provider.go

@@ -95,6 +95,12 @@ func (p *fwprovider) Schema(ctx context.Context, req provider.SchemaRequest, res
 				Optional:    true,
 				Description: "Set this to true to enable the request to use path-style addressing,\ni.e., https://s3.amazonaws.com/BUCKET/KEY. By default, the S3 client will\nuse virtual hosted bucket addressing when possible\n(https://BUCKET.s3.amazonaws.com/KEY). Specific to the Amazon S3 service.",
 			},
+			"s3_us_east_1_regional_endpoint": schema.StringAttribute{
+				Optional: true,
+				Description: "Specifies whether S3 API calls in the `us-east-1` region use the legacy global endpoint or a regional endpoint. " + //lintignore:AWSAT003
+					"Valid values are `legacy` or `regional`. " +
+					"Can also be configured using the `AWS_S3_US_EAST_1_REGIONAL_ENDPOINT` environment variable or the `s3_us_east_1_regional_endpoint` shared config file parameter",
+			},
 			"secret_key": schema.StringAttribute{
 				Optional:    true,
 				Description: "The secret key for API operations. You can retrieve this\nfrom the 'Security & Credentials' section of the AWS console.",

internal/provider/provider.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
+	"github.com/aws/aws-sdk-go/aws/endpoints"
 	awsbase "github.com/hashicorp/aws-sdk-go-base/v2"
 	multierror "github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/terraform-plugin-log/tflog"
@@ -158,6 +159,13 @@ func New(ctx context.Context) (*schema.Provider, error) {
 					"use virtual hosted bucket addressing when possible\n" +
 					"(https://BUCKET.s3.amazonaws.com/KEY). Specific to the Amazon S3 service.",
 			},
+			"s3_us_east_1_regional_endpoint": {
+				Type:     schema.TypeString,
+				Optional: true,
+				Description: "Specifies whether S3 API calls in the `us-east-1` region use the legacy global endpoint or a regional endpoint. " + //lintignore:AWSAT003
+					"Valid values are `legacy` or `regional`. " +
+					"Can also be configured using the `AWS_S3_US_EAST_1_REGIONAL_ENDPOINT` environment variable or the `s3_us_east_1_regional_endpoint` shared config file parameter",
+			},
 			"secret_key": {
 				Type:     schema.TypeString,
 				Optional: true,
@@ -478,6 +486,14 @@ func configure(ctx context.Context, provider *schema.Provider, d *schema.Resourc
 		config.RetryMode = mode
 	}
 
+	if v, ok := d.Get("s3_us_east_1_regional_endpoint").(string); ok && v != "" {
+		endpoint, err := endpoints.GetS3UsEast1RegionalEndpoint(v)
+		if err != nil {
+			return nil, sdkdiag.AppendFromErr(diags, err)
+		}
+		config.S3UsEast1RegionalEndpoint = endpoint
+	}
+
 	if v, ok := d.GetOk("allowed_account_ids"); ok && v.(*schema.Set).Len() > 0 {
 		config.AllowedAccountIds = flex.ExpandStringValueSet(v.(*schema.Set))
 	}

internal/service/s3/service_package.go

@@ -7,6 +7,7 @@ import (
 	"context"
 
 	aws_sdkv1 "github.com/aws/aws-sdk-go/aws"
+	endpoints_sdkv1 "github.com/aws/aws-sdk-go/aws/endpoints"
 	request_sdkv1 "github.com/aws/aws-sdk-go/aws/request"
 	session_sdkv1 "github.com/aws/aws-sdk-go/aws/session"
 	s3_sdkv1 "github.com/aws/aws-sdk-go/service/s3"
@@ -21,6 +22,10 @@ func (p *servicePackage) NewConn(ctx context.Context, m map[string]any) (*s3_sdk
 		S3ForcePathStyle: aws_sdkv1.Bool(m["s3_use_path_style"].(bool)),
 	}
 
+	if v, ok := m["s3_us_east_1_regional_endpoint"]; ok {
+		config.S3UsEast1RegionalEndpoint = v.(endpoints_sdkv1.S3UsEast1RegionalEndpoint)
+	}
+
 	return s3_sdkv1.New(sess.Copy(config)), nil
 }
 

website/docs/index.html.markdown

@@ -241,6 +241,7 @@ credential_process = custom-process --username jdoe
 |Retry Mode|`retry_mode`|`AWS_RETRY_MODE`|`retry_mode`|
 |Shared Config Files|`shared_config_files`|`AWS_CONFIG_FILE`|N/A|
 |Shared Credentials Files|`shared_credentials_files`|`AWS_SHARED_CREDENTIALS_FILE`|N/A|
+|S3 Use Regional Endpoint for `us-east-1`|`s3_us_east_1_regional_endpoint`|`AWS_S3_US_EAST_1_REGIONAL_ENDPOINT`|`s3_us_east_1_regional_endpoint`|
 |Use DualStack Endpoints|`use_dualstack_endpoint`|`AWS_USE_DUALSTACK_ENDPOINT`|`use_dualstack_endpoint`|
 |Use FIPS Endpoints|`use_fips_endpoint`|`AWS_USE_FIPS_ENDPOINT`|`use_fips_endpoint`|
 
@@ -326,7 +327,13 @@ In addition to [generic `provider` arguments](https://www.terraform.io/docs/conf
 * `retry_mode` - (Optional) Specifies how retries are attempted.
   Valid values are `standard` and `adaptive`.
   Can also be configured using the `AWS_RETRY_MODE` environment variable or the shared config file parameter `retry_mode`.
-* `s3_use_path_style` - (Optional) Whether to enable the request to use path-style addressing, i.e., `https://s3.amazonaws.com/BUCKET/KEY`. By default, the S3 client will use virtual hosted bucket addressing, `https://BUCKET.s3.amazonaws.com/KEY`, when possible. Specific to the Amazon S3 service.
+* `s3_use_path_style` - (Optional) Whether to enable the request to use path-style addressing, i.e., `https://s3.amazonaws.com/BUCKET/KEY`.
+  By default, the S3 client will use virtual hosted bucket addressing, `https://BUCKET.s3.amazonaws.com/KEY`, when possible.
+  Specific to the Amazon S3 service.
+* `s3_us_east_1_regional_endpoint` - (Optional) Specifies whether S3 API calls in the `us-east-1` region use the legacy global endpoint or a regional endpoint.
+  Valid values are `legacy` or `regional`.
+  Can also be configured using the `AWS_S3_US_EAST_1_REGIONAL_ENDPOINT` environment variable or the `s3_us_east_1_regional_endpoint` shared config file parameter.
+  Specific to the Amazon S3 service.
 * `secret_key` - (Optional) AWS secret key. Can also be set with the `AWS_SECRET_ACCESS_KEY` environment variable, or via a shared configuration and credentials files if `profile` is used. See also `access_key`.
 * `shared_config_files` - (Optional) List of paths to AWS shared config files. If not set, the default is `[~/.aws/config]`. A single value can also be set with the `AWS_CONFIG_FILE` environment variable.
 * `shared_credentials_files` - (Optional) List of paths to the shared credentials file. If not set and a profile is used, the default value is `[~/.aws/credentials]`. A single value can also be set with the `AWS_SHARED_CREDENTIALS_FILE` environment variable.