fix(database): Aurora Serverless setup and configuration

this commit fixes the original Aurora Serverless setup by setting
the initial database that copilot creates to postgres, and using the
ensure_db.py script to configure the Django database.

- copilot creates an Aurora Serverless storage cluster with an initial
  database called 'postgres'
- the web service manifest defines secrets related to the Django
  database
- to create the Django database, ensure_db.py and settings.py are
  updated to use the database credentials from the environment variable
  automatically created by copilot
- the start_aws.sh script now uses ensure_db.py
- the Aurora Data API is enabled to allow us to interact with the
  Aurora database by running SQL queries on the AWS console

Author: lalver1

bin/start_aws.sh

@@ -5,25 +5,10 @@ set -eu
 # since it was created via copilot storage init --name pems-db, the variable is 'PEMSDB_NAME'
 S3_BUCKET_NAME="$PEMSDB_NAME"
 S3_FIXTURE_PATH="fixtures.json"
-DJANGO_DB_FIXTURES="fixtures.json"
 
 echo "Downloading $S3_FIXTURE_PATH from bucket $S3_BUCKET_NAME"
 aws s3 cp "s3://${S3_BUCKET_NAME}/${S3_FIXTURE_PATH}" "${DJANGO_DB_FIXTURES}"
 echo "Download complete"
 
-# PostgreSQL database settings (username, host, dbname, password, port) are injected by Copilot as an environment variable
-# called 'POSTGRESWEB_SECRET' since the database was created via
-# copilot storage init -l workload -t Aurora -w web -n postgres-web --engine PostgreSQL --initial-db django
-
-python manage.py migrate
-
-# Load data fixtures (if any)
-valid_fixtures=$(echo "$DJANGO_DB_FIXTURES" | grep -e fixtures\.json$ || test $? = 1)
-
-if [[ -n "$valid_fixtures" ]]; then
-    python manage.py loaddata $DJANGO_DB_FIXTURES
-else
-    echo "No JSON fixtures to load"
-fi
-
+bin/setup.sh
 bin/start.sh

infra/copilot/web/addons/postgres-web.yml

@@ -12,59 +12,57 @@ Parameters:
   postgreswebDBName:
     Type: String
     Description: The name of the initial database to be created in the Aurora Serverless v2 cluster.
-    Default: django
+    Default: postgres
     # Cannot have special characters
     # Naming constraints: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Limits.html#RDS_Limits.Constraints
 Mappings:
   postgreswebEnvScalingConfigurationMap:
     dev:
       "DBMinCapacity": 0.5 # AllowedValues: from 0.5 through 128
-      "DBMaxCapacity": 8   # AllowedValues: from 0.5 through 128
+      "DBMaxCapacity": 8 # AllowedValues: from 0.5 through 128
 
     All:
       "DBMinCapacity": 0.5 # AllowedValues: from 0.5 through 128
-      "DBMaxCapacity": 8   # AllowedValues: from 0.5 through 128
+      "DBMaxCapacity": 8 # AllowedValues: from 0.5 through 128
 
 Resources:
   postgreswebDBSubnetGroup:
-    Type: 'AWS::RDS::DBSubnetGroup'
+    Type: "AWS::RDS::DBSubnetGroup"
     Properties:
       DBSubnetGroupDescription: Group of Copilot private subnets for Aurora Serverless v2 cluster.
       SubnetIds:
-        !Split [',', { 'Fn::ImportValue': !Sub '${App}-${Env}-PrivateSubnets' }]
+        !Split [",", { "Fn::ImportValue": !Sub "${App}-${Env}-PrivateSubnets" }]
   postgreswebSecurityGroup:
     Metadata:
-      'aws:copilot:description': 'A security group for your workload to access the Aurora Serverless v2 cluster postgresweb'
-    Type: 'AWS::EC2::SecurityGroup'
+      "aws:copilot:description": "A security group for your workload to access the Aurora Serverless v2 cluster postgresweb"
+    Type: "AWS::EC2::SecurityGroup"
     Properties:
-      GroupDescription: !Sub 'The Security Group for ${Name} to access Aurora Serverless v2 cluster postgresweb.'
+      GroupDescription: !Sub "The Security Group for ${Name} to access Aurora Serverless v2 cluster postgresweb."
       VpcId:
-        Fn::ImportValue:
-          !Sub '${App}-${Env}-VpcId'
+        Fn::ImportValue: !Sub "${App}-${Env}-VpcId"
       Tags:
         - Key: Name
-          Value: !Sub 'copilot-${App}-${Env}-${Name}-Aurora'
+          Value: !Sub "copilot-${App}-${Env}-${Name}-Aurora"
   postgreswebDBClusterSecurityGroup:
     Metadata:
-      'aws:copilot:description': 'A security group for your Aurora Serverless v2 cluster postgresweb'
+      "aws:copilot:description": "A security group for your Aurora Serverless v2 cluster postgresweb"
     Type: AWS::EC2::SecurityGroup
     Properties:
       GroupDescription: The Security Group for the Aurora Serverless v2 cluster.
       SecurityGroupIngress:
         - ToPort: 5432
           FromPort: 5432
           IpProtocol: tcp
-          Description: !Sub 'From the Aurora Security Group of the workload ${Name}.'
+          Description: !Sub "From the Aurora Security Group of the workload ${Name}."
           SourceSecurityGroupId: !Ref postgreswebSecurityGroup
       VpcId:
-        Fn::ImportValue:
-          !Sub '${App}-${Env}-VpcId'
+        Fn::ImportValue: !Sub "${App}-${Env}-VpcId"
       Tags:
         - Key: Name
-          Value: !Sub 'copilot-${App}-${Env}-${Name}-Aurora'
+          Value: !Sub "copilot-${App}-${Env}-${Name}-Aurora"
   postgreswebAuroraSecret:
     Metadata:
-      'aws:copilot:description': 'A Secrets Manager secret to store your DB credentials'
+      "aws:copilot:description": "A Secrets Manager secret to store your DB credentials"
     Type: AWS::SecretsManager::Secret
     Properties:
       Description: !Sub Aurora main user secret for ${AWS::StackName}
@@ -76,42 +74,59 @@ Resources:
         PasswordLength: 16
   postgreswebDBClusterParameterGroup:
     Metadata:
-      'aws:copilot:description': 'A DB parameter group for engine configuration values'
-    Type: 'AWS::RDS::DBClusterParameterGroup'
+      "aws:copilot:description": "A DB parameter group for engine configuration values"
+    Type: "AWS::RDS::DBClusterParameterGroup"
     Properties:
-      Description: !Ref 'AWS::StackName'
-      Family: 'aurora-postgresql16'
+      Description: !Ref "AWS::StackName"
+      Family: "aurora-postgresql16"
       Parameters:
-        client_encoding: 'UTF8'
+        client_encoding: "UTF8"
   postgreswebDBCluster:
     Metadata:
-      'aws:copilot:description': 'The postgresweb Aurora Serverless v2 database cluster'
-    Type: 'AWS::RDS::DBCluster'
+      "aws:copilot:description": "The postgresweb Aurora Serverless v2 database cluster"
+    Type: "AWS::RDS::DBCluster"
     Properties:
       MasterUsername:
-        !Join [ "",  [ '{{resolve:secretsmanager:', !Ref postgreswebAuroraSecret, ":SecretString:username}}" ]]
+        !Join [
+          "",
+          [
+            "{{resolve:secretsmanager:",
+            !Ref postgreswebAuroraSecret,
+            ":SecretString:username}}",
+          ],
+        ]
       MasterUserPassword:
-        !Join [ "",  [ '{{resolve:secretsmanager:', !Ref postgreswebAuroraSecret, ":SecretString:password}}" ]]
+        !Join [
+          "",
+          [
+            "{{resolve:secretsmanager:",
+            !Ref postgreswebAuroraSecret,
+            ":SecretString:password}}",
+          ],
+        ]
       DatabaseName: !Ref postgreswebDBName
-      Engine: 'aurora-postgresql'
-      EngineVersion: '16.2'
+      Engine: "aurora-postgresql"
+      EngineVersion: "16.2"
+      EnableHttpEndpoint: true # enable the Data API feature
       DBClusterParameterGroupName: !Ref postgreswebDBClusterParameterGroup
       DBSubnetGroupName: !Ref postgreswebDBSubnetGroup
       Port: 5432
       VpcSecurityGroupIds:
         - !Ref postgreswebDBClusterSecurityGroup
       ServerlessV2ScalingConfiguration:
         # Replace "All" below with "!Ref Env" to set different autoscaling limits per environment.
-        MinCapacity: !FindInMap [postgreswebEnvScalingConfigurationMap, All, DBMinCapacity]
-        MaxCapacity: !FindInMap [postgreswebEnvScalingConfigurationMap, All, DBMaxCapacity]
+        MinCapacity:
+          !FindInMap [postgreswebEnvScalingConfigurationMap, All, DBMinCapacity]
+        MaxCapacity:
+          !FindInMap [postgreswebEnvScalingConfigurationMap, All, DBMaxCapacity]
   postgreswebDBWriterInstance:
     Metadata:
-      'aws:copilot:description': 'The postgresweb Aurora Serverless v2 writer instance'
-    Type: 'AWS::RDS::DBInstance'
+      "aws:copilot:description": "The postgresweb Aurora Serverless v2 writer instance"
+    Type: "AWS::RDS::DBInstance"
     Properties:
       DBClusterIdentifier: !Ref postgreswebDBCluster
       DBInstanceClass: db.serverless
-      Engine: 'aurora-postgresql'
+      Engine: "aurora-postgresql"
       PromotionTier: 1
       AvailabilityZone: !Select
         - 0

infra/copilot/web/manifest.yml

@@ -42,7 +42,10 @@ variables: # Pass environment variables as key value pairs.
 
 secrets: # Pass secrets from AWS Systems Manager (SSM) Parameter Store.
   DJANGO_ALLOWED_HOSTS: /pems/web/DJANGO_ALLOWED_HOSTS # The key is the name of the environment variable, the value is the name of the SSM parameter.
-
+  DJANGO_DB_NAME: /pems/web/DJANGO_DB_NAME
+  DJANGO_DB_USER: /pems/web/DJANGO_DB_USER
+  DJANGO_DB_PASSWORD: /pems/web/DJANGO_DB_PASSWORD
+  DJANGO_DB_FIXTURES: /pems/web/DJANGO_DB_FIXTURES
 # You can override any of the values defined above by environment.
 #environments:
 #  test:

pems/core/management/commands/ensure_db.py

@@ -8,6 +8,8 @@
 from django.db import DEFAULT_DB_ALIAS
 from psycopg import Connection, sql
 
+from pems.settings import RUNTIME_ENVIRONMENT, RUNTIME_ENVS, aws_db_credentials
+
 
 class Command(BaseCommand):
     help = (
@@ -16,16 +18,22 @@ class Command(BaseCommand):
     )
 
     def _admin_connection(self, database=None):
-        # Try to get HOST/PORT from the default database settings first
-        # Fallback to environment variables if not found in settings
+        # Get HOST/PORT from the default database settings
         default_db_settings = settings.DATABASES.get(DEFAULT_DB_ALIAS, {})
         db_host = default_db_settings.get("HOST")
         db_port = default_db_settings.get("PORT")
 
-        postgres_maintenance_db = os.environ.get("POSTGRES_DB", "postgres")
+        if RUNTIME_ENVIRONMENT() == RUNTIME_ENVS.DEV:
+            db_credentials = aws_db_credentials()
+            postgres_maintenance_db = db_credentials.get("dbname")
+            admin_user = db_credentials.get("username")
+            admin_password = db_credentials.get("password")
+        else:
+            postgres_maintenance_db = os.environ.get("POSTGRES_DB", "postgres")
+            admin_user = os.environ.get("POSTGRES_USER", "postgres")
+            admin_password = os.environ.get("POSTGRES_PASSWORD")
+
         target_db = database or postgres_maintenance_db
-        admin_user = os.environ.get("POSTGRES_USER", "postgres")
-        admin_password = os.environ.get("POSTGRES_PASSWORD")
 
         if not admin_password:
             raise CommandError("POSTGRES_PASSWORD environment variable not set. Cannot establish admin connection.")

pems/settings.py

@@ -44,6 +44,12 @@ def RUNTIME_ENVIRONMENT():
     return env
 
 
+def aws_db_credentials():
+    """Helper to get AWS database credentials from environment variable."""
+    postgres_secret = os.environ.get("POSTGRESWEB_SECRET")
+    return json.loads(postgres_secret)
+
+
 # Application definition
 
 INSTALLED_APPS = [
@@ -101,26 +107,22 @@ def RUNTIME_ENVIRONMENT():
     "default": {
         "ENGINE": "django.db.backends.postgresql",
         "OPTIONS": {"sslmode": sslmode, "sslrootcert": sslrootcert},
+        "NAME": os.environ.get("DJANGO_DB_NAME", "django"),
+        "USER": os.environ.get("DJANGO_DB_USER", "django"),
+        "PASSWORD": os.environ.get("DJANGO_DB_PASSWORD"),
     }
 }
 if RUNTIME_ENVIRONMENT() == RUNTIME_ENVS.DEV:
-    postgres_web_secret = os.environ.get("POSTGRESWEB_SECRET")
-    db_credentials = json.loads(postgres_web_secret)
+    db_credentials = aws_db_credentials()
     DATABASES["default"].update(
         {
-            "NAME": db_credentials.get("dbname"),
-            "USER": db_credentials.get("username"),
-            "PASSWORD": db_credentials.get("password"),
             "HOST": db_credentials.get("host"),
             "PORT": db_credentials.get("port"),
         }
     )
 else:
     DATABASES["default"].update(
         {
-            "NAME": os.environ.get("DJANGO_DB_NAME", "django"),
-            "USER": os.environ.get("DJANGO_DB_USER", "django"),
-            "PASSWORD": os.environ.get("DJANGO_DB_PASSWORD"),
             "HOST": os.environ.get("POSTGRES_HOSTNAME", "postgres"),
             "PORT": os.environ.get("POSTGRES_PORT", "5432"),
         }

tests/pytest/test_settings.py

@@ -1,3 +1,8 @@
+import json
+
+from pems.settings import aws_db_credentials
+
+
 def test_runtime_environment_default(settings):
     assert settings.RUNTIME_ENVIRONMENT() == "local"
 
@@ -10,3 +15,9 @@ def test_runtime_environment__local(settings):
 def test_runtime_environment_dev(settings):
     settings.ALLOWED_HOSTS = ["*"]
     assert settings.RUNTIME_ENVIRONMENT() == "dev"
+
+
+def test_aws_db_credentials(monkeypatch):
+    creds = {"host": "db.example.com", "port": 5432}
+    monkeypatch.setenv("POSTGRESWEB_SECRET", json.dumps(creds))
+    assert aws_db_credentials() == creds