Separate app and infra deployments

.github/workflows/build-and-push-image.yml

@@ -3,21 +3,21 @@ name: Build and push image
 on:
   workflow_dispatch:
     inputs:
-      git-sha:
+      git_sha:
         description: The FULL git commit sha to build the image from (optional).
         type: string
   workflow_call:
     inputs:
-      git-sha:
+      git_sha:
         description: The git commit sha to build the image from.
         type: string
 
 env:
   PUSH_IMAGE_TO_PRODUCTION: ${{ github.ref_name == 'main' }}
-  git_ref: ${{ inputs.git-sha || github.sha }}
+  git_ref: ${{ inputs.git_sha || github.sha }}
 
 concurrency:
-  group: build-and-push-image-${{ inputs.git-sha || github.sha }}
+  group: build-and-push-image-${{ inputs.git_sha || github.sha }}
 
 permissions: {}
 

.github/workflows/data-replication-pipeline.yml

@@ -1,5 +1,5 @@
-name: Data replication pipeline
-run-name: ${{ inputs.deployment_type }} for data replication resources for ${{ inputs.environment }}
+name: Deploy data replication
+run-name: Deploy data replication to ${{ inputs.environment }}
 
 on:
   workflow_dispatch:
@@ -15,252 +15,156 @@ on:
           - qa
           - sandbox-alpha
           - sandbox-beta
-      deployment_type:
-        description: Deployment type
-        required: true
-        type: choice
-        options:
-          - Deployment with DB recreation
-          - Application only deployment
-      image_tag:
-        description: Docker image tag to deploy
-        required: false
-        type: string
-      db_snapshot_arn:
-        description: ARN of the DB snapshot to use (optional)
-        required: false
+      git_ref_to_deploy:
+        description:
+          | # Use blank unicode character (U+2800) to force line-break
+          Use code from: ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
+          (Git ref to deploy, for example, a tag, branch name or commit SHA. Will use workflow ref if not provided.)
         type: string
-      egress_cidr:
-        description: CIDR blocks to allow egress traffic.
-        type: string
-        required: true
-        default: "[]"
-      take_db_snapshot:
-        description: Take a new DB snapshot before creating the environment
-        type: boolean
-        default: false
+
+permissions: {}
 
 env:
   environment: ${{ inputs.environment }}
   deployment_type: ${{ inputs.deployment_type }}
   db_snapshot_arn: ${{ inputs.db_snapshot_arn }}
   egress_cidr: ${{ inputs.egress_cidr }}
   take_db_snapshot: ${{ inputs.take_db_snapshot }}
+  git_ref: ${{ inputs.git_ref_to_deploy || github.sha }}
   aws_role: ${{ inputs.environment == 'production'
     && 'arn:aws:iam::820242920762:role/GithubDeployDataReplicationInfrastructure'
     || 'arn:aws:iam::393416225559:role/GithubDeployDataReplicationInfrastructure' }}
-  db_snapshot_role: ${{ inputs.environment == 'production'
-    && 'arn:aws:iam::820242920762:role/DatabaseSnapshotRole'
-    || 'arn:aws:iam::393416225559:role/DatabaseSnapshotRole' }}
-
-defaults:
-  run:
-    working-directory: terraform/data_replication
+  aws_account_id: ${{ inputs.environment == 'production' && '820242920762' || '393416225559' }}
 
 concurrency:
   group: deploy-data-replica-${{ inputs.environment }}
 
 jobs:
-  prepare-db-replica:
-    if: ${{ inputs.deployment_type == 'Deployment with DB recreation' }}
-    name: Prepare data replica
+  validate-inputs:
     runs-on: ubuntu-latest
-    permissions:
-      id-token: write
+    permissions: {}
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v5
-      - name: Assume DB Snapshot role
-        if: inputs.take_db_snapshot
-        uses: aws-actions/configure-aws-credentials@v5
-        with:
-          role-to-assume: ${{ env.db_snapshot_role }}
-          aws-region: eu-west-2
-      - name: Take DB snapshot
-        if: inputs.take_db_snapshot
+      - name: Validate inputs
         run: |
-          set -e
-          snapshot_identifier=snapshot-for-data-replication-$(date +"%Y-%m-%d-%H-%M-%S")
-          aws rds create-db-cluster-snapshot --db-cluster-identifier mavis-$environment --db-cluster-snapshot-identifier $snapshot_identifier
-          echo "Waiting for snapshot to be available. This can take a while."
-          aws rds wait db-cluster-snapshot-available --db-cluster-snapshot-identifier $snapshot_identifier
-          echo "New snapshot is now available"
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v5
-        with:
-          role-to-assume: ${{ env.aws_role }}
-          aws-region: eu-west-2
-      - name: Get latest snapshot
-        id: get-latest-snapshot
-        run: |
-          set -e
-          if [ -z "$db_snapshot_arn" ]; then
-              echo "No snapshot ARN provided, fetching the latest snapshot"
-              SNAPSHOT_ARN=$(aws rds describe-db-cluster-snapshots \
-              --query "DBClusterSnapshots[?DBClusterIdentifier=='mavis-$environment'].[DBClusterSnapshotArn, SnapshotCreateTime]" \
-              --output text | sort -k2 -r | head -n 1 | cut -f1)
-
-              if [ -z "$SNAPSHOT_ARN" ]; then
-                  echo "No snapshots found for mavis-$environment"
-                  exit 1
-              fi
-          else
-              echo "Using provided snapshot ARN: $db_snapshot_arn"
-              SNAPSHOT_ARN="$db_snapshot_arn"
+          if [[ "$environment" == "preview" || "$environment" == "production" ]]; then
+            if [[ -z "$git_ref_to_deploy" ]]; then
+              echo "Error: git_ref_to_deploy is required for preview and production environments."
+              exit 1
+            fi
           fi
-          echo "Using snapshot ARN: $SNAPSHOT_ARN"
-          echo "SNAPSHOT_ARN=$SNAPSHOT_ARN" >> $GITHUB_OUTPUT
-      - name: Install terraform
-        uses: hashicorp/setup-terraform@v3
-        with:
-          terraform_version: 1.13.3
-    outputs:
-      SNAPSHOT_ARN: ${{ steps.get-latest-snapshot.outputs.SNAPSHOT_ARN }}
 
-  prepare-webapp:
-    name: Prepare webapp
+  determine-git-sha:
     runs-on: ubuntu-latest
-    permissions:
-      id-token: write
+    permissions: {}
+    needs: validate-inputs
+    outputs:
+      git-sha: ${{ steps.get-git-sha.outputs.git-sha }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v5
         with:
-          role-to-assume: ${{ env.aws_role }}
-          aws-region: eu-west-2
-      - name: ECR login
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
-      - name: Get docker image digest
-        id: get-docker-image-digest
-        env:
-          IMAGE_TAG: ${{ inputs.image_tag || github.sha }}
-        run: |
-          set -e
-          DOCKER_IMAGE="${{ steps.login-ecr.outputs.registry }}/mavis/webapp:$IMAGE_TAG"
-          docker pull "$DOCKER_IMAGE"
-          DOCKER_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' "$DOCKER_IMAGE")
-          DIGEST="${DOCKER_DIGEST#*@}"
-          echo "DIGEST=$DIGEST" >> $GITHUB_OUTPUT
-    outputs:
-      DOCKER_DIGEST: ${{ steps.get-docker-image-digest.outputs.DIGEST }}
+          ref: ${{ env.git_ref }}
+      - name: Get git sha
+        id: get-git-sha
+        run: echo "git-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
 
-  plan:
-    name: Terraform plan
+  build-and-push-image:
+    permissions:
+      id-token: write
+    needs: determine-git-sha
+    uses: ./.github/workflows/build-and-push-image.yml
+    with:
+      git_sha: ${{ needs.determine-git-sha.outputs.git-sha }}
+
+  prepare-deployment:
+    name: Prepare deployment
     runs-on: ubuntu-latest
-    needs:
-      - prepare-db-replica
-      - prepare-webapp
-    if: ${{ !cancelled() &&
-          (needs.prepare-db-replica.result == 'success' || needs.prepare-db-replica.result == 'skipped') &&
-          needs.prepare-webapp.result == 'success' }}
-    env:
-      SNAPSHOT_ARN: ${{ needs.prepare-db-replica.outputs.SNAPSHOT_ARN }}
-      DB_SECRET_ARN: ${{ needs.prepare-db-replica.outputs.DB_SECRET_ARN || 'arn:aws:secretsmanager:eu-west-2:000000000000:secret:placeholder' }}
-      DOCKER_DIGEST: ${{ needs.prepare-webapp.outputs.DOCKER_DIGEST }}
-      REPLACE_DB_CLUSTER: ${{ inputs.deployment_type == 'Deployment with DB recreation' }}
+    needs: build-and-push-image
     permissions:
       id-token: write
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
+        with:
+          ref: ${{ env.git_ref }}
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v5
         with:
           role-to-assume: ${{ env.aws_role }}
           aws-region: eu-west-2
-      - name: Install terraform
-        uses: hashicorp/setup-terraform@v3
-        with:
-          terraform_version: 1.13.3
-      - name: Get db secret arn
-        id: get-db-secret-arn
-        working-directory: terraform/app
+      - name: Get image digest
+        id: get-image-digest
         run: |
-          terraform init -backend-config="env/$environment-backend.hcl" -upgrade
-          DB_SECRET_ARN=$(terraform output --raw db_secret_arn)
-          echo "DB_SECRET_ARN=$DB_SECRET_ARN" >> $GITHUB_OUTPUT
-      - name: Terraform Plan
-        id: plan
+          digest=$(aws ecr describe-images \
+            --repository-name mavis/webapp \
+            --image-ids imageTag=$git_ref \
+            --query 'imageDetails[0].imageDigest' \
+            --output text)
+          echo "digest=$digest" >> $GITHUB_OUTPUT
+      - name: Parse environment variables
+        id: parse-environment-variables
+        env:
+          MAVIS__PDS__RATE_LIMIT_PER_SECOND: ${{ inputs.environment == 'production' && 50 || 5 }}
         run: |
-          set -eo pipefail
-          terraform init -backend-config="env/$environment-backend.hcl" -upgrade
-
-          PLAN_ARGS=(
-            "plan"
-            "-var=image_digest=$DOCKER_DIGEST"
-            "-var=db_secret_arn=${{ steps.get-db-secret-arn.outputs.DB_SECRET_ARN }}"
-            "-var=imported_snapshot=$SNAPSHOT_ARN"
-            "-var-file=env/$environment.tfvars"
-            "-var=allowed_egress_cidr_blocks=$egress_cidr"
-            "-out=${{ runner.temp }}/tfplan"
-          )
-
-          if [ "$REPLACE_DB_CLUSTER" = "true" ]; then
-            PLAN_ARGS+=("-replace" "aws_rds_cluster.cluster")
-          fi
-          terraform "${PLAN_ARGS[@]}" | tee ${{ runner.temp }}/tf_stdout
-      - name: Upload artifact
-        uses: actions/upload-artifact@v4
+          {
+            echo 'parsed_env_vars<<EOF'
+            echo "MAVIS__SPLUNK__ENABLED=false"
+            echo "MAVIS__CIS2__ENABLED=false"
+            echo "MAVIS__PDS__ENQUEUE_BULK_UPDATES=false"
+            echo "MAVIS__PDS__RATE_LIMIT_PER_SECOND=$MAVIS__PDS__RATE_LIMIT_PER_SECOND"
+            echo 'EOF'
+          } >> "$GITHUB_OUTPUT"
+      - name: Populate web task definition
+        id: create-task-definition
+        uses: aws-actions/amazon-ecs-render-task-definition@v1
         with:
-          name: tfplan_infrastructure-${{ inputs.environment }}
-          path: ${{ runner.temp }}/tfplan
-      - name: Notify pending approval
-        if: inputs.environment == 'production'
-        uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a
+          task-definition-family: "mavis-data-replication-task-definition-${{ inputs.environment }}-template"
+          container-name: "application"
+          image: "${{ env.aws_account_id }}.dkr.ecr.eu-west-2.amazonaws.com/mavis/webapp@${{ steps.get-image-digest.outputs.digest }}"
+          environment-variables: ${{ steps.parse-environment-variables.outputs.parsed_env_vars }}
+      - name: Rename task definition file
+        run: mv ${{ steps.create-task-definition.outputs.task-definition }} ${{ runner.temp }}/data-replication-task-definition.json
+      - name: Upload artifact for data-replication task definition
+        uses: actions/upload-artifact@v4
         with:
-          webhook: ${{ secrets.SLACK_MAVIS_TECH_WEBHOOK_URL }}
-          webhook-type: incoming-webhook
-          payload: |
-            text: ":hourglass: Approval required :hourglass:"
-            blocks:
-              - type: "section"
-                text:
-                  type: "mrkdwn"
-                  text: "${{ github.workflow }} requires approval"
-              - type: "section"
-                fields:
-                  - type: "mrkdwn"
-                    text: "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View workflow run>"
-                  - type: "mrkdwn"
-                    text: "*Triggered by:*\n${{ github.actor }}"
+          name: ${{ inputs.environment }}-data-replication-task-definition
+          path: ${{ runner.temp }}/data-replication-task-definition.json
 
-  apply:
-    name: Terraform apply
+  approve-deployments:
+    name: Wait for approval if required
     runs-on: ubuntu-latest
-    needs: plan
-    if: ${{ !cancelled() && needs.plan.result == 'success' }}
+    needs: prepare-deployment
     environment: ${{ inputs.environment }}
+    steps:
+      - run: echo "Proceeding with deployment to $environment environment"
+
+  deploy-data-replication:
+    name: Deploy data-replication service
+    runs-on: ubuntu-latest
+    needs: [prepare-deployment, approve-deployments]
     permissions:
       id-token: write
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v5
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v5
         with:
           role-to-assume: ${{ env.aws_role }}
           aws-region: eu-west-2
-      - name: Download artifact
+      - name: Download data-replication task definition artifact
         uses: actions/download-artifact@v5
         with:
-          name: tfplan_infrastructure-${{ inputs.environment }}
           path: ${{ runner.temp }}
-      - name: Install terraform
-        uses: hashicorp/setup-terraform@v3
-        with:
-          terraform_version: 1.13.3
-      - name: Apply the changes
-        run: |
-          set -e
-          terraform init -backend-config="env/$environment-backend.hcl" -upgrade
-          terraform apply ${{ runner.temp }}/tfplan
-      - name: Deploy db-access-service
+          name: ${{ inputs.environment }}-data-replication-task-definition
+      - name: Change family of task definition
         run: |
-          task_definition_arn=$(terraform output -raw task_definition_arn)
-          aws ecs update-service \
-            --cluster "mavis-$environment-data-replication" \
-            --service "mavis-$environment-data-replication" \
-            --task-definition $task_definition_arn
+          file_path="${{ runner.temp }}/data-replication-task-definition.json"
+          family_name="mavis-data-replication-task-definition-$environment"
+          echo "$(jq --arg f "$family_name" '.family = $f' "$file_path")" > "$file_path"
+      - name: Deploy data-replication service
+        uses: aws-actions/amazon-ecs-deploy-task-definition@v2
+        with:
+          task-definition: ${{ runner.temp }}/data-replication-task-definition.json
+          cluster: mavis-${{ inputs.environment }}-data-replication
+          service: mavis-${{ inputs.environment }}-data-replication
+          force-new-deployment: true
+          wait-for-service-stability: true