Update workflows

- Make matrix functionality respect server selection
- Update data replication to use new ECS service/deployment flow
- Cleanup unnecessary components

Author: TheOneFromNorway

.github/workflows/data-replication-pipeline.yml

@@ -1,4 +1,4 @@
-name: Data replication pipeline
+name: Deploy Data Replication
 run-name: ${{ inputs.deployment_type }} for data replication resources for ${{ inputs.environment }}
 
 on:
@@ -15,26 +15,10 @@ on:
           - qa
           - sandbox-alpha
           - sandbox-beta
-      deployment_type:
-        description: Deployment type
-        required: true
-        type: choice
-        options:
-          - Deployment with DB recreation
-          - Application only deployment
       image_tag:
         description: Docker image tag to deploy
         required: false
         type: string
-      db_snapshot_arn:
-        description: ARN of the DB snapshot to use (optional)
-        required: false
-        type: string
-      egress_cidr:
-        description: CIDR blocks to allow egress traffic.
-        type: string
-        required: true
-        default: "[]"
 
 env:
   aws_role: ${{ inputs.environment == 'production'
@@ -49,171 +33,46 @@ concurrency:
   group: deploy-data-replica-${{ inputs.environment }}
 
 jobs:
-  prepare-db-replica:
-    if: ${{ inputs.deployment_type == 'Deployment with DB recreation' }}
-    name: Prepare data replica
+  validate-inputs:
     runs-on: ubuntu-latest
-    permissions:
-      id-token: write
+    permissions: { }
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v5
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: ${{ env.aws_role }}
-          aws-region: eu-west-2
-      - name: get latest snapshot
-        id: get-latest-snapshot
+      - name: Validate inputs
         run: |
-          set -e
-          if [ -z "${{ inputs.db_snapshot_arn }}" ]; then
-              echo "No snapshot ARN provided, fetching the latest snapshot"
-              SNAPSHOT_ARN=$(aws rds describe-db-cluster-snapshots \
-              --query "DBClusterSnapshots[?DBClusterIdentifier=='mavis-${{ inputs.environment }}'].[DBClusterSnapshotArn, SnapshotCreateTime]" \
-              --output text | sort -k2 -r | head -n 1 | cut -f1)
-          
-              if [ -z "$SNAPSHOT_ARN" ]; then
-                  echo "No snapshots found for mavis-${{ inputs.environment }}"
-                  exit 1
-              fi
-          else
-              echo "Using provided snapshot ARN: ${{ inputs.db_snapshot_arn }}"
-              SNAPSHOT_ARN="${{ inputs.db_snapshot_arn }}"
+          if [[ "${{ inputs.environment }}" == "preview" || "${{ inputs.environment }}" == "production" ]]; then
+            if [[ -z "${{ inputs.git_ref_to_deploy }}" ]]; then
+              echo "Error: git_ref_to_deploy is required for preview and production environments."
+              exit 1
+            fi
           fi
-          echo "Using snapshot ARN: $SNAPSHOT_ARN"
-          echo "SNAPSHOT_ARN=$SNAPSHOT_ARN" >> $GITHUB_OUTPUT
-      - name: Install terraform
-        uses: hashicorp/setup-terraform@v3
-        with:
-          terraform_version: 1.11.4
-    outputs:
-      SNAPSHOT_ARN: ${{ steps.get-latest-snapshot.outputs.SNAPSHOT_ARN }}
-
-  prepare-webapp:
-    name: Prepare webapp
+  determine-git-sha:
     runs-on: ubuntu-latest
-    permissions:
-      id-token: write
+    permissions: { }
+    needs: validate-inputs
+    outputs:
+      git-sha: ${{ steps.get-git-sha.outputs.git-sha }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
         with:
-          role-to-assume: ${{ env.aws_role }}
-          aws-region: eu-west-2
-      - name: ECR login
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
-      - name: Get docker image digest
-        id: get-docker-image-digest
-        run: |
-          set -e
-          DOCKER_IMAGE="${{ steps.login-ecr.outputs.registry }}/mavis/webapp:${{ inputs.image_tag || github.sha }}"
-          docker pull "$DOCKER_IMAGE"
-          DOCKER_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' "$DOCKER_IMAGE")
-          DIGEST="${DOCKER_DIGEST#*@}"
-          echo "DIGEST=$DIGEST" >> $GITHUB_OUTPUT
-    outputs:
-      DOCKER_DIGEST: ${{ steps.get-docker-image-digest.outputs.DIGEST }}
-
-  plan:
-    name: Terraform plan
-    runs-on: ubuntu-latest
-    needs:
-      - prepare-db-replica
-      - prepare-webapp
-    if: ${{ !cancelled() &&
-          (needs.prepare-db-replica.result == 'success' || needs.prepare-db-replica.result == 'skipped') &&
-          needs.prepare-webapp.result == 'success' }}
-    env:
-      SNAPSHOT_ARN: ${{ needs.prepare-db-replica.outputs.SNAPSHOT_ARN }}
-      DB_SECRET_ARN: ${{ needs.prepare-db-replica.outputs.DB_SECRET_ARN || 'arn:aws:secretsmanager:eu-west-2:000000000000:secret:placeholder' }}
-      DOCKER_DIGEST: ${{ needs.prepare-webapp.outputs.DOCKER_DIGEST }}
-      REPLACE_DB_CLUSTER: ${{ inputs.deployment_type == 'Deployment with DB recreation' }}
+          ref: ${{ inputs.git_ref_to_deploy || github.sha }}
+      - name: Get git sha
+        id: get-git-sha
+        run: echo "git-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+  build-and-push-image:
     permissions:
       id-token: write
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v5
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: ${{ env.aws_role }}
-          aws-region: eu-west-2
-      - name: Install terraform
-        uses: hashicorp/setup-terraform@v3
-        with:
-          terraform_version: 1.11.4
-      - name: Get db secret arn
-        id: get-db-secret-arn
-        working-directory: terraform/app
-        run: |
-          terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
-          DB_SECRET_ARN=$(terraform output --raw db_secret_arn)
-          echo "DB_SECRET_ARN=$DB_SECRET_ARN" >> $GITHUB_OUTPUT
-      - name: Terraform Plan
-        id: plan
-        run: |
-          set -eo pipefail
-          terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
-
-          CIDR_BLOCKS='${{ inputs.egress_cidr }}'
-          PLAN_ARGS=(
-            "plan"
-            "-var=image_digest=${{ env.DOCKER_DIGEST }}"
-            "-var=db_secret_arn=${{ steps.get-db-secret-arn.outputs.DB_SECRET_ARN }}"
-            "-var=imported_snapshot=${{ env.SNAPSHOT_ARN }}"
-            "-var-file=env/${{ inputs.environment }}.tfvars"
-            "-var=allowed_egress_cidr_blocks=$CIDR_BLOCKS"
-            "-out=${{ runner.temp }}/tfplan"
-          )
-          
-          if [ "${{ env.REPLACE_DB_CLUSTER }}" = "true" ]; then
-            PLAN_ARGS+=("-replace" "aws_rds_cluster.cluster")
-          fi
-          terraform "${PLAN_ARGS[@]}" | tee ${{ runner.temp }}/tf_stdout
-      - name: Upload artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: tfplan_infrastructure-${{ inputs.environment }}
-          path: ${{ runner.temp }}/tfplan
-
-  apply:
-    name: Terraform apply
-    runs-on: ubuntu-latest
-    needs: plan
-    if: ${{ !cancelled() && needs.plan.result == 'success' }}
-    environment: ${{ inputs.environment }}
+    needs: determine-git-sha
+    uses: ./.github/workflows/build-and-push-image.yml
+    with:
+      git-sha: ${{ needs.determine-git-sha.outputs.git-sha }}
+  deploy-application:
     permissions:
       id-token: write
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v5
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: ${{ env.aws_role }}
-          aws-region: eu-west-2
-      - name: Download artifact
-        uses: actions/download-artifact@v5
-        with:
-          name: tfplan_infrastructure-${{ inputs.environment }}
-          path: ${{ runner.temp }}
-      - name: Install terraform
-        uses: hashicorp/setup-terraform@v3
-        with:
-          terraform_version: 1.11.4
-      - name: Apply the changes
-        run: |
-          set -e
-          terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
-          terraform apply ${{ runner.temp }}/tfplan
-      - name: Deploy db-access-service
-        run: |
-          task_definition_arn=$(terraform output -raw task_definition_arn)
-          aws ecs update-service \
-            --cluster mavis-${{ inputs.environment }}-data-replication \
-            --service mavis-${{ inputs.environment }}-data-replication \
-            --task-definition $task_definition_arn
+    needs: determine-git-sha
+    uses: ./.github/workflows/deploy-application.yml
+    with:
+      environment: ${{ inputs.environment }}
+      server_types: data-replication
+      git_sha_to_deploy: ${{ needs.determine-git-sha.outputs.git-sha }}
+      app_version: ${{ inputs.git_ref_to_deploy }}

.github/workflows/deploy-application.yml

@@ -69,7 +69,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        service: [ web, good-job, sidekiq ]
+        service: ${{ inputs.server_types == 'all' && fromJSON('["web", "good-job", "sidekiq"]') || fromJSON(format('["{0}"]', inputs.server_types)) }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
@@ -165,7 +165,7 @@ jobs:
         id: deploy-web-service
         uses: aws-actions/amazon-ecs-deploy-task-definition@v2
         with:
-          task-definition: ${{ runner.temp }}
+          task-definition: ${{ runner.temp }}/web-task-definition.json
           codedeploy-appspec: config/templates/appspec.yaml
           cluster: ${{ env.cluster_name }}
           service: mavis-${{ inputs.environment }}-web
@@ -197,13 +197,13 @@ jobs:
           name: ${{ inputs.environment }}-good-job-task-definition
       - name: Change family of task definition
         run: |
-          file_path="${{ runner.temp }}/web-task-definition.json"
+          file_path="${{ runner.temp }}/good-job-task-definition.json"
           family_name="mavis-good-job-task-definition-${{ inputs.environment }}"
           echo "$(jq --arg f "$family_name" '.family = $f' "$file_path")" > "$file_path"
       - name: Deploy good-job service
         uses: aws-actions/amazon-ecs-deploy-task-definition@v2
         with:
-          task-definition: ${{ runner.temp }}/web-task-definition.json
+          task-definition: ${{ runner.temp }}/good-job-task-definition.json
           cluster: ${{ env.cluster_name }}
           service: mavis-${{ inputs.environment }}-good-job
           force-new-deployment: true

.github/workflows/deploy-infrastructure.yml

@@ -8,12 +8,19 @@ on:
         description: Deployment environment
         required: true
         type: string
-      image_tag:
-        required: false
-        type: string
       git_ref_to_deploy:
         required: true
         type: string
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Deployment environment
+        required: true
+        type: string
+      git_ref_to_deploy:
+        description: The git commit SHA to deploy.
+        required: false
+        type: string
 
 permissions: {}
 

.github/workflows/deploy.yml

@@ -109,7 +109,7 @@ jobs:
   update-permissions:
     runs-on: ubuntu-latest
     needs: validate-permissions
-    if: always() && (inputs.environment == 'production' || inputs.environment == 'preview') && needs.validate-permissions.result == 'failure'
+    if: ${{ !cancelled() && (inputs.environment == 'production' || inputs.environment == 'preview') && needs.validate-permissions.result == 'failure' }}
     environment: ${{ inputs.environment }}
     defaults:
       run:
@@ -138,19 +138,18 @@ jobs:
         validate-permissions,
         update-permissions,
       ]
-    if: always() &&
+    if: ${{ !cancelled() &&
       ((inputs.environment != 'production' && inputs.environment != 'preview') ||
-      needs.validate-permissions.result == 'success' || needs.update-permissions.result == 'success')
+      needs.validate-permissions.result == 'success' || needs.update-permissions.result == 'success') }}
     uses: ./.github/workflows/deploy-infrastructure.yml
     with:
       environment: ${{ inputs.environment }}
-      image_tag: ${{ needs.determine-git-sha.outputs.git-sha }}
       git_ref_to_deploy: ${{ inputs.git_ref_to_deploy || github.ref_name }}
   deploy-application:
     permissions:
       id-token: write
     needs: [deploy-infrastructure, determine-git-sha]
-    if: always() && inputs.server_types != 'none' && needs.deploy-infrastructure.result == 'success'
+    if: ${{ !cancelled() && inputs.server_types != 'none' && needs.deploy-infrastructure.result == 'success' }}
     uses: ./.github/workflows/deploy-application.yml
     with:
       environment: ${{ inputs.environment }}