Separate app and infra deployments

.github/workflows/data-replication-pipeline.yml

@@ -1,5 +1,5 @@
-name: Data replication pipeline
-run-name: ${{ inputs.deployment_type }} for data replication resources for ${{ inputs.environment }}
+name: Deploy data replication
+run-name: Deploy data replication to ${{ inputs.environment }}
 
 on:
   workflow_dispatch:
@@ -15,227 +15,152 @@ on:
           - qa
           - sandbox-alpha
           - sandbox-beta
-      deployment_type:
-        description: Deployment type
-        required: true
-        type: choice
-        options:
-          - Deployment with DB recreation
-          - Application only deployment
-      image_tag:
-        description: Docker image tag to deploy
+      image-tag:
+        description: Docker image tag to deploy (if not provided, will build from git-ref-to-deploy)
         required: false
         type: string
-      db_snapshot_arn:
-        description: ARN of the DB snapshot to use (optional)
+      git-ref-to-deploy:
+        description: Git ref (branch/tag/SHA) to deploy
         required: false
         type: string
-      egress_cidr:
-        description: CIDR blocks to allow egress traffic.
-        type: string
-        required: true
-        default: "[]"
-      take_db_snapshot:
-        description: Take a new DB snapshot before creating the environment
-        type: boolean
-        default: false
+
+permissions: {}
 
 env:
   aws_role: ${{ inputs.environment == 'production'
     && 'arn:aws:iam::820242920762:role/GithubDeployDataReplicationInfrastructure'
     || 'arn:aws:iam::393416225559:role/GithubDeployDataReplicationInfrastructure' }}
-  db_snapshot_role: ${{ inputs.environment == 'production'
-    && 'arn:aws:iam::820242920762:role/DatabaseSnapshotRole'
-    || 'arn:aws:iam::393416225559:role/DatabaseSnapshotRole' }}
-
-defaults:
-  run:
-    working-directory: terraform/data_replication
+  aws_account_id: ${{ inputs.environment == 'production' && '820242920762' || '393416225559' }}
 
 concurrency:
   group: deploy-data-replica-${{ inputs.environment }}
 
 jobs:
-  prepare-db-replica:
-    if: ${{ inputs.deployment_type == 'Deployment with DB recreation' }}
-    name: Prepare data replica
+  validate-inputs:
     runs-on: ubuntu-latest
-    permissions:
-      id-token: write
+    permissions: {}
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v5
-      - name: Assume DB Snapshot role
-        if: inputs.take_db_snapshot
-        uses: aws-actions/configure-aws-credentials@v5
-        with:
-          role-to-assume: ${{ env.db_snapshot_role }}
-          aws-region: eu-west-2
-      - name: Take DB snapshot
-        if: inputs.take_db_snapshot
-        run: |
-          set -e
-          snapshot_identifier=snapshot-for-data-replication-$(date +"%Y-%m-%d-%H-%M-%S")
-          aws rds create-db-cluster-snapshot --db-cluster-identifier mavis-${{ inputs.environment }} --db-cluster-snapshot-identifier $snapshot_identifier
-          echo "Waiting for snapshot to be available. This can take a while."
-          aws rds wait db-cluster-snapshot-available --db-cluster-snapshot-identifier $snapshot_identifier
-          echo "New snapshot is now available"
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v5
-        with:
-          role-to-assume: ${{ env.aws_role }}
-          aws-region: eu-west-2
-      - name: Get latest snapshot
-        id: get-latest-snapshot
+      - name: Validate inputs
         run: |
-          set -e
-          if [ -z "${{ inputs.db_snapshot_arn }}" ]; then
-              echo "No snapshot ARN provided, fetching the latest snapshot"
-              SNAPSHOT_ARN=$(aws rds describe-db-cluster-snapshots \
-              --query "DBClusterSnapshots[?DBClusterIdentifier=='mavis-${{ inputs.environment }}'].[DBClusterSnapshotArn, SnapshotCreateTime]" \
-              --output text | sort -k2 -r | head -n 1 | cut -f1)
-
-              if [ -z "$SNAPSHOT_ARN" ]; then
-                  echo "No snapshots found for mavis-${{ inputs.environment }}"
-                  exit 1
-              fi
-          else
-              echo "Using provided snapshot ARN: ${{ inputs.db_snapshot_arn }}"
-              SNAPSHOT_ARN="${{ inputs.db_snapshot_arn }}"
+          if [[ "${{ inputs.environment }}" == "preview" || "${{ inputs.environment }}" == "production" ]]; then
+            if [[ -z "${{ inputs.git-ref-to-deploy }}" && -z "${{ inputs.image-tag }}" ]]; then
+            echo "Error: Either git-ref-to-deploy or image-tag is required for preview and production environment."
+              exit 1
+            fi
           fi
-          echo "Using snapshot ARN: $SNAPSHOT_ARN"
-          echo "SNAPSHOT_ARN=$SNAPSHOT_ARN" >> $GITHUB_OUTPUT
-      - name: Install terraform
-        uses: hashicorp/setup-terraform@v3
-        with:
-          terraform_version: 1.11.4
-    outputs:
-      SNAPSHOT_ARN: ${{ steps.get-latest-snapshot.outputs.SNAPSHOT_ARN }}
 
-  prepare-webapp:
-    name: Prepare webapp
+  determine-git-sha:
     runs-on: ubuntu-latest
-    permissions:
-      id-token: write
+    permissions: {}
+    needs: validate-inputs
+    outputs:
+      git-sha: ${{ steps.get-git-sha.outputs.git-sha }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v5
         with:
-          role-to-assume: ${{ env.aws_role }}
-          aws-region: eu-west-2
-      - name: ECR login
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
-      - name: Get docker image digest
-        id: get-docker-image-digest
-        run: |
-          set -e
-          DOCKER_IMAGE="${{ steps.login-ecr.outputs.registry }}/mavis/webapp:${{ inputs.image_tag || github.sha }}"
-          docker pull "$DOCKER_IMAGE"
-          DOCKER_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' "$DOCKER_IMAGE")
-          DIGEST="${DOCKER_DIGEST#*@}"
-          echo "DIGEST=$DIGEST" >> $GITHUB_OUTPUT
-    outputs:
-      DOCKER_DIGEST: ${{ steps.get-docker-image-digest.outputs.DIGEST }}
+          ref: ${{ inputs.git-ref-to-deploy || github.sha }}
+      - name: Get git sha
+        id: get-git-sha
+        run: echo "git-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+
+  build-and-push-image:
+    if: ${{ !inputs.image-tag }}
+    permissions:
+      id-token: write
+    needs: determine-git-sha
+    uses: ./.github/workflows/build-and-push-image.yml
+    with:
+      git-sha: ${{ needs.determine-git-sha.outputs.git-sha }}
 
-  plan:
-    name: Terraform plan
+  prepare-deployment:
+    name: Prepare deployment
     runs-on: ubuntu-latest
-    needs:
-      - prepare-db-replica
-      - prepare-webapp
-    if: ${{ !cancelled() &&
-          (needs.prepare-db-replica.result == 'success' || needs.prepare-db-replica.result == 'skipped') &&
-          needs.prepare-webapp.result == 'success' }}
-    env:
-      SNAPSHOT_ARN: ${{ needs.prepare-db-replica.outputs.SNAPSHOT_ARN }}
-      DB_SECRET_ARN: ${{ needs.prepare-db-replica.outputs.DB_SECRET_ARN || 'arn:aws:secretsmanager:eu-west-2:000000000000:secret:placeholder' }}
-      DOCKER_DIGEST: ${{ needs.prepare-webapp.outputs.DOCKER_DIGEST }}
-      REPLACE_DB_CLUSTER: ${{ inputs.deployment_type == 'Deployment with DB recreation' }}
+    needs: [determine-git-sha, build-and-push-image]
+    if: ${{ always() && (needs.build-and-push-image.result == 'success' || needs.build-and-push-image.result == 'skipped') }}
     permissions:
       id-token: write
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
+        with:
+          ref: ${{ inputs.git-ref-to-deploy || github.sha }}
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v5
         with:
           role-to-assume: ${{ env.aws_role }}
           aws-region: eu-west-2
-      - name: Install terraform
-        uses: hashicorp/setup-terraform@v3
-        with:
-          terraform_version: 1.11.4
-      - name: Get db secret arn
-        id: get-db-secret-arn
-        working-directory: terraform/app
+      - name: Get image digest
+        id: get-image-digest
         run: |
-          terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
-          DB_SECRET_ARN=$(terraform output --raw db_secret_arn)
-          echo "DB_SECRET_ARN=$DB_SECRET_ARN" >> $GITHUB_OUTPUT
-      - name: Terraform Plan
-        id: plan
+          digest=$(aws ecr describe-images \
+            --repository-name mavis/webapp \
+            --image-ids imageTag=${{ inputs.image-tag || needs.determine-git-sha.outputs.git-sha }} \
+            --query 'imageDetails[0].imageDigest' \
+            --output text)
+          echo "digest=$digest" >> $GITHUB_OUTPUT
+      - name: Parse environment variables
+        id: parse-environment-variables
         run: |
-          set -eo pipefail
-          terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
-
-          CIDR_BLOCKS='${{ inputs.egress_cidr }}'
-          PLAN_ARGS=(
-            "plan"
-            "-var=image_digest=${{ env.DOCKER_DIGEST }}"
-            "-var=db_secret_arn=${{ steps.get-db-secret-arn.outputs.DB_SECRET_ARN }}"
-            "-var=imported_snapshot=${{ env.SNAPSHOT_ARN }}"
-            "-var-file=env/${{ inputs.environment }}.tfvars"
-            "-var=allowed_egress_cidr_blocks=$CIDR_BLOCKS"
-            "-out=${{ runner.temp }}/tfplan"
-          )
-
-          if [ "${{ env.REPLACE_DB_CLUSTER }}" = "true" ]; then
-            PLAN_ARGS+=("-replace" "aws_rds_cluster.cluster")
-          fi
-          terraform "${PLAN_ARGS[@]}" | tee ${{ runner.temp }}/tf_stdout
-      - name: Upload artifact
+          {
+            echo 'parsed_env_vars<<EOF'
+            echo "MAVIS__SPLUNK__ENABLED=false"
+            echo "MAVIS__CIS2__ENABLED=false"
+            echo "MAVIS__PDS__ENQUEUE_BULK_UPDATES=false"
+            echo "MAVIS__PDS__RATE_LIMIT_PER_SECOND=${{ inputs.environment == 'production' && 50 || 5 }}"
+            echo 'EOF'
+          } >> "$GITHUB_OUTPUT"
+      - name: Populate web task definition
+        id: create-task-definition
+        uses: aws-actions/amazon-ecs-render-task-definition@v1
+        with:
+          task-definition-family: "mavis-data-replication-task-definition-${{ inputs.environment }}-template"
+          container-name: "application"
+          image: "${{ env.aws_account_id }}.dkr.ecr.eu-west-2.amazonaws.com/mavis/webapp@${{ steps.get-image-digest.outputs.digest }}"
+          environment-variables: ${{ steps.parse-environment-variables.outputs.parsed_env_vars }}
+      - name: Rename task definition file
+        run: mv ${{ steps.create-task-definition.outputs.task-definition }} ${{ runner.temp }}/data-replication-task-definition.json
+      - name: Upload artifact for data-replication task definition
         uses: actions/upload-artifact@v4
         with:
-          name: tfplan_infrastructure-${{ inputs.environment }}
-          path: ${{ runner.temp }}/tfplan
+          name: ${{ inputs.environment }}-data-replication-task-definition
+          path: ${{ runner.temp }}/data-replication-task-definition.json
 
-  apply:
-    name: Terraform apply
+  approve-deployments:
+    name: Wait for approval if required
     runs-on: ubuntu-latest
-    needs: plan
-    if: ${{ !cancelled() && needs.plan.result == 'success' }}
+    needs: prepare-deployment
     environment: ${{ inputs.environment }}
+    steps:
+      - run: echo "Proceeding with deployment to ${{ inputs.environment }} environment"
+
+  deploy-data-replication:
+    name: Deploy data-replication service
+    runs-on: ubuntu-latest
+    needs: [prepare-deployment, approve-deployments]
     permissions:
       id-token: write
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v5
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v5
         with:
           role-to-assume: ${{ env.aws_role }}
           aws-region: eu-west-2
-      - name: Download artifact
+      - name: Download data-replication task definition artifact
         uses: actions/download-artifact@v5
         with:
-          name: tfplan_infrastructure-${{ inputs.environment }}
           path: ${{ runner.temp }}
-      - name: Install terraform
-        uses: hashicorp/setup-terraform@v3
-        with:
-          terraform_version: 1.11.4
-      - name: Apply the changes
+          name: ${{ inputs.environment }}-data-replication-task-definition
+      - name: Change family of task definition
         run: |
-          set -e
-          terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
-          terraform apply ${{ runner.temp }}/tfplan
-      - name: Deploy db-access-service
-        run: |
-          task_definition_arn=$(terraform output -raw task_definition_arn)
-          aws ecs update-service \
-            --cluster mavis-${{ inputs.environment }}-data-replication \
-            --service mavis-${{ inputs.environment }}-data-replication \
-            --task-definition $task_definition_arn
+          file_path="${{ runner.temp }}/data-replication-task-definition.json"
+          family_name="mavis-data-replication-task-definition-${{ inputs.environment }}"
+          echo "$(jq --arg f "$family_name" '.family = $f' "$file_path")" > "$file_path"
+      - name: Deploy data-replication service
+        uses: aws-actions/amazon-ecs-deploy-task-definition@v2
+        with:
+          task-definition: ${{ runner.temp }}/data-replication-task-definition.json
+          cluster: mavis-${{ inputs.environment }}-data-replication
+          service: mavis-${{ inputs.environment }}-data-replication
+          force-new-deployment: true
+          wait-for-service-stability: true