Update deployment pipeline to work with modularized services

.github/workflows/continuous-deployment.yml

@@ -17,3 +17,4 @@ jobs:
     uses: ./.github/workflows/deploy.yml
     with:
       environment: ${{ matrix.environment }}
+      server_types: all

.github/workflows/deploy-application.yml

@@ -20,6 +20,15 @@ on:
         description: Docker image tag
         required: false
         type: string
+      server_types:
+        description: Server types to deploy
+        required: true
+        type: choice
+        options:
+          - all
+          - web
+          - good-job
+        default: all
   workflow_call:
     inputs:
       environment:
@@ -28,6 +37,9 @@ on:
       image_tag:
         required: false
         type: string
+      server_types:
+        required: true
+        type: string
 
 concurrency:
   group: deploy-application-${{ inputs.environment }}
@@ -37,6 +49,11 @@ env:
     && 'arn:aws:iam::820242920762:role/GithubDeployMavisAndInfrastructure'
     || 'arn:aws:iam::393416225559:role/GithubDeployMavisAndInfrastructure' }}
   terraform-working-directory: terraform/app
+  terraform-target-options: ${{ inputs.server_types == 'web'
+    && '-target=aws_ecs_task_definition.task_definition -target=aws_s3_object.appspec_object'
+    || inputs.server_types == 'good-job'
+    && '-target=module.good-job_service.aws_ecs_task_definition.this'
+    || '-target=aws_ecs_task_definition.task_definition -target=aws_s3_object.appspec_object -target=module.good-job_service.aws_ecs_task_definition.this' }}
 
 jobs:
   plan-changes:
@@ -74,8 +91,7 @@ jobs:
         working-directory: ${{ env.terraform-working-directory }}
         run: |
           terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
-          terraform plan -target=aws_ecs_task_definition.task_definition \
-          -target=aws_s3_object.appspec_object -var-file="env/${{ inputs.environment }}.tfvars" \
+          terraform plan ${{ env.terraform-target-options }} -var-file="env/${{ inputs.environment }}.tfvars" \
           -var="image_digest=$DIGEST" -out=${{ runner.temp }}/tfplan | tee ${{ runner.temp }}/tf_stdout
       - name: Validate the changes
         run: |
@@ -115,28 +131,30 @@ jobs:
         run: |
           terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
           terraform apply ${{ runner.temp }}/tfplan
-          echo "s3_bucket=$(terraform output -raw s3_bucket)" >> ${{ runner.temp }}/CODEDEPLOY_ENV
-          echo "s3_key=$(terraform output -raw s3_key)" >> ${{ runner.temp }}/CODEDEPLOY_ENV
-          echo "application=$(terraform output -raw codedeploy_application_name)" >> ${{ runner.temp }}/CODEDEPLOY_ENV
-          echo "application_group=$(terraform output -raw codedeploy_deployment_group_name)" >> ${{ runner.temp }}/CODEDEPLOY_ENV
+          echo "s3_bucket=$(terraform output -raw s3_bucket)" >> ${{ runner.temp }}/DEPLOYMENT_ENVS
+          echo "s3_key=$(terraform output -raw s3_key)" >> ${{ runner.temp }}/DEPLOYMENT_ENVS
+          echo "application=$(terraform output -raw codedeploy_application_name)" >> ${{ runner.temp }}/DEPLOYMENT_ENVS
+          echo "application_group=$(terraform output -raw codedeploy_deployment_group_name)" >> ${{ runner.temp }}/DEPLOYMENT_ENVS
+          echo "ecs_variables=$(terraform output -json ecs_variables | sed 's/\"/\\"/g')" >> ${{ runner.temp }}/DEPLOYMENT_ENVS
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
-          name: CODEDEPLOY_ENV-${{ inputs.environment }}
-          path: ${{ runner.temp }}/CODEDEPLOY_ENV
+          name: DEPLOYMENT_ENVS-${{ inputs.environment }}
+          path: ${{ runner.temp }}/DEPLOYMENT_ENVS
 
-  create-deployment:
-    name: Create deployment
+  create-web-deployment:
+    name: Create web deployment
     runs-on: ubuntu-latest
     needs: apply-changes
+    if: inputs.server_types == 'web' || inputs.server_types == 'all'
     environment: ${{ inputs.environment }}
     permissions:
       id-token: write
     steps:
       - name: Download artifact
         uses: actions/download-artifact@v4
         with:
-          name: CODEDEPLOY_ENV-${{ inputs.environment }}
+          name: DEPLOYMENT_ENVS-${{ inputs.environment }}
           path: ${{ runner.temp }}/artifact
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v4
@@ -147,7 +165,7 @@ jobs:
         run: sudo snap install --classic aws-cli
       - name: Trigger CodeDeploy deployment
         run: |
-          source ${{ runner.temp }}/artifact/CODEDEPLOY_ENV
+          source ${{ runner.temp }}/artifact/DEPLOYMENT_ENVS
           deployment_id=$(aws deploy create-deployment \
           --application-name "$application" --deployment-group-name "$application_group" \
           --s3-location bucket="$s3_bucket",key="$s3_key",bundleType=yaml | jq -r .deploymentId)
@@ -157,3 +175,54 @@ jobs:
         run: |
           aws deploy wait deployment-successful --deployment-id $deployment_id
           echo "Deployment successful"
+
+  create-good-job-deployment:
+    name: Create good-job deployment
+    runs-on: ubuntu-latest
+    needs: apply-changes
+    if: inputs.server_types == 'good-job'  || inputs.server_types == 'all'
+    environment: ${{ inputs.environment }}
+    permissions:
+      id-token: write
+    steps:
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: DEPLOYMENT_ENVS-${{ inputs.environment }}
+          path: ${{ runner.temp }}/artifact
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ env.aws-role }}
+          aws-region: eu-west-2
+      - name: Install AWS CLI
+        run: sudo snap install --classic aws-cli
+      - name: Trigger ECR Deployment
+        run: |
+          source ${{ runner.temp }}/artifact/DEPLOYMENT_ENVS
+          echo "$ecs_variables"
+          cluster_name=$(echo "$ecs_variables" | jq -r '.cluster_name')
+          service_name=$(echo "$ecs_variables" | jq -r '.good_job.service_name')
+          task_definition=$(echo "$ecs_variables" | jq -r '.good_job.task_definition')
+          DEPLOYMENT_ID=$(aws ecs update-service --cluster $cluster_name --service $service_name \
+          --task-definition $task_definition --force-new-deployment \
+          --query 'service.deployments[?rolloutState==`IN_PROGRESS`].[id][0]' --output text)
+          echo "Deployment started: $DEPLOYMENT_ID"
+          echo "deployment_id=$DEPLOYMENT_ID" >> $GITHUB_ENV
+      - name: Wait for deployment to complete
+        run: |
+          source ${{ runner.temp }}/artifact/DEPLOYMENT_ENVS
+          cluster_name=$(echo "$ecs_variables" | jq -r '.cluster_name')
+          service_name=$(echo "$ecs_variables" | jq -r '.good_job.service_name')
+          DEPLOYMENT_STATE=IN_PROGRESS
+          while [ "$DEPLOYMENT_STATE" == "IN_PROGRESS" ]; do
+            echo "Waiting for deployment to complete..."
+            sleep 30
+            DEPLOYMENT_STATE="$(aws ecs describe-services --cluster $cluster_name --services $service_name \
+            --query "services[0].deployments[?id == \`$deployment_id\`].[rolloutState][0]" --output text)"
+          done
+          if [ "$DEPLOYMENT_STATE" != "COMPLETED" ]; then
+            echo "Deployment failed with state: $DEPLOYMENT_STATE"
+            exit 1
+          fi
+          echo "Deployment successful"

.github/workflows/deploy-mavis.yml

@@ -0,0 +1,54 @@
+name: Deploy Mavis on Environment
+run-name: Deploying Mavis on ${{ inputs.environment }}
+
+concurrency:
+  group: deploy-mavis-${{ inputs.environment }}
+
+on:
+  workflow_call:
+    inputs:
+      environment:
+        required: true
+        type: string
+      server_types:
+        required: true
+        type: string
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Deployment environment
+        required: true
+        type: choice
+        options:
+          - qa
+          - poc
+          - test
+          - preview
+          - training
+          - production
+          - copilotmigration
+      server_types:
+        description: Server types to deploy
+        required: true
+        type: choice
+        options:
+          - all
+          - web
+          - good-job
+        default: all
+
+jobs:
+  build-and-push-image:
+    uses: ./.github/workflows/build-and-push-image.yml
+  deploy-infrastructure:
+    needs: build-and-push-image
+    uses: ./.github/workflows/deploy-infrastructure.yml
+    with:
+      environment: ${{ inputs.environment }}
+  deploy-application:
+    needs: deploy-infrastructure
+    uses: ./.github/workflows/deploy-application.yml
+    with:
+      environment: ${{ inputs.environment }}
+      image_tag: ${{ github.sha }}
+      server_types: ${{ inputs.server_types }}

terraform/app/codedeploy.tf

@@ -117,7 +117,7 @@ resource "aws_s3_object" "appspec_object" {
   acl    = "private"
   content = templatefile("templates/appspec.yaml.tpl", {
     task_definition_arn = module.web_service.task_definition.arn
-    container_name      = module.web_service.task_definition.arn
+    container_name      = module.web_service.task_definition.container_name
     container_port      = aws_lb_target_group.blue.port
   })
 

terraform/scripts/check_task_definition.sh

@@ -5,36 +5,54 @@ if [ "$#" -ne 1 ]; then
     exit 1
 fi
 
+valid_resources=(
+  "aws_ecs_task_definition\.task_definition" #TODO: Remove after release
+  "aws_s3_object\.appspec_object"
+  "module\.web_service\.aws_ecs_task_definition"
+  "module\.good_job_service\.aws_ecs_task_definition"
+)
+
 tf_stdout=$1
-# Check task definition is replaced
 if [[ $(grep -ce "No changes.*Your infrastructure matches the configuration" "$tf_stdout") -eq 1 ]]; then
   echo "No changes detected, continuing."
   exit 0
 fi
-if [[ $(grep -cE "aws_ecs_task_definition\.task_definition.*(replaced|created)" "$tf_stdout") -eq 1 ]]; then
-  echo "Task definition is being replaced or created"
-else
-  echo "Task definition is not being replaced, aborting."
-  exit 1
-fi
-if [[ $(grep -cE "aws_s3_object\.appspec_object.*(updated in-place|created)" "$tf_stdout") -eq 1 ]]; then
-  echo "S3 bucket object is being replaced or created"
-else
-  echo "S3 bucket object is not being replaced, aborting."
-  exit 1
-fi
+
 MODIFICATIONS=$(grep -E "[0-9]+ to add, [0-9]+ to change, [0-9]+ to destroy." "$tf_stdout") || exit 1
 ADDITIONS=$(echo "$MODIFICATIONS" | sed -E 's/.*([0-9]+) to add.*/\1/')  || exit 1
-CHANGES=$(echo "$MODIFICATIONS" | sed -E 's/.*([0-9]+) to change.*/\1/')  || exit 1
 DELETIONS=$(echo "$MODIFICATIONS" | sed -E 's/.*([0-9]+) to destroy.*/\1/') || exit 1
 if [[ $DELETIONS -gt $ADDITIONS ]]; then
-  echo "More resources are being destroyed than created."
-  echo "Other resources than task definition and s3 bucket object are being deleted, aborting."
+  echo "ERROR: More resources are being destroyed than created, run infrastructure deploy first."
   exit 1
+else
+  echo "CHECK_PASSED: No resources are being destroyed without replacement."
 fi
-if [[ $((CHANGES + ADDITIONS)) -gt 2 ]]; then
-  echo "More than 2 resources are being changed."
-  echo "Other changes than task definition and s3 bucket object are being made, aborting."
+
+mapfile -t PLANNED_CHANGES < <(grep -E "#.+(replaced|created|updated in-place|destroyed)" "$tf_stdout" || exit 1)
+
+invalid_modifications=()
+for change in "${PLANNED_CHANGES[@]}"; do
+  valid=0
+  for resource in "${valid_resources[@]}"; do
+    if [[ "$change" =~ $resource ]]; then
+      valid=1
+      break
+    fi
+  done
+  if [ $valid -eq 0 ]; then
+    invalid_modifications+=("$change")
+  fi
+done
+
+if [ ! ${#invalid_modifications[@]} -eq 0 ]; then
+  echo "FAILED_CHECK: Invalid resources modified"
+  for item in "${invalid_modifications[@]}"; do
+    echo "  $item"
+  done
+  echo "Please run an infrastructure deployment."
   exit 1
+else
+  echo "CHECK_PASSED: All modified resources are expected."
 fi
-echo "Basic checks passed, only task definition and S3 bucket changes observed."
+
+echo "Basic checks passed, if production please evaluate the plan before applying."

terraform/scripts/validate_plan.sh

@@ -23,8 +23,10 @@ down_time_if_destroyed=(
 "aws_security_group\.rds_security_group"
 "aws_db_subnet_group\.aurora_subnet_group"
 "aws_ecs_cluster\.cluster"
-"aws_ecs_service\.service"
-"aws_security_group\.ecs_service_sg"
+"aws_ecs_service\.service" #TODO: Remove after release
+"aws_security_group\.ecs_service_sg" #TODO: Remove after release
+"module\.[^.]+\.aws_ecs_service\.this"
+"module\.[^.]+\.aws_security_group"
 )
 
 if [ "$#" -ne 1 ]; then
@@ -36,7 +38,9 @@ tfstdout=$1
 
 for resource in "${down_time_if_destroyed[@]}"; do
   if [[ $(grep -cE "$resource.*(replaced|destroyed)" "$tfstdout") -ne 0 ]]; then
-    echo "$resource is being destroyed. This would cause a downtime. Aborting"
+    echo "A resource is being destroyed:"
+    grep -E "$resource.*(replaced|destroyed)" "$tfstdout"
+    echo "This would cause a downtime. Aborting"
     exit 1
   fi
 done