From fa023df12366d565edc5777ebc03a47dee3e1ff8 Mon Sep 17 00:00:00 2001 From: Brage Gording Date: Tue, 15 Apr 2025 13:48:49 +0100 Subject: [PATCH 1/2] Streamline github workflows for deployment - Running terraform apply introduces unnecessary complexity - Change infrastructure deploy to deploy with most up to date image - Unless specified otherwise - Deploy pipeline now can only initiate deploys, not modify any resources like task definitions --- .github/workflows/deploy-application.yml | 123 +++++--------------- .github/workflows/deploy-infrastructure.yml | 39 ++++--- .github/workflows/deploy.yml | 1 - terraform/scripts/check_task_definition.sh | 58 --------- 4 files changed, 53 insertions(+), 168 deletions(-) delete mode 100755 terraform/scripts/check_task_definition.sh diff --git a/.github/workflows/deploy-application.yml b/.github/workflows/deploy-application.yml index 8f6f79ff3f..89c633fae5 100644 --- a/.github/workflows/deploy-application.yml +++ b/.github/workflows/deploy-application.yml @@ -16,10 +16,6 @@ on: - preview - training - production - image_tag: - description: Docker image tag - required: false - type: string server_types: description: Server types to deploy required: true @@ -34,9 +30,6 @@ on: environment: required: true type: string - image_tag: - required: false - type: string server_types: required: true type: string @@ -48,60 +41,11 @@ env: aws-role: ${{ inputs.environment == 'production' && 'arn:aws:iam::820242920762:role/GithubDeployMavisAndInfrastructure' || 'arn:aws:iam::393416225559:role/GithubDeployMavisAndInfrastructure' }} - terraform-working-directory: terraform/app jobs: - plan-changes: - name: Plan task definition changes - runs-on: ubuntu-latest - permissions: - id-token: write - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.aws-role }} - aws-region: eu-west-2 - - name: Login to ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - name: pull docker image - run: | - DOCKER_IMAGE="${{ steps.login-ecr.outputs.registry }}/mavis/webapp:${{ inputs.image_tag || github.sha }}" - docker pull "$DOCKER_IMAGE" - echo "DOCKER_IMAGE=$DOCKER_IMAGE" >> $GITHUB_ENV - - name: Extract image digest - run: | - DOCKER_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' "$DOCKER_IMAGE") - DIGEST="${DOCKER_DIGEST#*@}" - echo "DIGEST=$DIGEST" >> $GITHUB_ENV - - name: Install terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.10.5 - - name: Update the task definition - id: plan - working-directory: ${{ env.terraform-working-directory }} - run: | - terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade - terraform plan -var-file="env/${{ inputs.environment }}.tfvars" \ - -var="image_digest=$DIGEST" -out=${{ runner.temp }}/tfplan | tee ${{ runner.temp }}/tf_stdout - - name: Validate the changes - run: | - ./terraform/scripts/check_task_definition.sh ${{ runner.temp }}/tf_stdout - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: tfplan_app-${{ inputs.environment }} - path: ${{ runner.temp }}/tfplan - - apply-changes: - name: Apply task definition changes + prepare-deployment: + name: Prepare deployment runs-on: ubuntu-latest - needs: plan-changes - environment: ${{ inputs.environment }} permissions: id-token: write steps: @@ -112,26 +56,25 @@ jobs: with: role-to-assume: ${{ env.aws-role }} aws-region: eu-west-2 - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: tfplan_app-${{ inputs.environment }} - path: ${{ runner.temp }} - name: Install terraform uses: hashicorp/setup-terraform@v3 with: terraform_version: 1.10.5 - - name: Apply the changes - working-directory: ${{ env.terraform-working-directory }} + - name: Get terraform output + id: terraform-output + working-directory: terraform/app run: | - terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade - terraform apply ${{ runner.temp }}/tfplan - echo "s3_bucket=$(terraform output -raw s3_bucket)" >> ${{ runner.temp }}/DEPLOYMENT_ENVS - echo "s3_key=$(terraform output -raw s3_key)" >> ${{ runner.temp }}/DEPLOYMENT_ENVS - echo "application=$(terraform output -raw codedeploy_application_name)" >> ${{ runner.temp }}/DEPLOYMENT_ENVS - echo "application_group=$(terraform output -raw codedeploy_deployment_group_name)" >> ${{ runner.temp }}/DEPLOYMENT_ENVS - echo "ecs_variables=$(terraform output -json ecs_variables | sed 's/\"/\\"/g')" >> ${{ runner.temp }}/DEPLOYMENT_ENVS - - name: Upload artifact + terraform init -backend-config=env/${{ inputs.environment }}-backend.hcl -reconfigure + terraform output -json | jq -r ' + "s3_bucket=" + .s3_bucket.value, + "s3_key=" + .s3_key.value, + "application=" + .codedeploy_application_name.value, + "application_group=" + .codedeploy_deployment_group_name.value, + "cluster_name=" + .ecs_variables.value.cluster_name, + "good_job_service=" + .ecs_variables.value.good_job.service_name, + "good_job_task_definition=" + .ecs_variables.value.good_job.task_definition.arn + ' > ${{ runner.temp }}/DEPLOYMENT_ENVS + - name: Upload Artifact uses: actions/upload-artifact@v4 with: name: DEPLOYMENT_ENVS-${{ inputs.environment }} @@ -140,7 +83,7 @@ jobs: create-web-deployment: name: Create web deployment runs-on: ubuntu-latest - needs: apply-changes + needs: prepare-deployment if: inputs.server_types == 'web' || inputs.server_types == 'all' environment: ${{ inputs.environment }} permissions: @@ -150,17 +93,15 @@ jobs: uses: actions/download-artifact@v4 with: name: DEPLOYMENT_ENVS-${{ inputs.environment }} - path: ${{ runner.temp }}/artifact + path: ${{ runner.temp }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ env.aws-role }} aws-region: eu-west-2 - - name: Install AWS CLI - run: sudo snap install --classic aws-cli - name: Trigger CodeDeploy deployment run: | - source ${{ runner.temp }}/artifact/DEPLOYMENT_ENVS + source ${{ runner.temp }}/DEPLOYMENT_ENVS deployment_id=$(aws deploy create-deployment \ --application-name "$application" --deployment-group-name "$application_group" \ --s3-location bucket="$s3_bucket",key="$s3_key",bundleType=yaml | jq -r .deploymentId) @@ -168,23 +109,23 @@ jobs: echo "deployment_id=$deployment_id" >> $GITHUB_ENV - name: Wait up to 30 minutes for deployment to complete run: | - aws deploy wait deployment-successful --deployment-id $deployment_id + aws deploy wait deployment-successful --deployment-id "$deployment_id" echo "Deployment successful" create-good-job-deployment: name: Create good-job deployment runs-on: ubuntu-latest - needs: apply-changes - if: inputs.server_types == 'good-job' || inputs.server_types == 'all' + needs: prepare-deployment + if: inputs.server_types == 'good-job' || inputs.server_types == 'all' environment: ${{ inputs.environment }} permissions: id-token: write steps: - - name: Download artifact + - name: Download Artifact uses: actions/download-artifact@v4 with: name: DEPLOYMENT_ENVS-${{ inputs.environment }} - path: ${{ runner.temp }}/artifact + path: ${{ runner.temp }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: @@ -194,26 +135,20 @@ jobs: run: sudo snap install --classic aws-cli - name: Trigger ECS Deployment run: | - source ${{ runner.temp }}/artifact/DEPLOYMENT_ENVS - echo "$ecs_variables" - cluster_name=$(echo "$ecs_variables" | jq -r '.cluster_name') - service_name=$(echo "$ecs_variables" | jq -r '.good_job.service_name') - task_definition=$(echo "$ecs_variables" | jq -r '.good_job.task_definition.arn') - DEPLOYMENT_ID=$(aws ecs update-service --cluster $cluster_name --service $service_name \ - --task-definition $task_definition --force-new-deployment \ + source ${{ runner.temp }}/DEPLOYMENT_ENVS + DEPLOYMENT_ID=$(aws ecs update-service --cluster $cluster_name --service $good_job_service \ + --task-definition $good_job_task_definition --force-new-deployment \ --query 'service.deployments[?rolloutState==`IN_PROGRESS`].[id][0]' --output text) echo "Deployment started: $DEPLOYMENT_ID" echo "deployment_id=$DEPLOYMENT_ID" >> $GITHUB_ENV - name: Wait for deployment to complete run: | - source ${{ runner.temp }}/artifact/DEPLOYMENT_ENVS - cluster_name=$(echo "$ecs_variables" | jq -r '.cluster_name') - service_name=$(echo "$ecs_variables" | jq -r '.good_job.service_name') + source ${{ runner.temp }}/DEPLOYMENT_ENVS DEPLOYMENT_STATE=IN_PROGRESS while [ "$DEPLOYMENT_STATE" == "IN_PROGRESS" ]; do echo "Waiting for deployment to complete..." sleep 30 - DEPLOYMENT_STATE="$(aws ecs describe-services --cluster $cluster_name --services $service_name \ + DEPLOYMENT_STATE="$(aws ecs describe-services --cluster $cluster_name --services $good_job_service \ --query "services[0].deployments[?id == \`$deployment_id\`].[rolloutState][0]" --output text)" done if [ "$DEPLOYMENT_STATE" != "COMPLETED" ]; then diff --git a/.github/workflows/deploy-infrastructure.yml b/.github/workflows/deploy-infrastructure.yml index fc1afcf399..b07860cde3 100644 --- a/.github/workflows/deploy-infrastructure.yml +++ b/.github/workflows/deploy-infrastructure.yml @@ -16,8 +16,8 @@ on: - preview - training - production - docker_sha: - description: "Docker image sha to deploy. This is used only if no existing task definition is found" + image_tag: + description: Docker image tag to deploy required: false type: string workflow_call: @@ -26,6 +26,9 @@ on: description: Deployment environment required: true type: string + image_tag: + required: false + type: string concurrency: group: deploy-infrastructure-${{ inputs.environment }} @@ -53,30 +56,36 @@ jobs: with: role-to-assume: ${{ env.aws_role }} aws-region: eu-west-2 + - name: Set image tag + run: | + IMAGE_TAG="${{ inputs.image_tag || github.sha }}" + echo "IMAGE_TAG=$IMAGE_TAG" >> $GITHUB_ENV + - name: Login to ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + - name: Pull Docker image + run: | + DOCKER_IMAGE="${{ steps.login-ecr.outputs.registry }}/mavis/webapp:${IMAGE_TAG}" + docker pull "$DOCKER_IMAGE" + echo "DOCKER_IMAGE=$DOCKER_IMAGE" >> $GITHUB_ENV + - name: Extract image digest + run: | + DOCKER_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' "$DOCKER_IMAGE") + DIGEST="${DOCKER_DIGEST#*@}" + echo "DIGEST=$DIGEST" >> $GITHUB_ENV - name: Install terraform uses: hashicorp/setup-terraform@v3 with: terraform_version: 1.10.5 - - name: Install AWS Cli + - name: Install AWS CLI run: sudo snap install --classic aws-cli - name: Check if any deployments are running run: ../scripts/check-for-running-deployments.sh ${{ inputs.environment }} - - name: Get image digest - run: | - DIGEST="${{ inputs.docker_sha }}" - if terraform state list | grep -q 'aws_ecs_task_definition.task_definition'; then - DIGEST=$(terraform state show aws_ecs_task_definition.task_definition | grep -oP '(?<=mavis/webapp@)sha256:[0-9a-z]{64}') - echo "Existing task definition found, using image digest from the state: $DIGEST" - elif [ -z "$DIGEST" ]; then - echo "Aborting infrastructure deployment: Missing existing task definition or image digest input parameter" - else - echo "No existing task definition found: Using image digest from the input parameter: $DIGEST" - fi - echo "DIGEST=$DIGEST" >> $GITHUB_ENV - name: Terraform Plan id: plan run: | set -e + terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade terraform plan -var="image_digest=$DIGEST" -var-file="env/${{ inputs.environment }}.tfvars" \ -out ${{ runner.temp }}/tfplan | tee ${{ runner.temp }}/tf_stdout - name: Validate the changes diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3f7182a118..12b084b5bb 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -50,5 +50,4 @@ jobs: uses: ./.github/workflows/deploy-application.yml with: environment: ${{ inputs.environment }} - image_tag: ${{ github.sha }} server_types: ${{ inputs.server_types }} diff --git a/terraform/scripts/check_task_definition.sh b/terraform/scripts/check_task_definition.sh deleted file mode 100755 index e12c8dbc0c..0000000000 --- a/terraform/scripts/check_task_definition.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash - -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " - exit 1 -fi - -valid_resources=( - "aws_ecs_task_definition\.task_definition" #TODO: Remove after release - "aws_s3_object\.appspec_object" - "module\.web_service\.aws_ecs_task_definition" - "module\.good_job_service\.aws_ecs_task_definition" -) - -tf_stdout=$1 -if [[ $(grep -ce "No changes.*Your infrastructure matches the configuration" "$tf_stdout") -eq 1 ]]; then - echo "No changes detected, continuing." - exit 0 -fi - -MODIFICATIONS=$(grep -E "[0-9]+ to add, [0-9]+ to change, [0-9]+ to destroy." "$tf_stdout") || exit 1 -ADDITIONS=$(echo "$MODIFICATIONS" | sed -E 's/.*([0-9]+) to add.*/\1/') || exit 1 -DELETIONS=$(echo "$MODIFICATIONS" | sed -E 's/.*([0-9]+) to destroy.*/\1/') || exit 1 -if [[ $DELETIONS -gt $ADDITIONS ]]; then - echo "ERROR: More resources are being destroyed than created, run infrastructure deploy first." - exit 1 -else - echo "CHECK_PASSED: No resources are being destroyed without replacement." -fi - -mapfile -t PLANNED_CHANGES < <(grep -E "#.+(replaced|created|updated in-place|destroyed)" "$tf_stdout" || exit 1) - -invalid_modifications=() -for change in "${PLANNED_CHANGES[@]}"; do - valid=0 - for resource in "${valid_resources[@]}"; do - if [[ "$change" =~ $resource ]]; then - valid=1 - break - fi - done - if [ $valid -eq 0 ]; then - invalid_modifications+=("$change") - fi -done - -if [ ! ${#invalid_modifications[@]} -eq 0 ]; then - echo "FAILED_CHECK: Invalid resources modified" - for item in "${invalid_modifications[@]}"; do - echo " $item" - done - echo "Please run an infrastructure deployment." - exit 1 -else - echo "CHECK_PASSED: All modified resources are expected." -fi - -echo "Basic checks passed, if production please evaluate the plan before applying." From c959f94e51719c64083955984a54d80e462572ff Mon Sep 17 00:00:00 2001 From: Brage Gording Date: Thu, 10 Apr 2025 13:56:35 +0100 Subject: [PATCH 2/2] Post release of v2.1.3 remove old service - The old service is replaced by identical service with new name - This used the module and simplifies setup/running/developing in the future --- terraform/app/ecs.tf | 108 ------------------------------------------- terraform/app/rds.tf | 13 ------ 2 files changed, 121 deletions(-) diff --git a/terraform/app/ecs.tf b/terraform/app/ecs.tf index 6ddeda5f7e..e0e1b3ac34 100644 --- a/terraform/app/ecs.tf +++ b/terraform/app/ecs.tf @@ -1,111 +1,3 @@ -#TODO: Remove after release -resource "aws_security_group" "ecs_service_sg" { - name = "ecs-service-sg" - description = "Security Group for communication with ECS" - vpc_id = aws_vpc.application_vpc.id - lifecycle { - ignore_changes = [description] - } -} - -#TODO: Remove after release -resource "aws_security_group_rule" "ecs_ingress_http" { - type = "ingress" - from_port = 4000 - to_port = 4000 - protocol = "tcp" - security_group_id = aws_security_group.ecs_service_sg.id - source_security_group_id = aws_security_group.lb_service_sg.id - lifecycle { - create_before_destroy = true - } -} - -#TODO: Remove after release -resource "aws_security_group_rule" "ecs_talk_to_internet" { - type = "egress" - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - security_group_id = aws_security_group.ecs_service_sg.id -} - -#TODO: Remove after release -resource "aws_ecs_service" "service" { - name = "mavis-${var.environment}" - cluster = aws_ecs_cluster.cluster.id - task_definition = aws_ecs_task_definition.task_definition.arn - desired_count = var.minimum_web_replicas - launch_type = "FARGATE" - enable_execute_command = true - health_check_grace_period_seconds = 60 - - network_configuration { - subnets = [aws_subnet.private_subnet_a.id, aws_subnet.private_subnet_b.id] - security_groups = [aws_security_group.ecs_service_sg.id] - } - - load_balancer { - target_group_arn = aws_lb_target_group.blue.arn - container_name = "mavis-${var.environment}" - container_port = 4000 - } - deployment_controller { - type = "CODE_DEPLOY" - } - - lifecycle { - ignore_changes = [ - load_balancer, - task_definition, - # desired_count TODO: uncomment this when we proceed with enabling autoscaler - ] - } -} - -#TODO: Remove after release -resource "aws_ecs_task_definition" "task_definition" { - family = "task-definition-${var.environment}" - requires_compatibilities = ["FARGATE"] - network_mode = "awsvpc" - cpu = 1024 - memory = 2048 - execution_role_arn = aws_iam_role.ecs_task_execution_role.arn - task_role_arn = aws_iam_role.ecs_task_role.arn - container_definitions = jsonencode([ - { - name = "mavis-${var.environment}" - image = "${var.account_id}.dkr.ecr.eu-west-2.amazonaws.com/${var.docker_image}@${var.image_digest}" - essential = true - portMappings = [ - { - containerPort = 4000 - hostPort = 4000 - } - ] - environment = concat(local.task_envs, [{ name = "SERVER_TYPE", value = "web" }]) - secrets = local.task_secrets - logConfiguration = { - logDriver = "awslogs" - options = { - awslogs-group = aws_cloudwatch_log_group.ecs_log_group.name - awslogs-region = var.region - awslogs-stream-prefix = "${var.environment}-logs" - } - } - healthCheck = { - command = ["CMD-SHELL", "curl -f http://localhost:4000/up || exit 1"] - interval = 30 - timeout = 5 - retries = 3 - startPeriod = 10 - } - } - ]) - depends_on = [aws_cloudwatch_log_group.ecs_log_group] -} - resource "aws_security_group_rule" "web_service_alb_ingress" { type = "ingress" from_port = 4000 diff --git a/terraform/app/rds.tf b/terraform/app/rds.tf index 78fff61e1c..944631a161 100644 --- a/terraform/app/rds.tf +++ b/terraform/app/rds.tf @@ -24,19 +24,6 @@ resource "aws_security_group_rule" "rds_ecs_ingress" { } } -#TODO: Remove after release -resource "aws_security_group_rule" "rds_ingress" { - type = "ingress" - from_port = 5432 - to_port = 5432 - protocol = "tcp" - security_group_id = aws_security_group.rds_security_group.id - source_security_group_id = aws_security_group.ecs_service_sg.id - lifecycle { - create_before_destroy = true - } -} - resource "aws_db_subnet_group" "aurora_subnet_group" { name = var.resource_name.dbsubnet_group description = "Group of private subnets for Aurora Serverless v2 cluster."