Skip to content

Commit 73d1fc9

Browse files
TheOneFromNorwayTheOneFromNorway
committed
Update workflows
- Make matrix functionality respect server selection - Update data replication to use new ECS service/deployment flow - Cleanup unnecessary components
1 parent 1c16c63 commit 73d1fc9

File tree

8 files changed

+281
-213
lines changed

8 files changed

+281
-213
lines changed

‎.github/workflows/data-replication-pipeline.yml‎

Lines changed: 98 additions & 152 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
name: Data replication pipeline
2-
run-name: ${{ inputs.deployment_type }} for data replication resources for ${{ inputs.environment }}
1+
name: Deploy Data Replication
2+
run-name: Deploy Data Replication service for ${{ inputs.environment }}
33

44
on:
55
workflow_dispatch:
@@ -15,205 +15,151 @@ on:
1515
- qa
1616
- sandbox-alpha
1717
- sandbox-beta
18-
deployment_type:
19-
description: Deployment type
20-
required: true
21-
type: choice
22-
options:
23-
- Deployment with DB recreation
24-
- Application only deployment
2518
image_tag:
2619
description: Docker image tag to deploy
2720
required: false
2821
type: string
29-
db_snapshot_arn:
30-
description: ARN of the DB snapshot to use (optional)
31-
required: false
32-
type: string
33-
egress_cidr:
34-
description: CIDR blocks to allow egress traffic.
35-
type: string
36-
required: true
37-
default: "[]"
3822

3923
env:
40-
aws_role: ${{ inputs.environment == 'production'
24+
aws-role: ${{ inputs.environment == 'production'
4125
&& 'arn:aws:iam::820242920762:role/GithubDeployDataReplicationInfrastructure'
4226
|| 'arn:aws:iam::393416225559:role/GithubDeployDataReplicationInfrastructure' }}
43-
44-
defaults:
45-
run:
46-
working-directory: terraform/data_replication
27+
aws_account_id: ${{ inputs.environment == 'production' && '820242920762' || '393416225559' }}
4728

4829
concurrency:
4930
group: deploy-data-replica-${{ inputs.environment }}
5031

5132
jobs:
52-
prepare-db-replica:
53-
if: ${{ inputs.deployment_type == 'Deployment with DB recreation' }}
54-
name: Prepare data replica
33+
validate-inputs:
5534
runs-on: ubuntu-latest
56-
permissions:
57-
id-token: write
35+
permissions: { }
5836
steps:
59-
- name: Checkout code
60-
uses: actions/checkout@v5
61-
- name: Configure AWS Credentials
62-
uses: aws-actions/configure-aws-credentials@v4
63-
with:
64-
role-to-assume: ${{ env.aws_role }}
65-
aws-region: eu-west-2
66-
- name: get latest snapshot
67-
id: get-latest-snapshot
37+
- name: Validate inputs
6838
run: |
69-
set -e
70-
if [ -z "${{ inputs.db_snapshot_arn }}" ]; then
71-
echo "No snapshot ARN provided, fetching the latest snapshot"
72-
SNAPSHOT_ARN=$(aws rds describe-db-cluster-snapshots \
73-
--query "DBClusterSnapshots[?DBClusterIdentifier=='mavis-${{ inputs.environment }}'].[DBClusterSnapshotArn, SnapshotCreateTime]" \
74-
--output text | sort -k2 -r | head -n 1 | cut -f1)
75-
76-
if [ -z "$SNAPSHOT_ARN" ]; then
77-
echo "No snapshots found for mavis-${{ inputs.environment }}"
78-
exit 1
79-
fi
80-
else
81-
echo "Using provided snapshot ARN: ${{ inputs.db_snapshot_arn }}"
82-
SNAPSHOT_ARN="${{ inputs.db_snapshot_arn }}"
39+
if [[ "${{ inputs.environment }}" == "preview" || "${{ inputs.environment }}" == "production" ]]; then
40+
if [[ -z "${{ inputs.git_ref_to_deploy }}" ]]; then
41+
echo "Error: git_ref_to_deploy is required for preview and production environments."
42+
exit 1
43+
fi
8344
fi
84-
echo "Using snapshot ARN: $SNAPSHOT_ARN"
85-
echo "SNAPSHOT_ARN=$SNAPSHOT_ARN" >> $GITHUB_OUTPUT
86-
- name: Install terraform
87-
uses: hashicorp/setup-terraform@v3
88-
with:
89-
terraform_version: 1.11.4
90-
outputs:
91-
SNAPSHOT_ARN: ${{ steps.get-latest-snapshot.outputs.SNAPSHOT_ARN }}
92-
93-
prepare-webapp:
94-
name: Prepare webapp
45+
determine-git-sha:
9546
runs-on: ubuntu-latest
96-
permissions:
97-
id-token: write
47+
permissions: { }
48+
needs: validate-inputs
49+
outputs:
50+
git-sha: ${{ steps.get-git-sha.outputs.git-sha }}
9851
steps:
9952
- name: Checkout code
10053
uses: actions/checkout@v5
101-
- name: Configure AWS Credentials
102-
uses: aws-actions/configure-aws-credentials@v4
10354
with:
104-
role-to-assume: ${{ env.aws_role }}
105-
aws-region: eu-west-2
106-
- name: ECR login
107-
id: login-ecr
108-
uses: aws-actions/amazon-ecr-login@v2
109-
- name: Get docker image digest
110-
id: get-docker-image-digest
111-
run: |
112-
set -e
113-
DOCKER_IMAGE="${{ steps.login-ecr.outputs.registry }}/mavis/webapp:${{ inputs.image_tag || github.sha }}"
114-
docker pull "$DOCKER_IMAGE"
115-
DOCKER_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' "$DOCKER_IMAGE")
116-
DIGEST="${DOCKER_DIGEST#*@}"
117-
echo "DIGEST=$DIGEST" >> $GITHUB_OUTPUT
118-
outputs:
119-
DOCKER_DIGEST: ${{ steps.get-docker-image-digest.outputs.DIGEST }}
55+
ref: ${{ inputs.git_ref_to_deploy || github.sha }}
56+
- name: Get git sha
57+
id: get-git-sha
58+
run: echo "git-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
59+
build-and-push-image:
60+
permissions:
61+
id-token: write
62+
needs: determine-git-sha
63+
uses: ./.github/workflows/build-and-push-image.yml
64+
with:
65+
git-sha: ${{ needs.determine-git-sha.outputs.git-sha }}
12066

121-
plan:
122-
name: Terraform plan
67+
prepare-deployment:
68+
name: Prepare deployment
12369
runs-on: ubuntu-latest
124-
needs:
125-
- prepare-db-replica
126-
- prepare-webapp
127-
if: ${{ !cancelled() &&
128-
(needs.prepare-db-replica.result == 'success' || needs.prepare-db-replica.result == 'skipped') &&
129-
needs.prepare-webapp.result == 'success' }}
130-
env:
131-
SNAPSHOT_ARN: ${{ needs.prepare-db-replica.outputs.SNAPSHOT_ARN }}
132-
DB_SECRET_ARN: ${{ needs.prepare-db-replica.outputs.DB_SECRET_ARN || 'arn:aws:secretsmanager:eu-west-2:000000000000:secret:placeholder' }}
133-
DOCKER_DIGEST: ${{ needs.prepare-webapp.outputs.DOCKER_DIGEST }}
134-
REPLACE_DB_CLUSTER: ${{ inputs.deployment_type == 'Deployment with DB recreation' }}
70+
needs: build-and-push-image
13571
permissions:
13672
id-token: write
13773
steps:
13874
- name: Checkout code
13975
uses: actions/checkout@v5
76+
with:
77+
ref: ${{ inputs.git_sha_to_deploy || github.sha }}
14078
- name: Configure AWS Credentials
14179
uses: aws-actions/configure-aws-credentials@v4
14280
with:
143-
role-to-assume: ${{ env.aws_role }}
81+
role-to-assume: ${{ env.aws-role }}
14482
aws-region: eu-west-2
145-
- name: Install terraform
146-
uses: hashicorp/setup-terraform@v3
83+
- name: Setup python
84+
uses: actions/setup-python@v4
14785
with:
148-
terraform_version: 1.11.4
149-
- name: Get db secret arn
150-
id: get-db-secret-arn
151-
working-directory: terraform/app
86+
python-version: 3.12.3
87+
cache: pip
88+
- name: Install Python dependencies
89+
run: python3 -m pip install -r script/requirements.txt
90+
- name: Get image digest
91+
id: get-image-digest
15292
run: |
153-
terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
154-
DB_SECRET_ARN=$(terraform output --raw db_secret_arn)
155-
echo "DB_SECRET_ARN=$DB_SECRET_ARN" >> $GITHUB_OUTPUT
156-
- name: Terraform Plan
157-
id: plan
93+
digest=$(aws ecr describe-images \
94+
--repository-name mavis/webapp \
95+
--image-ids imageTag=${{ inputs.git_sha_to_deploy || github.sha }} \
96+
--query 'imageDetails[0].imageDigest' \
97+
--output text)
98+
echo "digest=$digest" >> $GITHUB_OUTPUT
99+
- name: Parse environment variables
100+
id: parse-environment-variables
158101
run: |
159-
set -eo pipefail
160-
terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
161-
162-
CIDR_BLOCKS='${{ inputs.egress_cidr }}'
163-
PLAN_ARGS=(
164-
"plan"
165-
"-var=image_digest=${{ env.DOCKER_DIGEST }}"
166-
"-var=db_secret_arn=${{ steps.get-db-secret-arn.outputs.DB_SECRET_ARN }}"
167-
"-var=imported_snapshot=${{ env.SNAPSHOT_ARN }}"
168-
"-var-file=env/${{ inputs.environment }}.tfvars"
169-
"-var=allowed_egress_cidr_blocks=$CIDR_BLOCKS"
170-
"-out=${{ runner.temp }}/tfplan"
171-
)
172-
173-
if [ "${{ env.REPLACE_DB_CLUSTER }}" = "true" ]; then
174-
PLAN_ARGS+=("-replace" "aws_rds_cluster.cluster")
175-
fi
176-
terraform "${PLAN_ARGS[@]}" | tee ${{ runner.temp }}/tf_stdout
177-
- name: Upload artifact
102+
parsed_env_vars=$(yq -r '.environments.${{ inputs.environment }} | to_entries | .[] | .key + "=" + .value' config/container_variables.yml)
103+
{
104+
echo 'parsed_env_vars<<EOF'
105+
echo "$parsed_env_vars"
106+
echo "MAVIS__SPLUNK__ENABLED=false"
107+
echo "MAVIS__CIS2__ENABLED=false"
108+
echo "MAVIS__PDS__ENQUEUE_BULK_UPDATES=false"
109+
echo "MAVIS__PDS__RATE_LIMIT_PER_SECOND=${{ inputs.environment == 'production' && 50 || 5 }}"
110+
echo 'EOF'
111+
} >> "$GITHUB_OUTPUT"
112+
- name: Populate web task definition
113+
id: create-task-definition
114+
uses: aws-actions/amazon-ecs-render-task-definition@v1
115+
with:
116+
task-definition-family: "mavis-data-replication-task-definition-${{ inputs.environment }}-template"
117+
container-name: "application"
118+
image: "${{ env.aws_account_id }}.dkr.ecr.eu-west-2.amazonaws.com/mavis/webapp@${{ steps.get-image-digest.outputs.digest }}"
119+
environment-variables: ${{ steps.parse-environment-variables.outputs.parsed_env_vars }}
120+
- name: Rename task definition file
121+
run: mv ${{ steps.create-task-definition.outputs.task-definition }} ${{ runner.temp }}/data-replication-task-definition.json
122+
- name: Upload artifact for data-replication task definition
178123
uses: actions/upload-artifact@v4
179124
with:
180-
name: tfplan_infrastructure-${{ inputs.environment }}
181-
path: ${{ runner.temp }}/tfplan
125+
name: ${{ inputs.environment }}-data-replication-task-definition
126+
path: ${{ runner.temp }}/data-replication-task-definition.json
182127

183-
apply:
184-
name: Terraform apply
128+
approve-deployments:
129+
name: Wait for approval if required
185130
runs-on: ubuntu-latest
186-
needs: plan
187-
if: ${{ !cancelled() && needs.plan.result == 'success' }}
131+
needs: prepare-deployment
188132
environment: ${{ inputs.environment }}
133+
steps:
134+
- run: echo "Proceeding with deployment to ${{ inputs.environment }} environment"
135+
136+
deploy-data-replication:
137+
name: Deploy data-replication service
138+
runs-on: ubuntu-latest
139+
needs: [ prepare-deployment, approve-deployments ]
189140
permissions:
190141
id-token: write
191142
steps:
192-
- name: Checkout code
193-
uses: actions/checkout@v5
194143
- name: Configure AWS Credentials
195144
uses: aws-actions/configure-aws-credentials@v4
196145
with:
197-
role-to-assume: ${{ env.aws_role }}
146+
role-to-assume: ${{ env.aws-role }}
198147
aws-region: eu-west-2
199-
- name: Download artifact
148+
- name: Download data-replication task definition artifact
200149
uses: actions/download-artifact@v5
201150
with:
202-
name: tfplan_infrastructure-${{ inputs.environment }}
203151
path: ${{ runner.temp }}
204-
- name: Install terraform
205-
uses: hashicorp/setup-terraform@v3
206-
with:
207-
terraform_version: 1.11.4
208-
- name: Apply the changes
152+
name: ${{ inputs.environment }}-data-replication-task-definition
153+
- name: Change family of task definition
209154
run: |
210-
set -e
211-
terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
212-
terraform apply ${{ runner.temp }}/tfplan
213-
- name: Deploy db-access-service
214-
run: |
215-
task_definition_arn=$(terraform output -raw task_definition_arn)
216-
aws ecs update-service \
217-
--cluster mavis-${{ inputs.environment }}-data-replication \
218-
--service mavis-${{ inputs.environment }}-data-replication \
219-
--task-definition $task_definition_arn
155+
file_path="${{ runner.temp }}/data-replication-task-definition.json"
156+
family_name="mavis-data-replication-task-definition-${{ inputs.environment }}"
157+
echo "$(jq --arg f "$family_name" '.family = $f' "$file_path")" > "$file_path"
158+
- name: Deploy data-replication service
159+
uses: aws-actions/amazon-ecs-deploy-task-definition@v2
160+
with:
161+
task-definition: ${{ runner.temp }}/data-replication-task-definition.json
162+
cluster: mavis-${{ inputs.environment }}-data-replication
163+
service: mavis-${{ inputs.environment }}-data-replication
164+
force-new-deployment: true
165+
wait-for-service-stability: true

‎.github/workflows/deploy-application.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ jobs:
6969
strategy:
7070
fail-fast: true
7171
matrix:
72-
service: [ web, good-job, sidekiq ]
72+
service: ${{ inputs.server_types == 'all' && fromJSON('["web", "good-job", "sidekiq"]') || fromJSON(format('["{0}"]', inputs.server_types)) }}
7373
steps:
7474
- name: Checkout code
7575
uses: actions/checkout@v5
@@ -165,7 +165,7 @@ jobs:
165165
id: deploy-web-service
166166
uses: aws-actions/amazon-ecs-deploy-task-definition@v2
167167
with:
168-
task-definition: ${{ runner.temp }}
168+
task-definition: ${{ runner.temp }}/web-task-definition.json
169169
codedeploy-appspec: config/templates/appspec.yaml
170170
cluster: ${{ env.cluster_name }}
171171
service: mavis-${{ inputs.environment }}-web
@@ -197,13 +197,13 @@ jobs:
197197
name: ${{ inputs.environment }}-good-job-task-definition
198198
- name: Change family of task definition
199199
run: |
200-
file_path="${{ runner.temp }}/web-task-definition.json"
200+
file_path="${{ runner.temp }}/good-job-task-definition.json"
201201
family_name="mavis-good-job-task-definition-${{ inputs.environment }}"
202202
echo "$(jq --arg f "$family_name" '.family = $f' "$file_path")" > "$file_path"
203203
- name: Deploy good-job service
204204
uses: aws-actions/amazon-ecs-deploy-task-definition@v2
205205
with:
206-
task-definition: ${{ runner.temp }}/web-task-definition.json
206+
task-definition: ${{ runner.temp }}/good-job-task-definition.json
207207
cluster: ${{ env.cluster_name }}
208208
service: mavis-${{ inputs.environment }}-good-job
209209
force-new-deployment: true

‎.github/workflows/deploy-infrastructure.yml‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,19 @@ on:
88
description: Deployment environment
99
required: true
1010
type: string
11-
image_tag:
12-
required: false
13-
type: string
1411
git_ref_to_deploy:
1512
required: true
1613
type: string
14+
workflow_dispatch:
15+
inputs:
16+
environment:
17+
description: Deployment environment
18+
required: true
19+
type: string
20+
git_ref_to_deploy:
21+
description: The git commit SHA to deploy.
22+
required: false
23+
type: string
1724

1825
permissions: {}
1926

‎.github/workflows/deploy.yml‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ jobs:
109109
update-permissions:
110110
runs-on: ubuntu-latest
111111
needs: validate-permissions
112-
if: always() && (inputs.environment == 'production' || inputs.environment == 'preview') && needs.validate-permissions.result == 'failure'
112+
if: ${{ !cancelled() && (inputs.environment == 'production' || inputs.environment == 'preview') && needs.validate-permissions.result == 'failure' }}
113113
environment: ${{ inputs.environment }}
114114
defaults:
115115
run:
@@ -138,19 +138,18 @@ jobs:
138138
validate-permissions,
139139
update-permissions,
140140
]
141-
if: always() &&
141+
if: ${{ !cancelled() &&
142142
((inputs.environment != 'production' && inputs.environment != 'preview') ||
143-
needs.validate-permissions.result == 'success' || needs.update-permissions.result == 'success')
143+
needs.validate-permissions.result == 'success' || needs.update-permissions.result == 'success') }}
144144
uses: ./.github/workflows/deploy-infrastructure.yml
145145
with:
146146
environment: ${{ inputs.environment }}
147-
image_tag: ${{ needs.determine-git-sha.outputs.git-sha }}
148147
git_ref_to_deploy: ${{ inputs.git_ref_to_deploy || github.ref_name }}
149148
deploy-application:
150149
permissions:
151150
id-token: write
152151
needs: [deploy-infrastructure, determine-git-sha]
153-
if: always() && inputs.server_types != 'none' && needs.deploy-infrastructure.result == 'success'
152+
if: ${{ !cancelled() && inputs.server_types != 'none' && needs.deploy-infrastructure.result == 'success' }}
154153
uses: ./.github/workflows/deploy-application.yml
155154
with:
156155
environment: ${{ inputs.environment }}

0 commit comments

Comments
 (0)