Skip to content

Commit fd28a16

Browse files
TheOneFromNorwayTheOneFromNorway
committed
Also update deployment of data replication
- This is required due to the new template-mechanism - Adheres to split between application and infrastructure
1 parent c18ea90 commit fd28a16

File tree

2 files changed

+263
-152
lines changed

2 files changed

+263
-152
lines changed

‎.github/workflows/data-replication-pipeline.yml‎

Lines changed: 98 additions & 152 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
name: Data replication pipeline
2-
run-name: ${{ inputs.deployment_type }} for data replication resources for ${{ inputs.environment }}
1+
name: Deploy Data Replication
2+
run-name: Deploy Data Replication service for ${{ inputs.environment }}
33

44
on:
55
workflow_dispatch:
@@ -15,205 +15,151 @@ on:
1515
- qa
1616
- sandbox-alpha
1717
- sandbox-beta
18-
deployment_type:
19-
description: Deployment type
20-
required: true
21-
type: choice
22-
options:
23-
- Deployment with DB recreation
24-
- Application only deployment
2518
image_tag:
2619
description: Docker image tag to deploy
2720
required: false
2821
type: string
29-
db_snapshot_arn:
30-
description: ARN of the DB snapshot to use (optional)
31-
required: false
32-
type: string
33-
egress_cidr:
34-
description: CIDR blocks to allow egress traffic.
35-
type: string
36-
required: true
37-
default: "[]"
3822

3923
env:
40-
aws_role: ${{ inputs.environment == 'production'
24+
aws-role: ${{ inputs.environment == 'production'
4125
&& 'arn:aws:iam::820242920762:role/GithubDeployDataReplicationInfrastructure'
4226
|| 'arn:aws:iam::393416225559:role/GithubDeployDataReplicationInfrastructure' }}
43-
44-
defaults:
45-
run:
46-
working-directory: terraform/data_replication
27+
aws_account_id: ${{ inputs.environment == 'production' && '820242920762' || '393416225559' }}
4728

4829
concurrency:
4930
group: deploy-data-replica-${{ inputs.environment }}
5031

5132
jobs:
52-
prepare-db-replica:
53-
if: ${{ inputs.deployment_type == 'Deployment with DB recreation' }}
54-
name: Prepare data replica
33+
validate-inputs:
5534
runs-on: ubuntu-latest
56-
permissions:
57-
id-token: write
35+
permissions: { }
5836
steps:
59-
- name: Checkout code
60-
uses: actions/checkout@v5
61-
- name: Configure AWS Credentials
62-
uses: aws-actions/configure-aws-credentials@v4
63-
with:
64-
role-to-assume: ${{ env.aws_role }}
65-
aws-region: eu-west-2
66-
- name: get latest snapshot
67-
id: get-latest-snapshot
37+
- name: Validate inputs
6838
run: |
69-
set -e
70-
if [ -z "${{ inputs.db_snapshot_arn }}" ]; then
71-
echo "No snapshot ARN provided, fetching the latest snapshot"
72-
SNAPSHOT_ARN=$(aws rds describe-db-cluster-snapshots \
73-
--query "DBClusterSnapshots[?DBClusterIdentifier=='mavis-${{ inputs.environment }}'].[DBClusterSnapshotArn, SnapshotCreateTime]" \
74-
--output text | sort -k2 -r | head -n 1 | cut -f1)
75-
76-
if [ -z "$SNAPSHOT_ARN" ]; then
77-
echo "No snapshots found for mavis-${{ inputs.environment }}"
78-
exit 1
79-
fi
80-
else
81-
echo "Using provided snapshot ARN: ${{ inputs.db_snapshot_arn }}"
82-
SNAPSHOT_ARN="${{ inputs.db_snapshot_arn }}"
39+
if [[ "${{ inputs.environment }}" == "preview" || "${{ inputs.environment }}" == "production" ]]; then
40+
if [[ -z "${{ inputs.git_ref_to_deploy }}" ]]; then
41+
echo "Error: git_ref_to_deploy is required for preview and production environments."
42+
exit 1
43+
fi
8344
fi
84-
echo "Using snapshot ARN: $SNAPSHOT_ARN"
85-
echo "SNAPSHOT_ARN=$SNAPSHOT_ARN" >> $GITHUB_OUTPUT
86-
- name: Install terraform
87-
uses: hashicorp/setup-terraform@v3
88-
with:
89-
terraform_version: 1.11.4
90-
outputs:
91-
SNAPSHOT_ARN: ${{ steps.get-latest-snapshot.outputs.SNAPSHOT_ARN }}
92-
93-
prepare-webapp:
94-
name: Prepare webapp
45+
determine-git-sha:
9546
runs-on: ubuntu-latest
96-
permissions:
97-
id-token: write
47+
permissions: { }
48+
needs: validate-inputs
49+
outputs:
50+
git-sha: ${{ steps.get-git-sha.outputs.git-sha }}
9851
steps:
9952
- name: Checkout code
10053
uses: actions/checkout@v5
101-
- name: Configure AWS Credentials
102-
uses: aws-actions/configure-aws-credentials@v4
10354
with:
104-
role-to-assume: ${{ env.aws_role }}
105-
aws-region: eu-west-2
106-
- name: ECR login
107-
id: login-ecr
108-
uses: aws-actions/amazon-ecr-login@v2
109-
- name: Get docker image digest
110-
id: get-docker-image-digest
111-
run: |
112-
set -e
113-
DOCKER_IMAGE="${{ steps.login-ecr.outputs.registry }}/mavis/webapp:${{ inputs.image_tag || github.sha }}"
114-
docker pull "$DOCKER_IMAGE"
115-
DOCKER_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' "$DOCKER_IMAGE")
116-
DIGEST="${DOCKER_DIGEST#*@}"
117-
echo "DIGEST=$DIGEST" >> $GITHUB_OUTPUT
118-
outputs:
119-
DOCKER_DIGEST: ${{ steps.get-docker-image-digest.outputs.DIGEST }}
55+
ref: ${{ inputs.git_ref_to_deploy || github.sha }}
56+
- name: Get git sha
57+
id: get-git-sha
58+
run: echo "git-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
59+
build-and-push-image:
60+
permissions:
61+
id-token: write
62+
needs: determine-git-sha
63+
uses: ./.github/workflows/build-and-push-image.yml
64+
with:
65+
git-sha: ${{ needs.determine-git-sha.outputs.git-sha }}
12066

121-
plan:
122-
name: Terraform plan
67+
prepare-deployment:
68+
name: Prepare deployment
12369
runs-on: ubuntu-latest
124-
needs:
125-
- prepare-db-replica
126-
- prepare-webapp
127-
if: ${{ !cancelled() &&
128-
(needs.prepare-db-replica.result == 'success' || needs.prepare-db-replica.result == 'skipped') &&
129-
needs.prepare-webapp.result == 'success' }}
130-
env:
131-
SNAPSHOT_ARN: ${{ needs.prepare-db-replica.outputs.SNAPSHOT_ARN }}
132-
DB_SECRET_ARN: ${{ needs.prepare-db-replica.outputs.DB_SECRET_ARN || 'arn:aws:secretsmanager:eu-west-2:000000000000:secret:placeholder' }}
133-
DOCKER_DIGEST: ${{ needs.prepare-webapp.outputs.DOCKER_DIGEST }}
134-
REPLACE_DB_CLUSTER: ${{ inputs.deployment_type == 'Deployment with DB recreation' }}
70+
needs: build-and-push-image
13571
permissions:
13672
id-token: write
13773
steps:
13874
- name: Checkout code
13975
uses: actions/checkout@v5
76+
with:
77+
ref: ${{ inputs.git_sha_to_deploy || github.sha }}
14078
- name: Configure AWS Credentials
14179
uses: aws-actions/configure-aws-credentials@v4
14280
with:
143-
role-to-assume: ${{ env.aws_role }}
81+
role-to-assume: ${{ env.aws-role }}
14482
aws-region: eu-west-2
145-
- name: Install terraform
146-
uses: hashicorp/setup-terraform@v3
83+
- name: Setup python
84+
uses: actions/setup-python@v4
14785
with:
148-
terraform_version: 1.11.4
149-
- name: Get db secret arn
150-
id: get-db-secret-arn
151-
working-directory: terraform/app
86+
python-version: 3.12.3
87+
cache: pip
88+
- name: Install Python dependencies
89+
run: python3 -m pip install -r script/requirements.txt
90+
- name: Get image digest
91+
id: get-image-digest
15292
run: |
153-
terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
154-
DB_SECRET_ARN=$(terraform output --raw db_secret_arn)
155-
echo "DB_SECRET_ARN=$DB_SECRET_ARN" >> $GITHUB_OUTPUT
156-
- name: Terraform Plan
157-
id: plan
93+
digest=$(aws ecr describe-images \
94+
--repository-name mavis/webapp \
95+
--image-ids imageTag=${{ inputs.git_sha_to_deploy || github.sha }} \
96+
--query 'imageDetails[0].imageDigest' \
97+
--output text)
98+
echo "digest=$digest" >> $GITHUB_OUTPUT
99+
- name: Parse environment variables
100+
id: parse-environment-variables
158101
run: |
159-
set -eo pipefail
160-
terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
161-
162-
CIDR_BLOCKS='${{ inputs.egress_cidr }}'
163-
PLAN_ARGS=(
164-
"plan"
165-
"-var=image_digest=${{ env.DOCKER_DIGEST }}"
166-
"-var=db_secret_arn=${{ steps.get-db-secret-arn.outputs.DB_SECRET_ARN }}"
167-
"-var=imported_snapshot=${{ env.SNAPSHOT_ARN }}"
168-
"-var-file=env/${{ inputs.environment }}.tfvars"
169-
"-var=allowed_egress_cidr_blocks=$CIDR_BLOCKS"
170-
"-out=${{ runner.temp }}/tfplan"
171-
)
172-
173-
if [ "${{ env.REPLACE_DB_CLUSTER }}" = "true" ]; then
174-
PLAN_ARGS+=("-replace" "aws_rds_cluster.cluster")
175-
fi
176-
terraform "${PLAN_ARGS[@]}" | tee ${{ runner.temp }}/tf_stdout
177-
- name: Upload artifact
102+
parsed_env_vars=$(yq -r '.environments.${{ inputs.environment }} | to_entries | .[] | .key + "=" + .value' config/container_variables.yml)
103+
{
104+
echo 'parsed_env_vars<<EOF'
105+
echo "$parsed_env_vars"
106+
echo "MAVIS__SPLUNK__ENABLED=false"
107+
echo "MAVIS__CIS2__ENABLED=false"
108+
echo "MAVIS__PDS__ENQUEUE_BULK_UPDATES=false"
109+
echo "MAVIS__PDS__RATE_LIMIT_PER_SECOND=${{ inputs.environment == 'production' && 50 || 5 }}"
110+
echo 'EOF'
111+
} >> "$GITHUB_OUTPUT"
112+
- name: Populate web task definition
113+
id: create-task-definition
114+
uses: aws-actions/amazon-ecs-render-task-definition@v1
115+
with:
116+
task-definition-family: "mavis-data-replication-task-definition-${{ inputs.environment }}-template"
117+
container-name: "application"
118+
image: "${{ env.aws_account_id }}.dkr.ecr.eu-west-2.amazonaws.com/mavis/webapp@${{ steps.get-image-digest.outputs.digest }}"
119+
environment-variables: ${{ steps.parse-environment-variables.outputs.parsed_env_vars }}
120+
- name: Rename task definition file
121+
run: mv ${{ steps.create-task-definition.outputs.task-definition }} ${{ runner.temp }}/data-replication-task-definition.json
122+
- name: Upload artifact for data-replication task definition
178123
uses: actions/upload-artifact@v4
179124
with:
180-
name: tfplan_infrastructure-${{ inputs.environment }}
181-
path: ${{ runner.temp }}/tfplan
125+
name: ${{ inputs.environment }}-data-replication-task-definition
126+
path: ${{ runner.temp }}/data-replication-task-definition.json
182127

183-
apply:
184-
name: Terraform apply
128+
approve-deployments:
129+
name: Wait for approval if required
185130
runs-on: ubuntu-latest
186-
needs: plan
187-
if: ${{ !cancelled() && needs.plan.result == 'success' }}
131+
needs: prepare-deployment
188132
environment: ${{ inputs.environment }}
133+
steps:
134+
- run: echo "Proceeding with deployment to ${{ inputs.environment }} environment"
135+
136+
deploy-data-replication:
137+
name: Deploy data-replication service
138+
runs-on: ubuntu-latest
139+
needs: [ prepare-deployment, approve-deployments ]
189140
permissions:
190141
id-token: write
191142
steps:
192-
- name: Checkout code
193-
uses: actions/checkout@v5
194143
- name: Configure AWS Credentials
195144
uses: aws-actions/configure-aws-credentials@v4
196145
with:
197-
role-to-assume: ${{ env.aws_role }}
146+
role-to-assume: ${{ env.aws-role }}
198147
aws-region: eu-west-2
199-
- name: Download artifact
148+
- name: Download data-replication task definition artifact
200149
uses: actions/download-artifact@v5
201150
with:
202-
name: tfplan_infrastructure-${{ inputs.environment }}
203151
path: ${{ runner.temp }}
204-
- name: Install terraform
205-
uses: hashicorp/setup-terraform@v3
206-
with:
207-
terraform_version: 1.11.4
208-
- name: Apply the changes
152+
name: ${{ inputs.environment }}-data-replication-task-definition
153+
- name: Change family of task definition
209154
run: |
210-
set -e
211-
terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
212-
terraform apply ${{ runner.temp }}/tfplan
213-
- name: Deploy db-access-service
214-
run: |
215-
task_definition_arn=$(terraform output -raw task_definition_arn)
216-
aws ecs update-service \
217-
--cluster mavis-${{ inputs.environment }}-data-replication \
218-
--service mavis-${{ inputs.environment }}-data-replication \
219-
--task-definition $task_definition_arn
155+
file_path="${{ runner.temp }}/data-replication-task-definition.json"
156+
family_name="mavis-data-replication-task-definition-${{ inputs.environment }}"
157+
echo "$(jq --arg f "$family_name" '.family = $f' "$file_path")" > "$file_path"
158+
- name: Deploy data-replication service
159+
uses: aws-actions/amazon-ecs-deploy-task-definition@v2
160+
with:
161+
task-definition: ${{ runner.temp }}/data-replication-task-definition.json
162+
cluster: mavis-${{ inputs.environment }}-data-replication
163+
service: mavis-${{ inputs.environment }}-data-replication
164+
force-new-deployment: true
165+
wait-for-service-stability: true

0 commit comments

Comments
 (0)