Feature/updated gh action (#42)

sks · web-flow · commit 4c2f4048eae6 · 2024-05-14T22:44:32.000+05:30
* update pr diff

* tag

* use latest image

* this better

* use latewt

* fix

* limit runs

* more perm issions

* Adds IaC dependency files

* check this

* more privs

* add all ew file

* ignore cpg

* use scan folder

* change port

* upload logs

* more verbose logs

* perm

* Safe

* skip docker login

* which dir

* new files track
diff --git a/.github/workflows/appcd-iac-pr-diff.yml b/.github/workflows/appcd-iac-pr-diff.yml
@@ -5,40 +5,74 @@ on:
       - main
 jobs:
   compare-artifacts:
+    permissions:
+      contents: write
+      pull-requests: write
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Main Branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           ref: main
           path: main_branch
-      - name: Login to docker
-        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
-
+          fetch-depth: 1
       - name: Generate IaC from Main Branch
+        env:
+          APPCD_TOKEN: ${{ secrets.APPCD_TOKEN }}
+          APPCD_URL: ${{ secrets.APPCD_URL }}
         run: |
-          mkdir -p artifact/main/
+          mkdir -p artifact/main/ ./artifact/tmp
           docker run --rm \
-          --workdir=/code  \
-          -v ./main_branch:/code -v ./artifact/main:/artifact/main ghcr.io/appcd-dev/appcd-dist/appcd@sha256:a38ade31e60f3f7f76b1135a388db158eed3c90816d5b5c09e33dd806efb67d5 \
-          generate --mode ci --output=/artifact/main/.appcd/charts
+            --workdir=/app/scan  \
+            -e APPCD_TOKEN=$APPCD_TOKEN \
+            -e APPCD_URL=$APPCD_URL \
+            -v ./main_branch:/app/scan \
+            -v ./artifact/tmp:/tmp \
+            -v ./artifact/main:/artifact/main \
+            --entrypoint=appcd \
+          ghcr.io/appcd-dev/appcd-dist/appcd-cli:v0.9.1 \
+          generate --mode ci --lang Python --log 2 --output=/artifact/main/.appcd/charts --iac-type Helm
+          cd artifact/main/.appcd/charts
+          unzip scan.zip && rm scan.zip && ls -latr && pwd
+      - name: Upload logs
+        uses: actions/upload-artifact@v2
+        with:
+          name: analyzer_logs_1
+          path: artifact
       - name: Checkout PR Branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.head_ref }}
           path: pr_branch
+          fetch-depth: 1
       - name: Extract branch name
         id: extract_branch
         run: echo "branch=$(basename ${{ github.head_ref}})" >> $GITHUB_OUTPUT
       - name: echo branch name
         run: echo ${{ steps.extract_branch.outputs.branch }}
       - name: Generate IaC from PR branch
+        env:
+          APPCD_TOKEN: ${{ secrets.APPCD_TOKEN }}
+          APPCD_URL: ${{ secrets.APPCD_URL }}
         run: |
-          mkdir -p artifact/${{ steps.extract_branch.outputs.branch }}/
+          mkdir -p artifact/${{ steps.extract_branch.outputs.branch }}/ ./artifact/tmp
           docker run --rm \
-          --workdir=/code  \
-          -v ./pr_branch/:/code -v ./artifact/${{ steps.extract_branch.outputs.branch }}:/artifact/${{ steps.extract_branch.outputs.branch }} ghcr.io/appcd-dev/appcd-dist/appcd@sha256:a38ade31e60f3f7f76b1135a388db158eed3c90816d5b5c09e33dd806efb67d5 \
-          generate --mode ci --output=/artifact/${{ steps.extract_branch.outputs.branch }}/.appcd/charts
+            --workdir=/app/scan  \
+            -v ./pr_branch/:/app/scan \
+            -v ./artifact/${{ steps.extract_branch.outputs.branch }}:/artifact/${{ steps.extract_branch.outputs.branch }} \
+            -v ./artifact/tmp:/tmp \
+            -e APPCD_TOKEN=$APPCD_TOKEN \
+            -e APPCD_URL=$APPCD_URL \
+            --entrypoint=appcd \
+            ghcr.io/appcd-dev/appcd-dist/appcd-cli:v0.9.1 \
+            generate --mode ci --lang Python --log 2 --iac-type Helm --output=/artifact/${{ steps.extract_branch.outputs.branch }}/.appcd/charts
+          cd artifact/${{ steps.extract_branch.outputs.branch }}/.appcd/charts
+          unzip scan.zip && rm scan.zip && ls -latr
+      - name: Upload logs
+        uses: actions/upload-artifact@v2
+        with:
+          name: analyzer_logs_2
+          path: artifact
       - name: Copy infrastructure files if empty
         run: |
           cd pr_branch
@@ -51,22 +85,22 @@ jobs:
             git commit -m "Adds IaC dependency files"
             git push
           fi
-          cd ../
       - name: Generate diff between Main and PR branch
         run: |
           mkdir -p pr_branch/deployment_files
-          mv ./artifact/main/.appcd pr_branch/deployment_files/
+          mv ./artifact/main/.appcd/charts/helm/scan_*/* pr_branch/deployment_files/
           cd pr_branch
           git config --local user.email "action@github.com"
           git config --local user.name "GitHub Action"
           git add deployment_files
           git commit -m "staging deployment files from main to compare them"
-          rm -rf deploment_files/*
-          rm -rf deployment_files/.appcd
+          rm -rf deploment_files
           cd ..
-          mv artifact/${{ steps.extract_branch.outputs.branch }}/.appcd pr_branch/deployment_files/
+          mkdir -p pr_branch/deployment_files/
+          cp -R artifact/${{ steps.extract_branch.outputs.branch }}/.appcd/charts/helm/scan_*/* pr_branch/deployment_files/
           cd pr_branch
-          git diff --output=../diff.txt deployment_files/ | cat
+          git add .
+          git diff --staged --output=../diff.txt deployment_files/ | cat
           cat ../diff.txt
       - name: Comment PR with IaC Changes
         uses: actions/github-script@v6
diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
@@ -24,7 +24,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
 
       - name: Run tests
         run: |
@@ -50,6 +52,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
 
       - name: Build image
         run: docker build . --file Dockerfile --tag $IMAGE_NAME
@@ -91,9 +95,11 @@ jobs:
       - name: run appCD
         run: |
           docker run --rm \
-          --workdir=/code  \
-          -v $PWD:/code ghcr.io/appcd-dev/appcd-dist/appcd@sha256:a38ade31e60f3f7f76b1135a388db158eed3c90816d5b5c09e33dd806efb67d5 \
-          generate --mode ci --output=/code/.appcd/charts
+            --workdir=/code  \
+            -v $PWD:/code ghcr.io/appcd-dev/appcd-dist/appcd-cli:v0.9.0 \
+            generate --mode ci --lang Python --iac-type Helm --output=/code/.appcd/charts
+          cd .appcd/charts && ls
+          unzip DogeAPI.zip && rm DogeAPI.zip
 
       - name: Inflate helm chart in gitops/
         run: |
diff --git a/.gitignore b/.gitignore
@@ -174,3 +174,4 @@ dev.pem
 .appcd/charts/
 cpg.bin
 analyzer.log
+*.cpg.bin
diff --git a/Dockerfile b/Dockerfile
@@ -26,7 +26,7 @@ USER 1000
 ENV ACCESS_LOG=${ACCESS_LOG:-/proc/1/fd/1}
 ENV ERROR_LOG=${ERROR_LOG:-/proc/1/fd/2}
 
-EXPOSE 8000
+EXPOSE 8080
 
 # Define the Uvicorn command to run our application
-CMD ["uvicorn", "main:app", "--reload", "--workers", "1", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "main:app", "--reload", "--workers", "1", "--host", "0.0.0.0", "--port", "8080"]
diff --git a/infrastructure/rds/README.md b/infrastructure/rds/README.md
@@ -0,0 +1,63 @@
+# RDS
+
+Infrastructure templates used to manage the Postgres RDS DB cluster
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.13.1 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 4.9 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | 4.65.0 |
+| <a name="provider_terraform"></a> [terraform](#provider\_terraform) | n/a |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_aurora_postgresql_v2"></a> [aurora\_postgresql\_v2](#module\_aurora\_postgresql\_v2) | terraform-aws-modules/rds-aurora/aws | n/a |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_rds_engine_version.postgresql](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/rds_engine_version) | data source |
+| [terraform_remote_state.vpc](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/data-sources/remote_state) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_aws_region"></a> [aws\_region](#input\_aws\_region) | AWS region | `any` | n/a | yes |
+| <a name="input_database_name"></a> [database\_name](#input\_database\_name) | Database name | `any` | n/a | yes |
+| <a name="input_environment"></a> [environment](#input\_environment) | Environment name | `any` | n/a | yes |
+| <a name="input_postgresql_engine_version"></a> [postgresql\_engine\_version](#input\_postgresql\_engine\_version) | PostgreSQL engine version | `string` | `"15.2"` | no |
+| <a name="input_state_bucket_name"></a> [state\_bucket\_name](#input\_state\_bucket\_name) | State bucket name | `string` | `"cafi-dev-state-bucket"` | no |
+| <a name="input_state_bucket_region"></a> [state\_bucket\_region](#input\_state\_bucket\_region) | State bucket region | `string` | `"us-east-1"` | no |
+| <a name="input_state_bucket_vpc_key"></a> [state\_bucket\_vpc\_key](#input\_state\_bucket\_vpc\_key) | State bucket VPC key | `string` | `"cafi-dev/cafi/infrastructure/vpc/terraform.tfstate"` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_aurora_postgresql_v2_additional_cluster_endpoints"></a> [aurora\_postgresql\_v2\_additional\_cluster\_endpoints](#output\_aurora\_postgresql\_v2\_additional\_cluster\_endpoints) | A map of additional cluster endpoints and their attributes |
+| <a name="output_aurora_postgresql_v2_cluster_arn"></a> [aurora\_postgresql\_v2\_cluster\_arn](#output\_aurora\_postgresql\_v2\_cluster\_arn) | Amazon Resource Name (ARN) of cluster |
+| <a name="output_aurora_postgresql_v2_cluster_database_name"></a> [aurora\_postgresql\_v2\_cluster\_database\_name](#output\_aurora\_postgresql\_v2\_cluster\_database\_name) | Name for an automatically created database on cluster creation |
+| <a name="output_aurora_postgresql_v2_cluster_endpoint"></a> [aurora\_postgresql\_v2\_cluster\_endpoint](#output\_aurora\_postgresql\_v2\_cluster\_endpoint) | Writer endpoint for the cluster |
+| <a name="output_aurora_postgresql_v2_cluster_engine_version_actual"></a> [aurora\_postgresql\_v2\_cluster\_engine\_version\_actual](#output\_aurora\_postgresql\_v2\_cluster\_engine\_version\_actual) | The running version of the cluster database |
+| <a name="output_aurora_postgresql_v2_cluster_hosted_zone_id"></a> [aurora\_postgresql\_v2\_cluster\_hosted\_zone\_id](#output\_aurora\_postgresql\_v2\_cluster\_hosted\_zone\_id) | The Route53 Hosted Zone ID of the endpoint |
+| <a name="output_aurora_postgresql_v2_cluster_id"></a> [aurora\_postgresql\_v2\_cluster\_id](#output\_aurora\_postgresql\_v2\_cluster\_id) | The RDS Cluster Identifier |
+| <a name="output_aurora_postgresql_v2_cluster_instances"></a> [aurora\_postgresql\_v2\_cluster\_instances](#output\_aurora\_postgresql\_v2\_cluster\_instances) | A map of cluster instances and their attributes |
+| <a name="output_aurora_postgresql_v2_cluster_master_password"></a> [aurora\_postgresql\_v2\_cluster\_master\_password](#output\_aurora\_postgresql\_v2\_cluster\_master\_password) | The database master password |
+| <a name="output_aurora_postgresql_v2_cluster_master_username"></a> [aurora\_postgresql\_v2\_cluster\_master\_username](#output\_aurora\_postgresql\_v2\_cluster\_master\_username) | The database master username |
+| <a name="output_aurora_postgresql_v2_cluster_members"></a> [aurora\_postgresql\_v2\_cluster\_members](#output\_aurora\_postgresql\_v2\_cluster\_members) | List of RDS Instances that are a part of this cluster |
+| <a name="output_aurora_postgresql_v2_cluster_port"></a> [aurora\_postgresql\_v2\_cluster\_port](#output\_aurora\_postgresql\_v2\_cluster\_port) | The database port |
+| <a name="output_aurora_postgresql_v2_cluster_reader_endpoint"></a> [aurora\_postgresql\_v2\_cluster\_reader\_endpoint](#output\_aurora\_postgresql\_v2\_cluster\_reader\_endpoint) | A read-only endpoint for the cluster, automatically load-balanced across replicas |
+| <a name="output_aurora_postgresql_v2_cluster_resource_id"></a> [aurora\_postgresql\_v2\_cluster\_resource\_id](#output\_aurora\_postgresql\_v2\_cluster\_resource\_id) | The RDS Cluster Resource ID |
+| <a name="output_aurora_postgresql_v2_cluster_role_associations"></a> [aurora\_postgresql\_v2\_cluster\_role\_associations](#output\_aurora\_postgresql\_v2\_cluster\_role\_associations) | A map of IAM roles associated with the cluster and their attributes |
+| <a name="output_aurora_postgresql_v2_db_subnet_group_name"></a> [aurora\_postgresql\_v2\_db\_subnet\_group\_name](#output\_aurora\_postgresql\_v2\_db\_subnet\_group\_name) | The db subnet group name |
diff --git a/infrastructure/rds/main.tf b/infrastructure/rds/main.tf
@@ -0,0 +1,76 @@
+terraform {
+  required_version = "~> 1.5.6"
+
+  backend "s3" {
+    bucket   = "cafi-demo1"
+    key      = "demos/dogeapi/rds/terraform.tfstate"
+    region   = "us-east-1"
+    role_arn = "arn:aws:iam::180217099948:role/atlantis-access"
+  }
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+  }
+}
+
+provider "aws" {
+  region = var.aws_region
+  assume_role {
+    role_arn = "arn:aws:iam::180217099948:role/atlantis-access"
+  }
+}
+
+locals {
+  tags = {
+    Terraform   = "True"
+    Environment = var.environment
+  }
+}
+
+# PostgreSQL Serverless v2
+data "aws_rds_engine_version" "postgresql" {
+  engine  = "aurora-postgresql"
+  version = var.postgresql_engine_version
+}
+
+module "aurora_postgresql_v2" {
+  source = "terraform-aws-modules/rds-aurora/aws"
+
+  name                        = "${var.database_name}-${var.environment}"
+  engine                      = data.aws_rds_engine_version.postgresql.engine
+  engine_mode                 = "provisioned"
+  engine_version              = data.aws_rds_engine_version.postgresql.version
+  storage_encrypted           = true
+  manage_master_user_password = true
+  master_username             = "root"
+
+  vpc_id               = var.vpc_id
+  db_subnet_group_name = var.db_subnet_group_name
+  security_group_rules = {
+    vpc_ingress = {
+      cidr_blocks = var.private_subnets_cidr_blocks
+    }
+  }
+
+  # Enhanced monitoring disabled for now
+  monitoring_interval = 0
+
+  apply_immediately   = true
+  skip_final_snapshot = true
+
+  serverlessv2_scaling_configuration = {
+    min_capacity = 0.5
+    max_capacity = 1
+  }
+
+  instance_class = "db.serverless"
+  instances = {
+    one = {}
+    two = {}
+  }
+
+  tags = local.tags
+}
diff --git a/infrastructure/rds/output.tf b/infrastructure/rds/output.tf
@@ -0,0 +1,87 @@
+# aws_db_subnet_group
+output "aurora_postgresql_v2_db_subnet_group_name" {
+  description = "The db subnet group name"
+  value       = module.aurora_postgresql_v2.db_subnet_group_name
+}
+
+# aws_rds_cluster
+output "aurora_postgresql_v2_cluster_arn" {
+  description = "Amazon Resource Name (ARN) of cluster"
+  value       = module.aurora_postgresql_v2.cluster_arn
+}
+
+output "aurora_postgresql_v2_cluster_id" {
+  description = "The RDS Cluster Identifier"
+  value       = module.aurora_postgresql_v2.cluster_id
+}
+
+output "aurora_postgresql_v2_cluster_resource_id" {
+  description = "The RDS Cluster Resource ID"
+  value       = module.aurora_postgresql_v2.cluster_resource_id
+}
+
+output "aurora_postgresql_v2_cluster_members" {
+  description = "List of RDS Instances that are a part of this cluster"
+  value       = module.aurora_postgresql_v2.cluster_members
+}
+
+output "aurora_postgresql_v2_cluster_endpoint" {
+  description = "Writer endpoint for the cluster"
+  value       = module.aurora_postgresql_v2.cluster_endpoint
+}
+
+output "aurora_postgresql_v2_cluster_reader_endpoint" {
+  description = "A read-only endpoint for the cluster, automatically load-balanced across replicas"
+  value       = module.aurora_postgresql_v2.cluster_reader_endpoint
+}
+
+output "aurora_postgresql_v2_cluster_engine_version_actual" {
+  description = "The running version of the cluster database"
+  value       = module.aurora_postgresql_v2.cluster_engine_version_actual
+}
+
+# database_name is not set on `aws_rds_cluster` resource if it was not specified, so can't be used in output
+output "aurora_postgresql_v2_cluster_database_name" {
+  description = "Name for an automatically created database on cluster creation"
+  value       = module.aurora_postgresql_v2.cluster_database_name
+}
+
+output "aurora_postgresql_v2_cluster_port" {
+  description = "The database port"
+  value       = module.aurora_postgresql_v2.cluster_port
+}
+
+output "aurora_postgresql_v2_cluster_master_password" {
+  description = "The database master password"
+  value       = module.aurora_postgresql_v2.cluster_master_password
+  sensitive   = true
+}
+
+output "aurora_postgresql_v2_cluster_master_username" {
+  description = "The database master username"
+  value       = module.aurora_postgresql_v2.cluster_master_username
+  sensitive   = true
+}
+
+output "aurora_postgresql_v2_cluster_hosted_zone_id" {
+  description = "The Route53 Hosted Zone ID of the endpoint"
+  value       = module.aurora_postgresql_v2.cluster_hosted_zone_id
+}
+
+# aws_rds_cluster_instances
+output "aurora_postgresql_v2_cluster_instances" {
+  description = "A map of cluster instances and their attributes"
+  value       = module.aurora_postgresql_v2.cluster_instances
+}
+
+# aws_rds_cluster_endpoint
+output "aurora_postgresql_v2_additional_cluster_endpoints" {
+  description = "A map of additional cluster endpoints and their attributes"
+  value       = module.aurora_postgresql_v2.additional_cluster_endpoints
+}
+
+# aws_rds_cluster_role_association
+output "aurora_postgresql_v2_cluster_role_associations" {
+  description = "A map of IAM roles associated with the cluster and their attributes"
+  value       = module.aurora_postgresql_v2.cluster_role_associations
+}
diff --git a/infrastructure/rds/terraform.tfvars b/infrastructure/rds/terraform.tfvars
diff --git a/infrastructure/rds/variables.tf b/infrastructure/rds/variables.tf
