Include autoscaling for ECS service, but enabled for qa environment only (#3429)

- Ensure we handle both services which do and which do not autoscale
  - Scaling out at 5min intervals is likely sufficient based on past behavior of production
- We have data resolution at 1 minute so 5x resolution is responsive without being too fickle to spikes
- 60% as a target for web-service still gives us plenty of headroom for increase as we do not see any massive spikes in production
- Only enable on QA for now to evaluate configuration in non-prod environments

Author: TheOneFromNorway

terraform/app/autoscaling.tf

@@ -41,10 +41,20 @@ module "web_service" {
     target_group_arn = local.ecs_initial_lb_target_group
     container_port   = 4000
   }
+  autoscaling_policies = tomap({
+    cpu = {
+      predefined_metric_type = "ECSServiceAverageCPUUtilization"
+      target_value           = 60
+      scale_in_cooldown      = 600
+      scale_out_cooldown     = 300
+    }
+  })
   cluster_id            = aws_ecs_cluster.cluster.id
+  cluster_name          = aws_ecs_cluster.cluster.name
+  minimum_replica_count = var.minimum_web_replicas
+  maximum_replica_count = var.maximum_web_replicas
   environment           = var.environment
   server_type           = "web"
-  desired_count         = var.minimum_web_replicas
   deployment_controller = "CODE_DEPLOY"
 }
 
@@ -66,8 +76,10 @@ module "good_job_service" {
     subnets = [aws_subnet.private_subnet_a.id, aws_subnet.private_subnet_b.id]
     vpc_id  = aws_vpc.application_vpc.id
   }
-  cluster_id    = aws_ecs_cluster.cluster.id
-  environment   = var.environment
-  server_type   = "good-job"
-  desired_count = var.minimum_good_job_replicas
+  minimum_replica_count = var.good_job_replicas
+  maximum_replica_count = var.good_job_replicas
+  cluster_id            = aws_ecs_cluster.cluster.id
+  cluster_name          = aws_ecs_cluster.cluster.name
+  environment           = var.environment
+  server_type           = "good-job"
 }

terraform/app/ecs.tf

@@ -24,5 +24,4 @@ http_hosts = {
   MAVIS__GIVE_OR_REFUSE_CONSENT_HOST = "preview.mavistesting.com"
 }
 
-appspec_bucket         = "nhse-mavis-appspec-bucket-preview"
-active_lb_target_group = "green"
+appspec_bucket = "nhse-mavis-appspec-bucket-preview"

terraform/app/env/preview.tfvars

@@ -22,4 +22,6 @@ http_hosts = {
   MAVIS__HOST                        = "qa.mavistesting.com"
   MAVIS__GIVE_OR_REFUSE_CONSENT_HOST = "qa.mavistesting.com"
 }
-appspec_bucket = "nhse-mavis-appspec-bucket-qa"
+appspec_bucket       = "nhse-mavis-appspec-bucket-qa"
+minimum_web_replicas = 2
+maximum_web_replicas = 4

terraform/app/env/qa.tfvars

@@ -21,4 +21,6 @@ enable_cis2                     = false
 enable_pds_enqueue_bulk_updates = false
 
 appspec_bucket       = "nhse-mavis-appspec-bucket-sandbox-alpha"
-minimum_web_replicas = 2
+minimum_web_replicas = 1
+maximum_web_replicas = 2
+good_job_replicas    = 1

terraform/app/env/sandbox-alpha.tfvars

@@ -21,4 +21,6 @@ enable_cis2                     = false
 enable_pds_enqueue_bulk_updates = false
 
 appspec_bucket       = "nhse-mavis-appspec-bucket-sandbox-beta"
-minimum_web_replicas = 2
+minimum_web_replicas = 1
+maximum_web_replicas = 2
+good_job_replicas    = 1

terraform/app/env/sandbox-beta.tfvars

@@ -0,0 +1,27 @@
+resource "aws_appautoscaling_target" "this" {
+  count              = length(var.autoscaling_policies) > 0 && local.autoscaling_enabled ? 1 : 0
+  resource_id        = "service/${var.cluster_name}/${aws_ecs_service.this.name}"
+  max_capacity       = var.maximum_replica_count
+  min_capacity       = var.minimum_replica_count
+  service_namespace  = "ecs"
+  scalable_dimension = "ecs:service:DesiredCount"
+}
+
+
+resource "aws_appautoscaling_policy" "this" {
+  for_each           = local.autoscaling_enabled ? var.autoscaling_policies : {}
+  name               = "${var.server_type}-${each.key}-scaling-${var.environment}"
+  policy_type        = "TargetTrackingScaling"
+  resource_id        = aws_appautoscaling_target.this[0].resource_id
+  scalable_dimension = aws_appautoscaling_target.this[0].scalable_dimension
+  service_namespace  = aws_appautoscaling_target.this[0].service_namespace
+
+  target_tracking_scaling_policy_configuration {
+    predefined_metric_specification {
+      predefined_metric_type = each.value.predefined_metric_type
+    }
+    target_value       = each.value.target_value
+    scale_in_cooldown  = each.value.scale_in_cooldown
+    scale_out_cooldown = each.value.scale_out_cooldown
+  }
+}

terraform/app/modules/ecs_service/autoscaling.tf

@@ -30,7 +30,7 @@ resource "aws_ecs_service" "this" {
   name                              = "mavis-${var.environment}-${var.server_type}"
   cluster                           = var.cluster_id
   task_definition                   = aws_ecs_task_definition.this.arn
-  desired_count                     = var.desired_count
+  desired_count                     = var.minimum_replica_count
   launch_type                       = "FARGATE"
   enable_execute_command            = true
   health_check_grace_period_seconds = 60
@@ -63,7 +63,7 @@ resource "aws_ecs_service" "this" {
     ignore_changes = [
       task_definition,
       load_balancer,
-      # desired_count TODO: Uncomment once we include autoscaling
+      desired_count
     ]
     create_before_destroy = true
   }

terraform/app/modules/ecs_service/main.tf

@@ -10,12 +10,35 @@ variable "server_type" {
   nullable    = false
 }
 
-variable "desired_count" {
+variable "minimum_replica_count" {
   type        = number
-  description = "The initial amount of instances when creating the service"
+  description = "Minimum amount of allowed replicas for the service. Also the replica count when creating th service."
   nullable    = false
 }
 
+variable "maximum_replica_count" {
+  type        = number
+  description = "The maximum amount of instances by which the service can scale. If equal to the minimum_replica_count, autoscaling will be disabled."
+  nullable    = false
+  validation {
+    condition     = var.maximum_replica_count >= var.minimum_replica_count
+    error_message = "Maximum replica count must be greater than initial replica count when autoscaling policies are defined and null otherwise"
+  }
+}
+
+variable "autoscaling_policies" {
+  type = map(object({
+    predefined_metric_type = string
+    target_value           = number
+    scale_in_cooldown      = number
+    scale_out_cooldown     = number
+  }))
+  description = "List of autoscaling policy configuration parameters for the ECS service"
+  default     = {}
+  nullable    = false
+}
+
+
 variable "task_config" {
   type = object({
     environment = list(object({
@@ -45,6 +68,12 @@ variable "cluster_id" {
   nullable    = false
 }
 
+variable "cluster_name" {
+  type        = string
+  description = "The name of the ECS cluster."
+  nullable    = false
+}
+
 variable "network_params" {
   type = object({
     subnets = list(string)
@@ -77,3 +106,7 @@ variable "container_name" {
   default     = "application"
   nullable    = false
 }
+
+locals {
+  autoscaling_enabled = var.maximum_replica_count > var.minimum_replica_count
+}

terraform/app/modules/ecs_service/variables.tf

@@ -230,16 +230,11 @@ variable "backup_retention_period" {
 }
 
 ########## ESC/Scaling Configuration ##########
-variable "enable_autoscaling" {
-  type        = bool
-  default     = false
-  description = "Boolean toggle to determine whether the ECS service should have autoscaling enabled."
-}
 
 variable "minimum_web_replicas" {
   type        = number
   default     = 3
-  description = "Minimum amount of allowed replicas for web service"
+  description = "Minimum amount of allowed replicas for web service. Also the replica count when creating th service."
 }
 
 variable "maximum_web_replicas" {
@@ -248,10 +243,10 @@ variable "maximum_web_replicas" {
   description = "Maximum amount of allowed replicas for web service"
 }
 
-variable "minimum_good_job_replicas" {
+variable "good_job_replicas" {
   type        = number
   default     = 2
-  description = "Minimum amount of allowed replicas for good-job service"
+  description = "Amount of replicas for the good-job service"
 }
 
 variable "max_aurora_capacity_units" {