Merge pull request #3809 from nhsuk/create_valkey_config_for_sidekiq

TheOneFromNorway · web-flow · commit dadbc87be230 · 2025-08-15T17:02:54.000+01:00
Complete implementation of valkey with a connected sidekiq service
diff --git a/bin/docker-start b/bin/docker-start
@@ -8,7 +8,7 @@ if [ "$SERVER_TYPE" == "web" ]; then
 elif [ "$SERVER_TYPE" == "good-job" ]; then
   echo "Starting good-job server..."
   exec "$BIN_DIR"/good_job start
-elif [ "$SERVER_TYPE" == "none" ]; then
+elif [ "$SERVER_TYPE" == "none" ] || [ "$SERVER_TYPE" == "sidekiq" ]; then #TODO: Implement sidekiq in application
   echo "No server started"
   exec tail -f /dev/null  # Keep container running
 else
diff --git a/terraform/account/resources/iam_policy_DeployMavisResources.json b/terraform/account/resources/iam_policy_DeployMavisResources.json
@@ -92,6 +92,14 @@
         "iam:DetachRolePolicy",
         "kms:CreateGrant",
         "kms:Decrypt",
+        "logs:PutResourcePolicy",
+        "logs:DescribeResourcePolicies",
+        "logs:DescribeLogGroups",
+        "logs:CreateLogDelivery",
+        "logs:UpdateLogDelivery",
+        "logs:DeleteLogDelivery",
+        "logs:GetLogDelivery",
+        "logs:ListLogDeliveries",
         "logs:CreateLogGroup",
         "logs:DeleteLogGroup",
         "logs:PutRetentionPolicy",
@@ -133,7 +141,21 @@
         "secretsmanager:CancelRotateSecret",
         "ssm:DeleteParameter",
         "ssm:DeleteParameters",
-        "ssm:PutParameter"
+        "ssm:PutParameter",
+        "elasticache:CreateCacheParameterGroup",
+        "elasticache:AuthorizeCacheSecurityGroupIngress",
+        "elasticache:CreateReplicationGroup",
+        "elasticache:CreateCacheSubnetGroup",
+        "elasticache:DecreaseReplicaCount",
+        "elasticache:DeleteCacheCluster",
+        "elasticache:DeleteCacheParameterGroup",
+        "elasticache:DeleteCacheSubnetGroup",
+        "elasticache:DeleteReplicationGroup",
+        "elasticache:ModifyReplicationGroup",
+        "elasticache:ModifyCacheCluster",
+        "elasticache:ModifyCacheParameterGroup",
+        "elasticache:ModifyCacheSubnetGroup",
+        "elasticache:IncreaseReplicaCount"
       ],
       "Resource": ["*"]
     }
diff --git a/terraform/app/ecs.tf b/terraform/app/ecs.tf
@@ -83,3 +83,33 @@ module "good_job_service" {
   environment           = var.environment
   server_type           = "good-job"
 }
+
+module "sidekiq_service" {
+  source = "./modules/ecs_service"
+  task_config = {
+    environment          = local.task_envs
+    secrets              = local.task_secrets
+    cpu                  = 1024
+    memory               = 2048
+    docker_image         = "${var.account_id}.dkr.ecr.eu-west-2.amazonaws.com/${var.docker_image}@${var.image_digest}"
+    execution_role_arn   = aws_iam_role.ecs_task_execution_role.arn
+    task_role_arn        = aws_iam_role.ecs_task_role.arn
+    log_group_name       = aws_cloudwatch_log_group.ecs_log_group.name
+    region               = var.region
+    health_check_command = ["CMD-SHELL", "echo true || exit 1"]
+  }
+  network_params = {
+    subnets = [aws_subnet.private_subnet_a.id, aws_subnet.private_subnet_b.id]
+    vpc_id  = aws_vpc.application_vpc.id
+  }
+  minimum_replica_count = var.sidekiq_replicas
+  maximum_replica_count = var.sidekiq_replicas
+  cluster_id            = aws_ecs_cluster.cluster.id
+  cluster_name          = aws_ecs_cluster.cluster.name
+  environment           = var.environment
+  server_type           = "sidekiq"
+
+  depends_on = [
+    aws_elasticache_replication_group.valkey
+  ]
+}
diff --git a/terraform/app/env/preview.tfvars b/terraform/app/env/preview.tfvars
@@ -22,3 +22,7 @@ http_hosts = {
 appspec_bucket       = "nhse-mavis-appspec-bucket-preview"
 minimum_web_replicas = 2
 maximum_web_replicas = 4
+
+valkey_node_type          = "cache.t4g.micro"
+valkey_log_retention_days = 3
+valkey_failover_enabled   = false
diff --git a/terraform/app/env/sandbox-alpha.tfvars b/terraform/app/env/sandbox-alpha.tfvars
@@ -20,3 +20,8 @@ appspec_bucket       = "nhse-mavis-appspec-bucket-sandbox-alpha"
 minimum_web_replicas = 1
 maximum_web_replicas = 2
 good_job_replicas    = 1
+
+valkey_node_type          = "cache.t4g.micro"
+valkey_log_retention_days = 3
+valkey_failover_enabled   = false
+sidekiq_replicas          = 1
diff --git a/terraform/app/env/sandbox-beta.tfvars b/terraform/app/env/sandbox-beta.tfvars
@@ -20,3 +20,9 @@ appspec_bucket       = "nhse-mavis-appspec-bucket-sandbox-beta"
 minimum_web_replicas = 1
 maximum_web_replicas = 2
 good_job_replicas    = 1
+
+# Valkey serverless configuration - minimal settings for sandbox
+valkey_node_type          = "cache.t4g.micro"
+valkey_log_retention_days = 3
+valkey_failover_enabled   = false
+sidekiq_replicas          = 1
diff --git a/terraform/app/env/test.tfvars b/terraform/app/env/test.tfvars
@@ -20,3 +20,7 @@ http_hosts = {
 appspec_bucket       = "nhse-mavis-appspec-bucket-test"
 minimum_web_replicas = 2
 maximum_web_replicas = 4
+
+valkey_node_type          = "cache.t4g.micro"
+valkey_log_retention_days = 3
+valkey_failover_enabled   = false
diff --git a/terraform/app/env/training.tfvars b/terraform/app/env/training.tfvars
@@ -24,3 +24,7 @@ http_hosts = {
 appspec_bucket       = "nhse-mavis-appspec-bucket-training"
 minimum_web_replicas = 2
 maximum_web_replicas = 4
+
+valkey_node_type          = "cache.t4g.micro"
+valkey_log_retention_days = 3
+valkey_failover_enabled   = false
diff --git a/terraform/app/outputs.tf b/terraform/app/outputs.tf
@@ -30,8 +30,12 @@ output "ecs_variables" {
       service_name    = module.good_job_service.service.name
       task_definition = module.good_job_service.task_definition
     }
+    sidekiq = {
+      service_name    = module.sidekiq_service.service.name
+      task_definition = module.sidekiq_service.task_definition
+    }
   }
-  description = "Essential attributes of the ECS service"
+  description = "Essential attributes of the ECS services"
 }
 
 output "db_secret_arn" {
diff --git a/terraform/app/valkey.tf b/terraform/app/valkey.tf
@@ -0,0 +1,112 @@
+resource "aws_security_group" "valkey" {
+  name        = "mavis-cache-${var.environment}"
+  description = "Security group for Valkey ElastiCache (self-designed cluster)"
+  vpc_id      = aws_vpc.application_vpc.id
+
+  tags = {
+    Name = "mavis-cache-${var.environment}"
+  }
+
+  lifecycle {
+    ignore_changes = [description]
+  }
+}
+
+resource "aws_security_group_rule" "valkey_ecs_services_ingress" {
+  count                    = length(local.ecs_sg_ids)
+  type                     = "ingress"
+  from_port                = var.valkey_port
+  to_port                  = var.valkey_port
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.valkey.id
+  source_security_group_id = local.ecs_sg_ids[count.index]
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "aws_elasticache_subnet_group" "valkey" {
+  name       = "mavis-cache-subnet-group-${var.environment}"
+  subnet_ids = [aws_subnet.private_subnet_a.id, aws_subnet.private_subnet_b.id]
+
+  tags = {
+    Name = "mavis-cache-subnet-group-${var.environment}"
+  }
+}
+
+resource "aws_elasticache_replication_group" "valkey" {
+  replication_group_id = "mavis-cache-${var.environment}"
+  description          = "Valkey cluster for Sidekiq"
+
+  engine               = "valkey"
+  engine_version       = var.valkey_engine_version
+  node_type            = var.valkey_node_type
+  port                 = var.valkey_port
+  parameter_group_name = aws_elasticache_parameter_group.valkey.name
+
+  automatic_failover_enabled  = var.valkey_failover_enabled
+  num_cache_clusters          = length(local.valkey_cache_availability_zones)
+  subnet_group_name           = aws_elasticache_subnet_group.valkey.name
+  security_group_ids          = [aws_security_group.valkey.id]
+  preferred_cache_cluster_azs = local.valkey_cache_availability_zones
+  snapshot_retention_limit    = var.valkey_snapshot_retention_limit
+  snapshot_window             = var.valkey_snapshot_window
+  maintenance_window          = var.valkey_maintenance_window
+
+  at_rest_encryption_enabled = true
+  transit_encryption_enabled = true
+
+  log_delivery_configuration {
+    destination      = aws_cloudwatch_log_group.valkey_slow_log.name
+    destination_type = "cloudwatch-logs"
+    log_format       = "json"
+    log_type         = "slow-log"
+  }
+
+  log_delivery_configuration {
+    destination      = aws_cloudwatch_log_group.valkey_engine_log.name
+    destination_type = "cloudwatch-logs"
+    log_format       = "json"
+    log_type         = "engine-log"
+  }
+
+  tags = {
+    Name    = "mavis-cache-${var.environment}"
+    Purpose = "sidekiq-job-processing"
+  }
+  apply_immediately = true
+}
+
+resource "aws_elasticache_parameter_group" "valkey" {
+  family = "valkey8"
+  name   = "mavis-cache-params-${var.environment}"
+
+  # Optimize for Sidekiq workload
+  parameter {
+    name  = "maxmemory-policy"
+    value = "noeviction"
+  }
+
+  tags = {
+    Name = "mavis-cache-params-${var.environment}"
+  }
+}
+
+resource "aws_cloudwatch_log_group" "valkey_slow_log" {
+  name              = "/aws/elasticache/valkey/${var.environment}/slow-log"
+  retention_in_days = var.valkey_log_retention_days
+
+  tags = {
+    Name = "mavis-cache-slow-log-${var.environment}"
+  }
+}
+
+resource "aws_cloudwatch_log_group" "valkey_engine_log" {
+  name              = "/aws/elasticache/valkey/${var.environment}/engine-log"
+  retention_in_days = var.valkey_log_retention_days
+
+  tags = {
+    Name = "mavis-cache-engine-log-${var.environment}"
+  }
+}
diff --git a/terraform/app/variables.tf b/terraform/app/variables.tf
@@ -183,6 +183,7 @@ locals {
     MAVIS__ACADEMIC_YEAR_TODAY_OVERRIDE             = var.academic_year_today_override
     MAVIS__ACADEMIC_YEAR_NUMBER_OF_PREPARATION_DAYS = var.academic_year_number_of_preparation_days
     GOOD_JOB_MAX_THREADS                            = 5
+    SIDEKIQ_CONCURRENCY                             = 5
   })
   parameter_store_config_list = [for key, value in local.parameter_store_variables : {
     name      = key
@@ -226,7 +227,11 @@ locals {
     {
       name  = "APP_VERSION"
       value = var.app_version
-    }
+    },
+    {
+      name  = "SIDEKIQ_REDIS_URL"
+      value = "rediss://${aws_elasticache_replication_group.valkey.primary_endpoint_address}:${var.valkey_port}"
+    },
   ]
   task_secrets = concat([
     {
@@ -305,6 +310,12 @@ variable "good_job_replicas" {
   description = "Amount of replicas for the good-job service"
 }
 
+variable "sidekiq_replicas" {
+  type        = number
+  default     = 2
+  description = "Amount of replicas for the sidekiq service"
+}
+
 variable "max_aurora_capacity_units" {
   type        = number
   default     = 8
@@ -321,7 +332,78 @@ variable "active_lb_target_group" {
   }
 }
 
+########## Valkey Configuration ##########
+
+variable "valkey_port" {
+  type        = number
+  default     = 6379
+  description = "Port number for Valkey cluster"
+  nullable    = false
+  validation {
+    condition     = var.valkey_port > 0 && var.valkey_port <= 65535
+    error_message = "Port must be between 1 and 65535."
+  }
+}
+
+variable "valkey_engine_version" {
+  type        = string
+  default     = "8.0"
+  description = "Valkey engine version"
+  nullable    = false
+}
+
+variable "valkey_node_type" {
+  type        = string
+  default     = "cache.m7g.large"
+  description = "ElastiCache node type for Valkey (use cache.t3.micro for sandbox, cache.m7g.large+ for production)"
+  nullable    = false
+}
+
+variable "valkey_failover_enabled" {
+  type        = bool
+  default     = true
+  description = "Enable automatic failover for Valkey cluster"
+  nullable    = false
+}
+
+variable "valkey_snapshot_retention_limit" {
+  type        = number
+  default     = 7
+  description = "Number of days to retain Valkey snapshots"
+  nullable    = false
+  validation {
+    condition     = var.valkey_snapshot_retention_limit >= 0 && var.valkey_snapshot_retention_limit <= 35
+    error_message = "Snapshot retention must be between 0 and 35 days."
+  }
+}
+
+variable "valkey_snapshot_window" {
+  type        = string
+  default     = "00:00-02:00"
+  description = "Daily snapshot window for Valkey (HH:MM-HH:MM format, UTC)"
+  nullable    = false
+}
+
+variable "valkey_maintenance_window" {
+  type        = string
+  default     = "sun:02:00-sun:04:00"
+  description = "Weekly maintenance window for Valkey (ddd:HH:MM-ddd:HH:MM format, UTC)"
+  nullable    = false
+}
+
+variable "valkey_log_retention_days" {
+  type        = number
+  default     = 14
+  description = "Number of days to retain Valkey logs (minimum 3 for sandbox, 14+ for production)"
+  nullable    = false
+  validation {
+    condition     = var.valkey_log_retention_days >= 1 && var.valkey_log_retention_days <= 365
+    error_message = "Log retention must be between 1 and 365 days."
+  }
+}
+
 locals {
-  ecs_initial_lb_target_group = var.active_lb_target_group == "green" ? aws_lb_target_group.green.arn : aws_lb_target_group.blue.arn
-  ecs_sg_ids                  = [module.web_service.security_group_id, module.good_job_service.security_group_id]
+  ecs_initial_lb_target_group     = var.active_lb_target_group == "green" ? aws_lb_target_group.green.arn : aws_lb_target_group.blue.arn
+  ecs_sg_ids                      = [module.web_service.security_group_id, module.good_job_service.security_group_id, module.sidekiq_service.security_group_id]
+  valkey_cache_availability_zones = var.valkey_failover_enabled ? [aws_subnet.private_subnet_a.availability_zone, aws_subnet.private_subnet_b.availability_zone] : [aws_subnet.private_subnet_a.availability_zone]
 }
diff --git a/terraform/documentation/valkey-sidekiq-setup.md b/terraform/documentation/valkey-sidekiq-setup.md

Original file line number	Diff line number	Diff line change
`@@ -30,8 +30,12 @@ output "ecs_variables" {`
`30`	`30`	`service_name = module.good_job_service.service.name`
`31`	`31`	`task_definition = module.good_job_service.task_definition`
`32`	`32`	`}`
	`33`	`+ sidekiq = {`
	`34`	`+ service_name = module.sidekiq_service.service.name`
	`35`	`+ task_definition = module.sidekiq_service.task_definition`
	`36`	`+ }`
`33`	`37`	`}`
`34`		`- description = "Essential attributes of the ECS service"`
	`38`	`+ description = "Essential attributes of the ECS services"`
`35`	`39`	`}`
`36`	`40`
`37`	`41`	`output "db_secret_arn" {`