From 2aa91465c34ab6d42159ead33a22b631a93df5b5 Mon Sep 17 00:00:00 2001 From: Julian Kuypers Date: Mon, 12 Feb 2024 12:13:09 +0000 Subject: [PATCH 1/5] add variable for ECS task ephemeral_storage --- main.tf | 37 ++++++++++++++------------- modules/metadata-service/ecs.tf | 1 + modules/metadata-service/variables.tf | 6 +++++ variables.tf | 6 +++++ 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/main.tf b/main.tf index 1a0cf31..9fcf00b 100644 --- a/main.tf +++ b/main.tf @@ -23,24 +23,25 @@ module "metaflow-metadata-service" { resource_prefix = local.resource_prefix resource_suffix = local.resource_suffix - access_list_cidr_blocks = var.access_list_cidr_blocks - database_name = module.metaflow-datastore.database_name - database_password = module.metaflow-datastore.database_password - database_username = module.metaflow-datastore.database_username - db_migrate_lambda_zip_file = var.db_migrate_lambda_zip_file - datastore_s3_bucket_kms_key_arn = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn - enable_api_basic_auth = var.metadata_service_enable_api_basic_auth - enable_api_gateway = var.metadata_service_enable_api_gateway - fargate_execution_role_arn = module.metaflow-computation.ecs_execution_role_arn - iam_partition = var.iam_partition - metadata_service_container_image = local.metadata_service_container_image - metaflow_vpc_id = var.vpc_id - rds_master_instance_endpoint = module.metaflow-datastore.rds_master_instance_endpoint - s3_bucket_arn = module.metaflow-datastore.s3_bucket_arn - subnet1_id = var.subnet1_id - subnet2_id = var.subnet2_id - vpc_cidr_blocks = var.vpc_cidr_blocks - with_public_ip = var.with_public_ip + access_list_cidr_blocks = var.access_list_cidr_blocks + database_name = module.metaflow-datastore.database_name + database_password = module.metaflow-datastore.database_password + database_username = module.metaflow-datastore.database_username + db_migrate_lambda_zip_file = var.db_migrate_lambda_zip_file + datastore_s3_bucket_kms_key_arn = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn + enable_api_basic_auth = var.metadata_service_enable_api_basic_auth + enable_api_gateway = var.metadata_service_enable_api_gateway + fargate_execution_role_arn = module.metaflow-computation.ecs_execution_role_arn + iam_partition = var.iam_partition + metadata_service_container_image = local.metadata_service_container_image + metadata_serviceephemeral_storage = var.metadata_service_ephemeral_storage + metaflow_vpc_id = var.vpc_id + rds_master_instance_endpoint = module.metaflow-datastore.rds_master_instance_endpoint + s3_bucket_arn = module.metaflow-datastore.s3_bucket_arn + subnet1_id = var.subnet1_id + subnet2_id = var.subnet2_id + vpc_cidr_blocks = var.vpc_cidr_blocks + with_public_ip = var.with_public_ip standard_tags = var.tags } diff --git a/modules/metadata-service/ecs.tf b/modules/metadata-service/ecs.tf index 1abd89d..71868ac 100644 --- a/modules/metadata-service/ecs.tf +++ b/modules/metadata-service/ecs.tf @@ -56,6 +56,7 @@ EOF execution_role_arn = var.fargate_execution_role_arn cpu = var.metadata_service_cpu memory = var.metadata_service_memory + ephemeral_storage = var.metadata_service_ephemeral_storage tags = merge( var.standard_tags, diff --git a/modules/metadata-service/variables.tf b/modules/metadata-service/variables.tf index b38f99c..a4e0117 100644 --- a/modules/metadata-service/variables.tf +++ b/modules/metadata-service/variables.tf @@ -78,6 +78,12 @@ variable "metadata_service_memory" { description = "ECS task memory in MiB for metadata service" } +variable "metadata_service_ephemeral_storage" { + type = number + default = 21 + description = "The Gb amount of disk storage to set for the ECS task [21-200]" +} + variable "metaflow_vpc_id" { type = string description = "ID of the Metaflow VPC this SageMaker notebook instance is to be deployed in" diff --git a/variables.tf b/variables.tf index d05c5a6..f1cdeca 100644 --- a/variables.tf +++ b/variables.tf @@ -108,6 +108,12 @@ variable "metadata_service_container_image" { description = "Container image for metadata service" } +variable "metadata_service_ephemeral_storage" { + type = number + default = 21 + description = "The Gb amount of disk storage to set for the ECS task [21-200]" +} + variable "metadata_service_enable_api_basic_auth" { type = bool default = true From 2a0df44b22fe804f3e25c2fc73a63b1a2c4395f7 Mon Sep 17 00:00:00 2001 From: Julian Kuypers Date: Mon, 12 Feb 2024 12:17:03 +0000 Subject: [PATCH 2/5] fix typo + format --- main.tf | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/main.tf b/main.tf index 9fcf00b..88a6df9 100644 --- a/main.tf +++ b/main.tf @@ -23,25 +23,25 @@ module "metaflow-metadata-service" { resource_prefix = local.resource_prefix resource_suffix = local.resource_suffix - access_list_cidr_blocks = var.access_list_cidr_blocks - database_name = module.metaflow-datastore.database_name - database_password = module.metaflow-datastore.database_password - database_username = module.metaflow-datastore.database_username - db_migrate_lambda_zip_file = var.db_migrate_lambda_zip_file - datastore_s3_bucket_kms_key_arn = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn - enable_api_basic_auth = var.metadata_service_enable_api_basic_auth - enable_api_gateway = var.metadata_service_enable_api_gateway - fargate_execution_role_arn = module.metaflow-computation.ecs_execution_role_arn - iam_partition = var.iam_partition - metadata_service_container_image = local.metadata_service_container_image - metadata_serviceephemeral_storage = var.metadata_service_ephemeral_storage - metaflow_vpc_id = var.vpc_id - rds_master_instance_endpoint = module.metaflow-datastore.rds_master_instance_endpoint - s3_bucket_arn = module.metaflow-datastore.s3_bucket_arn - subnet1_id = var.subnet1_id - subnet2_id = var.subnet2_id - vpc_cidr_blocks = var.vpc_cidr_blocks - with_public_ip = var.with_public_ip + access_list_cidr_blocks = var.access_list_cidr_blocks + database_name = module.metaflow-datastore.database_name + database_password = module.metaflow-datastore.database_password + database_username = module.metaflow-datastore.database_username + db_migrate_lambda_zip_file = var.db_migrate_lambda_zip_file + datastore_s3_bucket_kms_key_arn = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn + enable_api_basic_auth = var.metadata_service_enable_api_basic_auth + enable_api_gateway = var.metadata_service_enable_api_gateway + fargate_execution_role_arn = module.metaflow-computation.ecs_execution_role_arn + iam_partition = var.iam_partition + metadata_service_container_image = local.metadata_service_container_image + metadata_service_ephemeral_storage = var.metadata_service_ephemeral_storage + metaflow_vpc_id = var.vpc_id + rds_master_instance_endpoint = module.metaflow-datastore.rds_master_instance_endpoint + s3_bucket_arn = module.metaflow-datastore.s3_bucket_arn + subnet1_id = var.subnet1_id + subnet2_id = var.subnet2_id + vpc_cidr_blocks = var.vpc_cidr_blocks + with_public_ip = var.with_public_ip standard_tags = var.tags } From ba59eae87242dcc27373e3bf9ddf6153c2b0269d Mon Sep 17 00:00:00 2001 From: Julian Kuypers Date: Mon, 12 Feb 2024 20:26:08 +0000 Subject: [PATCH 3/5] add README definitions --- README.md | 1 + modules/metadata-service/README.md | 2 ++ 2 files changed, 3 insertions(+) diff --git a/README.md b/README.md index 2cce4f8..73c2a4e 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,7 @@ resource "local_file" "metaflow_config" { | [launch\_template\_http\_put\_response\_hop\_limit](#input\_launch\_template\_http\_put\_response\_hop\_limit) | The desired HTTP PUT response hop limit for instance metadata requests. Can be an integer from 1 to 64 | `number` | `2` | no | | [launch\_template\_http\_tokens](#input\_launch\_template\_http\_tokens) | Whether or not the metadata service requires session tokens, also referred to as Instance Metadata Service Version 2 (IMDSv2). Can be 'optional' or 'required' | `string` | `"optional"` | no | | [metadata\_service\_container\_image](#input\_metadata\_service\_container\_image) | Container image for metadata service | `string` | `""` | no | +| [metadata\_service\_ephemeral\_storage](#metadata\_service\_ephemeral\_storage) | Disk space for disk ECS task [21-200] | `number` | `21` | no | | [metadata\_service\_enable\_api\_basic\_auth](#input\_metadata\_service\_enable\_api\_basic\_auth) | Enable basic auth for API Gateway? (requires key export) | `bool` | `true` | no | | [metadata\_service\_enable\_api\_gateway](#input\_metadata\_service\_enable\_api\_gateway) | Enable API Gateway for public metadata service endpoint | `bool` | `true` | no | | [resource\_prefix](#input\_resource\_prefix) | string prefix for all resources | `string` | `"metaflow"` | no | diff --git a/modules/metadata-service/README.md b/modules/metadata-service/README.md index cbed1ef..a411a27 100644 --- a/modules/metadata-service/README.md +++ b/modules/metadata-service/README.md @@ -29,6 +29,8 @@ If the `access_list_cidr_blocks` variable is set, only traffic originating from | [metadata\_service\_container\_image](#input\_metadata\_service\_container\_image) | Container image for metadata service | `string` | `""` | no | | [metadata\_service\_cpu](#input\_metadata\_service\_cpu) | ECS task CPU unit for metadata service | `number` | `512` | no | | [metadata\_service\_memory](#input\_metadata\_service\_memory) | ECS task memory in MiB for metadata service | `number` | `1024` | no | +[metadata\_service\_ephemeral\_storage](#metadata\_service\_ephemeral\_storage) | Disk space for disk ECS task [21-200] | `number` | `21` | no | +| | [metaflow\_vpc\_id](#input\_metaflow\_vpc\_id) | ID of the Metaflow VPC this SageMaker notebook instance is to be deployed in | `string` | n/a | yes | | [rds\_master\_instance\_endpoint](#input\_rds\_master\_instance\_endpoint) | The database connection endpoint in address:port format | `string` | n/a | yes | | [resource\_prefix](#input\_resource\_prefix) | Prefix given to all AWS resources to differentiate between applications | `string` | n/a | yes | From 5de55dfa58f995c7f0b83df92eeab881ed6348d1 Mon Sep 17 00:00:00 2001 From: Julian Kuypers Date: Wed, 14 Feb 2024 15:25:04 +0000 Subject: [PATCH 4/5] set ephemeral storage as block + correct readme --- README.md | 2 +- modules/metadata-service/README.md | 3 +-- modules/metadata-service/ecs.tf | 4 +++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 73c2a4e..d30d848 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ resource "local_file" "metaflow_config" { | [launch\_template\_http\_put\_response\_hop\_limit](#input\_launch\_template\_http\_put\_response\_hop\_limit) | The desired HTTP PUT response hop limit for instance metadata requests. Can be an integer from 1 to 64 | `number` | `2` | no | | [launch\_template\_http\_tokens](#input\_launch\_template\_http\_tokens) | Whether or not the metadata service requires session tokens, also referred to as Instance Metadata Service Version 2 (IMDSv2). Can be 'optional' or 'required' | `string` | `"optional"` | no | | [metadata\_service\_container\_image](#input\_metadata\_service\_container\_image) | Container image for metadata service | `string` | `""` | no | -| [metadata\_service\_ephemeral\_storage](#metadata\_service\_ephemeral\_storage) | Disk space for disk ECS task [21-200] | `number` | `21` | no | +| [metadata\_service\_ephemeral\_storage](#input\_metadata\_service\_ephemeral\_storage) | Disk space for disk ECS task [21-200] | `number` | `21` | no | | [metadata\_service\_enable\_api\_basic\_auth](#input\_metadata\_service\_enable\_api\_basic\_auth) | Enable basic auth for API Gateway? (requires key export) | `bool` | `true` | no | | [metadata\_service\_enable\_api\_gateway](#input\_metadata\_service\_enable\_api\_gateway) | Enable API Gateway for public metadata service endpoint | `bool` | `true` | no | | [resource\_prefix](#input\_resource\_prefix) | string prefix for all resources | `string` | `"metaflow"` | no | diff --git a/modules/metadata-service/README.md b/modules/metadata-service/README.md index a411a27..991c5c7 100644 --- a/modules/metadata-service/README.md +++ b/modules/metadata-service/README.md @@ -29,8 +29,7 @@ If the `access_list_cidr_blocks` variable is set, only traffic originating from | [metadata\_service\_container\_image](#input\_metadata\_service\_container\_image) | Container image for metadata service | `string` | `""` | no | | [metadata\_service\_cpu](#input\_metadata\_service\_cpu) | ECS task CPU unit for metadata service | `number` | `512` | no | | [metadata\_service\_memory](#input\_metadata\_service\_memory) | ECS task memory in MiB for metadata service | `number` | `1024` | no | -[metadata\_service\_ephemeral\_storage](#metadata\_service\_ephemeral\_storage) | Disk space for disk ECS task [21-200] | `number` | `21` | no | -| +| [metadata\_service\_ephemeral\_storage](#input\_metadata\_service\_ephemeral\_storage) | Disk space for disk ECS task [21-200] | `number` | `21` | no | | [metaflow\_vpc\_id](#input\_metaflow\_vpc\_id) | ID of the Metaflow VPC this SageMaker notebook instance is to be deployed in | `string` | n/a | yes | | [rds\_master\_instance\_endpoint](#input\_rds\_master\_instance\_endpoint) | The database connection endpoint in address:port format | `string` | n/a | yes | | [resource\_prefix](#input\_resource\_prefix) | Prefix given to all AWS resources to differentiate between applications | `string` | n/a | yes | diff --git a/modules/metadata-service/ecs.tf b/modules/metadata-service/ecs.tf index 71868ac..bbb495f 100644 --- a/modules/metadata-service/ecs.tf +++ b/modules/metadata-service/ecs.tf @@ -56,7 +56,9 @@ EOF execution_role_arn = var.fargate_execution_role_arn cpu = var.metadata_service_cpu memory = var.metadata_service_memory - ephemeral_storage = var.metadata_service_ephemeral_storage + ephemeral_storage { + size_in_gib = var.metadata_service_ephemeral_storage + } tags = merge( var.standard_tags, From 56be04a6e932c23d20341972a0aa13ec6614db09 Mon Sep 17 00:00:00 2001 From: Julian Kuypers Date: Fri, 16 Feb 2024 17:13:17 +0000 Subject: [PATCH 5/5] fix docs format --- README.md | 2 +- modules/metadata-service/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d30d848..d725c4a 100644 --- a/README.md +++ b/README.md @@ -121,9 +121,9 @@ resource "local_file" "metaflow_config" { | [launch\_template\_http\_put\_response\_hop\_limit](#input\_launch\_template\_http\_put\_response\_hop\_limit) | The desired HTTP PUT response hop limit for instance metadata requests. Can be an integer from 1 to 64 | `number` | `2` | no | | [launch\_template\_http\_tokens](#input\_launch\_template\_http\_tokens) | Whether or not the metadata service requires session tokens, also referred to as Instance Metadata Service Version 2 (IMDSv2). Can be 'optional' or 'required' | `string` | `"optional"` | no | | [metadata\_service\_container\_image](#input\_metadata\_service\_container\_image) | Container image for metadata service | `string` | `""` | no | -| [metadata\_service\_ephemeral\_storage](#input\_metadata\_service\_ephemeral\_storage) | Disk space for disk ECS task [21-200] | `number` | `21` | no | | [metadata\_service\_enable\_api\_basic\_auth](#input\_metadata\_service\_enable\_api\_basic\_auth) | Enable basic auth for API Gateway? (requires key export) | `bool` | `true` | no | | [metadata\_service\_enable\_api\_gateway](#input\_metadata\_service\_enable\_api\_gateway) | Enable API Gateway for public metadata service endpoint | `bool` | `true` | no | +| [metadata\_service\_ephemeral\_storage](#input\_metadata\_service\_ephemeral\_storage) | The Gb amount of disk storage to set for the ECS task [21-200] | `number` | `21` | no | | [resource\_prefix](#input\_resource\_prefix) | string prefix for all resources | `string` | `"metaflow"` | no | | [resource\_suffix](#input\_resource\_suffix) | string suffix for all resources | `string` | `""` | no | | [subnet1\_id](#input\_subnet1\_id) | First subnet used for availability zone redundancy | `string` | n/a | yes | diff --git a/modules/metadata-service/README.md b/modules/metadata-service/README.md index 991c5c7..d9a8021 100644 --- a/modules/metadata-service/README.md +++ b/modules/metadata-service/README.md @@ -53,4 +53,4 @@ If the `access_list_cidr_blocks` variable is set, only traffic originating from | [metadata\_svc\_ecs\_task\_role\_arn](#output\_metadata\_svc\_ecs\_task\_role\_arn) | This role is passed to AWS ECS' task definition as the `task_role`. This allows the running of the Metaflow Metadata Service to have the proper permissions to speak to other AWS resources. | | [migration\_function\_arn](#output\_migration\_function\_arn) | ARN of DB Migration Function | | [network\_load\_balancer\_dns\_name](#output\_network\_load\_balancer\_dns\_name) | The DNS addressable name for the Network Load Balancer that accepts requests and forwards them to our Fargate MetaData service instance(s) | - + \ No newline at end of file