From 85b88e26c0381725c8f154c5fcc2920ae15cbafe Mon Sep 17 00:00:00 2001 From: paduin Date: Mon, 3 Nov 2025 16:51:49 +0100 Subject: [PATCH 1/9] Adding access --- CHANGELOG.md | 4 ++++ lf.tf | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a3298b..2c43744 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [7.12.5] - 2025-11-03 +### Changed +- Fixed LakeFormation `DATA_LOCATION_ACCESS` to `lf_catalog_producer_arns`. + ## [7.12.4] - 2025-10-24 ### Changed - Fixed `GlueStatsServiceRole` name to avoid name conflicts when there are multiple apiary instances in same account. diff --git a/lf.tf b/lf.tf index 8dd97b1..40cbb47 100644 --- a/lf.tf +++ b/lf.tf @@ -101,6 +101,17 @@ resource "aws_lakeformation_permissions" "hms_sys_loc_permissions" { } } +resource "aws_lakeformation_permissions" "producer_loc_permissions" { + count = var.disable_glue_db_init && var.create_lf_resource ? 1 : 0 + + principal = each.value.lf_catalog_producer_arns + permissions = ["DATA_LOCATION_ACCESS"] + + data_location { + arn = aws_lakeformation_resource.apiary_system_bucket[0].arn + } +} + locals { # Read clients catalog_client_schemas = [ From ddea2be43ccc6b7912b352688d299a87379fdfb7 Mon Sep 17 00:00:00 2001 From: paduin Date: Mon, 3 Nov 2025 18:34:15 +0100 Subject: [PATCH 2/9] Adding access --- lf.tf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lf.tf b/lf.tf index 40cbb47..2a87c93 100644 --- a/lf.tf +++ b/lf.tf @@ -102,13 +102,15 @@ resource "aws_lakeformation_permissions" "hms_sys_loc_permissions" { } resource "aws_lakeformation_permissions" "producer_loc_permissions" { - count = var.disable_glue_db_init && var.create_lf_resource ? 1 : 0 + for_each = var.disable_glue_db_init && var.create_lf_resource ? { + for schema in local.schemas_info : "${schema["schema_name"]}" => schema + } : {} principal = each.value.lf_catalog_producer_arns permissions = ["DATA_LOCATION_ACCESS"] data_location { - arn = aws_lakeformation_resource.apiary_system_bucket[0].arn + arn = aws_lakeformation_resource.apiary_data_bucket[each.key].arn } } From c785fcfbfb53ee1bfc62373bb0c48cef9da9823d Mon Sep 17 00:00:00 2001 From: paduin Date: Mon, 3 Nov 2025 20:07:32 +0100 Subject: [PATCH 3/9] Adding access --- lf.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lf.tf b/lf.tf index 2a87c93..5aea009 100644 --- a/lf.tf +++ b/lf.tf @@ -103,7 +103,7 @@ resource "aws_lakeformation_permissions" "hms_sys_loc_permissions" { resource "aws_lakeformation_permissions" "producer_loc_permissions" { for_each = var.disable_glue_db_init && var.create_lf_resource ? { - for schema in local.schemas_info : "${schema["schema_name"]}" => schema + for schema in local.catalog_producer_schemas : "${schema["schema_name"]}" => schema } : {} principal = each.value.lf_catalog_producer_arns From 030edbdd16154221b4486e4ba174defccab75ab1 Mon Sep 17 00:00:00 2001 From: paduin Date: Tue, 4 Nov 2025 09:25:35 +0100 Subject: [PATCH 4/9] Adding access --- lf.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lf.tf b/lf.tf index 5aea009..d15dae2 100644 --- a/lf.tf +++ b/lf.tf @@ -103,14 +103,14 @@ resource "aws_lakeformation_permissions" "hms_sys_loc_permissions" { resource "aws_lakeformation_permissions" "producer_loc_permissions" { for_each = var.disable_glue_db_init && var.create_lf_resource ? { - for schema in local.catalog_producer_schemas : "${schema["schema_name"]}" => schema + for schema in local.catalog_producer_schemas : "${schema["schema_name"]}-${schema["producer_arn"]}" => schema } : {} - principal = each.value.lf_catalog_producer_arns + principal = each.value.producer_arn permissions = ["DATA_LOCATION_ACCESS"] data_location { - arn = aws_lakeformation_resource.apiary_data_bucket[each.key].arn + arn = aws_lakeformation_resource.apiary_data_bucket[each.value.schema_name].arn } } From cef57c3b35cb1d7e5bf1c0a48a45c514dfd28e48 Mon Sep 17 00:00:00 2001 From: paduin Date: Tue, 4 Nov 2025 12:42:38 +0100 Subject: [PATCH 5/9] separated in own var --- VARIABLES.md | 2 ++ lf.tf | 10 ++++++++-- variables.tf | 6 ++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/VARIABLES.md b/VARIABLES.md index 19d46ac..b78d092 100644 --- a/VARIABLES.md +++ b/VARIABLES.md @@ -34,6 +34,8 @@ | lf\_readonly\_client\_arns | AWS IAM role ARNs granted describe and select permissions on all glue databases and tables using LakeFormation. | `list(string)` | `[]` | no | | lf\_catalog\_client\_arns | AWS IAM role ARNs granted describe permissions on all glue databases and tables using LakeFormation. | `list(string)` | `[]` | no | | lf\_customer\_accounts | AWS account IDs granted describe permissions on all glue databases using LakeFormation. | `list(string)` | `[]` | no | +| lf\_catalog\_producer\_arns | AWS IAM role ARNs granted ALL permissions on all glue databases and tables using LakeFormation. | `list(string)` | `[]` | no | +| lf\_catalog\_data\_location\_access\_producer\_arns | AWS IAM role ARNs granted `DATA_LOCATION_ACCESS` permissions on all database s3 locations using LakeFormation. NOTE this permission is not granted by `lf_catalog_producer_arns` | `list(string)` | `[]` | no | | dashboard\_namespace | k8s namespace to deploy grafana dashboard. | `string` | `"monitoring"` | no | | db\_apply\_immediately | Specifies whether any cluster modifications are applied immediately, or during the next maintenance window. | `bool` | `false` | no | | db\_backup\_retention | The number of days to retain backups for the RDS Metastore DB. | `string` | `"7"` | yes | diff --git a/lf.tf b/lf.tf index d15dae2..a859986 100644 --- a/lf.tf +++ b/lf.tf @@ -101,9 +101,9 @@ resource "aws_lakeformation_permissions" "hms_sys_loc_permissions" { } } -resource "aws_lakeformation_permissions" "producer_loc_permissions" { +resource "aws_lakeformation_permissions" "data_location_access_permissions" { for_each = var.disable_glue_db_init && var.create_lf_resource ? { - for schema in local.catalog_producer_schemas : "${schema["schema_name"]}-${schema["producer_arn"]}" => schema + for schema in local.catalog_data_location_access_producer_schemas : "${schema["schema_name"]}-${schema["producer_arn"]}" => schema } : {} principal = each.value.producer_arn @@ -142,6 +142,12 @@ locals { producer_arn = pair[1] } ] + catalog_data_location_access_producer_schemas = [ + for pair in setproduct(local.schemas_info[*]["schema_name"], var.lf_catalog_data_location_access_producer_arns) : { + schema_name = pair[0] + producer_arn = pair[1] + } + ] } resource "aws_lakeformation_permissions" "catalog_client_permissions" { diff --git a/variables.tf b/variables.tf index adc1dac..156a8df 100644 --- a/variables.tf +++ b/variables.tf @@ -627,6 +627,12 @@ variable "lf_catalog_producer_arns" { default = [] } +variable lf_catalog_data_location_access_producer_arns { + description = "AWS IAM role ARNs granted `DATA_LOCATION_ACCESS` permissions on all database s3 locations using LakeFormation. NOTE this permission is not granted by `lf_catalog_producer_arns`" + type = list(string) + default = [] +} + variable "lf_catalog_glue_sync_arn" { description = "AWS IAM role ARN for glue sync to update table metadata. If empty, aws_iam_role.apiary_hms_readwrite.arn will be used." type = string From 35d143a4d7bc62659bf808743a96b19c0525cfd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Beltr=C3=A1n?= <36443689+javsanbel2@users.noreply.github.com> Date: Tue, 4 Nov 2025 13:11:57 +0100 Subject: [PATCH 6/9] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c43744..6e97927 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [7.12.5] - 2025-11-03 +## [7.12.5] - 2025-11-04 ### Changed - Fixed LakeFormation `DATA_LOCATION_ACCESS` to `lf_catalog_producer_arns`. From 098dd2b07da78fcdebf1d6362f5b56b603ae0e55 Mon Sep 17 00:00:00 2001 From: paduin Date: Tue, 4 Nov 2025 16:06:57 +0100 Subject: [PATCH 7/9] Clarifying changelog description --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e97927..04f4249 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [7.12.5] - 2025-11-04 -### Changed -- Fixed LakeFormation `DATA_LOCATION_ACCESS` to `lf_catalog_producer_arns`. +### Added +- Added var `lf_catalog_data_location_access_producer_arns` that is used to give `DATA_LOCATION_ACCESS` LakeFormation on s3 locations of all databases. ## [7.12.4] - 2025-10-24 ### Changed From 397b57029cbba47784ca22c7b98f680755c4d767 Mon Sep 17 00:00:00 2001 From: paduin Date: Tue, 4 Nov 2025 16:10:03 +0100 Subject: [PATCH 8/9] Clarifying changelog description --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04f4249..6e6f80d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [7.12.5] - 2025-11-04 ### Added -- Added var `lf_catalog_data_location_access_producer_arns` that is used to give `DATA_LOCATION_ACCESS` LakeFormation on s3 locations of all databases. +- Added var `lf_catalog_data_location_access_producer_arns` that is used to give `DATA_LOCATION_ACCESS` permission in LakeFormation on s3 locations of all databases. ## [7.12.4] - 2025-10-24 ### Changed From 37f284d6d9bb3009c9a0d1d4fbcbb08781e3e735 Mon Sep 17 00:00:00 2001 From: Patrick Duin Date: Wed, 12 Nov 2025 09:54:18 +0100 Subject: [PATCH 9/9] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e6f80d..e3666ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [7.12.5] - 2025-11-04 +## [7.12.5] - 2025-11-12 ### Added - Added var `lf_catalog_data_location_access_producer_arns` that is used to give `DATA_LOCATION_ACCESS` permission in LakeFormation on s3 locations of all databases.