diff --git a/blueprints/data-solutions/composer-2/README.md b/blueprints/data-solutions/composer-2/README.md index 1c05ec6a7..e8c15eeca 100644 --- a/blueprints/data-solutions/composer-2/README.md +++ b/blueprints/data-solutions/composer-2/README.md @@ -1,6 +1,10 @@ # Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key -This blueprint creates a Private instance of [Cloud Composer version 2](https://cloud.google.com/composer/docs/composer-2/composer-versioning-overview) on a VPC with a dedicated service account. +This blueprint creates a Private instance of [Cloud Composer version 2](https://cloud.google.com/composer/docs/composer-2/composer-versioning-overview) on a VPC with a dedicated service account. Cloud Composer 2 is the new major verion for Cloud Composer that supports: + - environment autoscaling + - workloads configuration: CPU, memory, and storage parameters for Airflow workers, schedulers, web server, and database. + +Please consult the [documentation page](https://cloud.google.com/composer/docs/composer-2/composer-versioning-overview) for an exaustive comparison between Composer Version 1 and Version 2. The solution will use: - Cloud Composer @@ -23,20 +27,20 @@ If `project_create` is left to null, the identity performing the deployment need # Deployment Run Terraform init: -``` +```bash $ terraform init ``` Configure the Terraform variable in your terraform.tfvars file. You need to spefify at least the following variables: -``` +```tfvars project_id = "lcaggioni-sandbox" prefix = "lc" ``` You can run now: -``` +```bash $ terraform apply ``` @@ -48,7 +52,7 @@ You can now connect to your instance. As is often the case in real-world configurations, this blueprint accepts as input an existing [`Shared-VPC`](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. Example: -``` +```tfvars network_config = { host_project = "PROJECT" network_self_link = "projects/PROJECT/global/networks/VPC_NAME" @@ -75,7 +79,7 @@ In order to run the example and deploy Cloud Composer on a shared VPC the identi As is often the case in real-world configurations, this blueprint accepts as input an existing [`Cloud KMS keys`](https://cloud.google.com/kms/docs/cmek) via the `service_encryption_keys` variable. Example: -``` +```tfvars service_encryption_keys = { `europe/west1` = `projects/PROJECT/locations/REGION/keyRings/KR_NAME/cryptoKeys/KEY_NAME` } @@ -86,15 +90,14 @@ service_encryption_keys = { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [organization_domain](variables.tf#L51) | Organization domain. | string | ✓ | | -| [prefix](variables.tf#L56) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | string | ✓ | | -| [project_id](variables.tf#L70) | Project id, references existing project if `project_create` is null. | string | ✓ | | +| [prefix](variables.tf#L49) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | string | ✓ | | +| [project_id](variables.tf#L63) | Project id, references existing project if `project_create` is null. | string | ✓ | | | [composer_config](variables.tf#L17) | Composer environemnt configuration. | object({…}) | | {…} | -| [groups](variables.tf#L29) | User groups. | map(string) | | {…} | -| [network_config](variables.tf#L37) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | -| [project_create](variables.tf#L61) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | -| [region](variables.tf#L75) | Region where instances will be deployed. | string | | "europe-west1" | -| [service_encryption_keys](variables.tf#L81) | Cloud KMS keys to use to encrypt resources. Provide a key for each reagion in use. | map(string) | | null | +| [iam_groups_map](variables.tf#L29) | Map of Role => groups to be added on the project. Example: { \"roles/composer.admin\" = [\"group:gcp-data-engineers@example.com\"]}. | map(list(string)) | | null | +| [network_config](variables.tf#L35) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | +| [project_create](variables.tf#L54) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | +| [region](variables.tf#L68) | Region where instances will be deployed. | string | | "europe-west1" | +| [service_encryption_keys](variables.tf#L74) | Cloud KMS keys to use to encrypt resources. Provide a key for each reagion in use. | map(string) | | null | ## Outputs diff --git a/blueprints/data-solutions/composer-2/composer.tf b/blueprints/data-solutions/composer-2/composer.tf index dd26a4c1c..e22d91ff9 100644 --- a/blueprints/data-solutions/composer-2/composer.tf +++ b/blueprints/data-solutions/composer-2/composer.tf @@ -20,9 +20,6 @@ module "comp-sa" { prefix = var.prefix name = "cmp" display_name = "Composer service account" - iam = { - "roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers] - } } resource "google_composer_environment" "env" { @@ -81,12 +78,12 @@ resource "google_composer_environment" "env" { } dynamic "encryption_config" { for_each = ( - try(lookup(var.service_encryption_keys, var.region, null) != null, false) + try(var.service_encryption_keys[var.region], null) != null ? { 1 = 1 } : {} ) content { - kms_key_name = try(lookup(var.service_encryption_keys, var.region, null), null) + kms_key_name = try(var.service_encryption_keys[var.region], null) } } } diff --git a/blueprints/data-solutions/composer-2/main.tf b/blueprints/data-solutions/composer-2/main.tf index faf9e6dc5..e85562964 100644 --- a/blueprints/data-solutions/composer-2/main.tf +++ b/blueprints/data-solutions/composer-2/main.tf @@ -15,14 +15,13 @@ */ locals { - iam = { - "roles/composer.worker" = [ - module.comp-sa.iam_email - ] - "roles/composer.ServiceAgentV2Ext" = [ - "serviceAccount:${module.project.service_accounts.robots.composer}" - ] - } + iam = merge( + { + "roles/composer.worker" = [module.comp-sa.iam_email] + "roles/composer.ServiceAgentV2Ext" = ["serviceAccount:${module.project.service_accounts.robots.composer}"] + }, + var.iam_groups_map + ) _shared_vpc_bindings = { "roles/compute.networkUser" = [ @@ -69,12 +68,6 @@ locals { ? var.network_config.network_self_link : module.vpc.0.self_link ) - groups = { - for k, v in var.groups : k => "${v}@${var.organization_domain}" - } - groups_iam = { - for k, v in local.groups : k => "group:${v}" - } } module "project" { diff --git a/blueprints/data-solutions/composer-2/variables.tf b/blueprints/data-solutions/composer-2/variables.tf index 36ba2edf1..b4c8ed982 100644 --- a/blueprints/data-solutions/composer-2/variables.tf +++ b/blueprints/data-solutions/composer-2/variables.tf @@ -26,12 +26,10 @@ variable "composer_config" { } } -variable "groups" { - description = "User groups." - type = map(string) - default = { - data-engineers = "gcp-data-engineers" - } +variable "iam_groups_map" { + description = "Map of Role => groups to be added on the project. Example: { \"roles/composer.admin\" = [\"group:gcp-data-engineers@example.com\"]}." + type = map(list(string)) + default = null } variable "network_config" { @@ -48,11 +46,6 @@ variable "network_config" { default = null } -variable "organization_domain" { - description = "Organization domain." - type = string -} - variable "prefix" { description = "Unique prefix used for resource names. Not used for project if 'project_create' is null." type = string diff --git a/tests/blueprints/data_solutions/composer_2/fixture/main.tf b/tests/blueprints/data_solutions/composer_2/fixture/main.tf index 384a069a8..4b35e6f80 100644 --- a/tests/blueprints/data_solutions/composer_2/fixture/main.tf +++ b/tests/blueprints/data_solutions/composer_2/fixture/main.tf @@ -18,7 +18,6 @@ module "test" { source = "../../../../../blueprints/data-solutions/composer-2/" project_id = "project" - organization_domain = "example.com" project_create = { billing_account_id = "123456-123456-123456" parent = "folders/12345678" diff --git a/tests/blueprints/data_solutions/composer_2/test_plan.py b/tests/blueprints/data_solutions/composer_2/test_plan.py index 017d9979f..5f69f1850 100644 --- a/tests/blueprints/data_solutions/composer_2/test_plan.py +++ b/tests/blueprints/data_solutions/composer_2/test_plan.py @@ -16,4 +16,4 @@ def test_resources(e2e_plan_runner): "Test that plan works and the numbers of resources is as expected." modules, resources = e2e_plan_runner() assert len(modules) == 5 - assert len(resources) == 28 + assert len(resources) == 29