From 886734e1e97363bf2a97bcc3a5399196deb5ceec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Mon, 11 Dec 2023 12:54:49 +0100 Subject: [PATCH] Add trigger configuration for Composer (#1916) --- .../data-solutions/composer-2/README.md | 29 +++++----- .../data-solutions/composer-2/composer.tf | 8 ++- blueprints/data-solutions/composer-2/main.tf | 2 +- .../data-solutions/composer-2/outputs.tf | 2 +- .../data-solutions/composer-2/variables.tf | 55 ++++++++++--------- 5 files changed, 54 insertions(+), 42 deletions(-) diff --git a/blueprints/data-solutions/composer-2/README.md b/blueprints/data-solutions/composer-2/README.md index 6cf927b7b..511dde045 100644 --- a/blueprints/data-solutions/composer-2/README.md +++ b/blueprints/data-solutions/composer-2/README.md @@ -41,6 +41,7 @@ Configure the Terraform variable in your terraform.tfvars file. You need to spec ```tfvars project_id = "lcaggioni-sandbox" prefix = "lc" +region = "europe-west1" ``` You can run now: @@ -72,6 +73,7 @@ Example: network_config = { host_project = "PROJECT" network_self_link = "projects/PROJECT/global/networks/VPC_NAME" + region = "europe-west3" subnet_self_link = "projects/PROJECT/regions/REGION/subnetworks/VPC_NAME" composer_secondary_ranges = { pods = "pods" @@ -111,14 +113,14 @@ service_encryption_keys = { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [prefix](variables.tf#L83) | Prefix used for resource names. | string | ✓ | | -| [project_id](variables.tf#L101) | Project id, references existing project if `project_create` is null. | string | ✓ | | -| [composer_config](variables.tf#L17) | Composer environment configuration. It accepts only following attributes: `environment_size`, `software_config` and `workloads_config`. See [attribute reference](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/composer_environment#argument-reference---cloud-composer-2) for details on settings variables. | object({…}) | | {…} | -| [iam_bindings_additive](variables.tf#L58) | Map of Role => principal in IAM format (`group:foo@example.org`) to be added on the project. | map(list(string)) | | {} | -| [network_config](variables.tf#L65) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | -| [project_create](variables.tf#L92) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | -| [region](variables.tf#L106) | Reagion where instances will be deployed. | string | | "europe-west1" | -| [service_encryption_keys](variables.tf#L112) | Cloud KMS keys to use to encrypt resources. Provide a key for each reagion in use. | map(string) | | null | +| [prefix](variables.tf#L87) | Prefix used for resource names. | string | ✓ | | +| [project_id](variables.tf#L105) | Project id, references existing project if `project_create` is null. | string | ✓ | | +| [region](variables.tf#L110) | Region where instances will be deployed. | string | ✓ | | +| [composer_config](variables.tf#L17) | Composer environment configuration. It accepts only following attributes: `environment_size`, `software_config` and `workloads_config`. See [attribute reference](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/composer_environment#argument-reference---cloud-composer-2) for details on settings variables. | object({…}) | | {…} | +| [iam_bindings_additive](variables.tf#L62) | Map of Role => principal in IAM format (`group:foo@example.org`) to be added on the project. | map(list(string)) | | {} | +| [network_config](variables.tf#L69) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | +| [project_create](variables.tf#L96) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | +| [service_encryption_keys](variables.tf#L115) | Cloud KMS keys to use to encrypt resources. Provide a key for each region in use. | map(string) | | null | ## Outputs @@ -132,12 +134,13 @@ service_encryption_keys = { ```hcl module "test" { source = "./fabric/blueprints/data-solutions/composer-2/" - project_id = "project" + project_id = "composer" project_create = { - billing_account_id = "123456-123456-123456" - parent = "folders/12345678" + billing_account_id = var.billing_account_id + parent = var.folder_id } - prefix = "prefix" + prefix = var.prefix + region = var.region } -# tftest modules=5 resources=29 +# tftest modules=5 resources=29 e2e ``` diff --git a/blueprints/data-solutions/composer-2/composer.tf b/blueprints/data-solutions/composer-2/composer.tf index 1217e0d41..6e07de877 100644 --- a/blueprints/data-solutions/composer-2/composer.tf +++ b/blueprints/data-solutions/composer-2/composer.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -52,6 +52,12 @@ resource "google_composer_environment" "env" { storage_gb = try(var.composer_config.workloads_config.scheduler.storage_gb, null) count = try(var.composer_config.workloads_config.scheduler.count, null) } + triggerer { + cpu = try(var.composer_config.workloads_config.triggerer.cpu, null) + memory_gb = try(var.composer_config.workloads_config.triggerer.memory_gb, null) + count = try(var.composer_config.workloads_config.triggerer.count, null) + } + web_server { cpu = try(var.composer_config.workloads_config.web_server.cpu, null) memory_gb = try(var.composer_config.workloads_config.web_server.memory_gb, null) diff --git a/blueprints/data-solutions/composer-2/main.tf b/blueprints/data-solutions/composer-2/main.tf index 3f2f9e846..7dfe086b5 100644 --- a/blueprints/data-solutions/composer-2/main.tf +++ b/blueprints/data-solutions/composer-2/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/blueprints/data-solutions/composer-2/outputs.tf b/blueprints/data-solutions/composer-2/outputs.tf index 4e09a0498..2b1138a60 100644 --- a/blueprints/data-solutions/composer-2/outputs.tf +++ b/blueprints/data-solutions/composer-2/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/blueprints/data-solutions/composer-2/variables.tf b/blueprints/data-solutions/composer-2/variables.tf index c5d6ff890..5a8501277 100644 --- a/blueprints/data-solutions/composer-2/variables.tf +++ b/blueprints/data-solutions/composer-2/variables.tf @@ -17,41 +17,45 @@ variable "composer_config" { description = "Composer environment configuration. It accepts only following attributes: `environment_size`, `software_config` and `workloads_config`. See [attribute reference](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/composer_environment#argument-reference---cloud-composer-2) for details on settings variables." type = object({ - environment_size = string - software_config = any - workloads_config = object({ - scheduler = object( + environment_size = optional(string) + software_config = optional(any) + workloads_config = optional(object({ + scheduler = optional(object( { - cpu = number - memory_gb = number - storage_gb = number - count = number + count = optional(number) + cpu = optional(number) + memory_gb = optional(number) + storage_gb = optional(number) } - ) - web_server = object( + )) + triggerer = optional(object({ + count = number + cpu = number + memory_gb = number + })) + web_server = optional(object( { - cpu = number - memory_gb = number - storage_gb = number + cpu = optional(number) + memory_gb = optional(number) + storage_gb = optional(number) } - ) - worker = object( + )) + worker = optional(object( { - cpu = number - memory_gb = number - storage_gb = number - min_count = number - max_count = number + cpu = optional(number) + memory_gb = optional(number) + min_count = optional(number) + max_count = optional(number) + storage_gb = optional(number) } - ) - }) + )) + })) }) default = { environment_size = "ENVIRONMENT_SIZE_SMALL" software_config = { image_version = "composer-2-airflow-2" } - workloads_config = null } } @@ -104,13 +108,12 @@ variable "project_id" { } variable "region" { - description = "Reagion where instances will be deployed." + description = "Region where instances will be deployed." type = string - default = "europe-west1" } variable "service_encryption_keys" { - description = "Cloud KMS keys to use to encrypt resources. Provide a key for each reagion in use." + description = "Cloud KMS keys to use to encrypt resources. Provide a key for each region in use." type = map(string) default = null }