diff --git a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf index 3dc554173..a93bc2046 100644 --- a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf +++ b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf @@ -45,7 +45,7 @@ module "drop-project" { source = "../../../modules/project" parent = var.project_config.parent billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null + project_create = var.project_config.project_create prefix = local.use_projects ? null : var.prefix name = ( local.use_projects diff --git a/blueprints/data-solutions/data-platform-foundations/02-load.tf b/blueprints/data-solutions/data-platform-foundations/02-load.tf index e262e960f..6f3ceb63b 100644 --- a/blueprints/data-solutions/data-platform-foundations/02-load.tf +++ b/blueprints/data-solutions/data-platform-foundations/02-load.tf @@ -39,7 +39,7 @@ module "load-project" { source = "../../../modules/project" parent = var.project_config.parent billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null + project_create = var.project_config.project_create prefix = local.use_projects ? null : var.prefix name = ( local.use_projects diff --git a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf index 3412ab6bc..e8b5a8061 100644 --- a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf +++ b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf @@ -61,7 +61,7 @@ module "orch-project" { source = "../../../modules/project" parent = var.project_config.parent billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null + project_create = var.project_config.project_create prefix = local.use_projects ? null : var.prefix name = ( local.use_projects @@ -162,8 +162,8 @@ module "orch-cs-df-template" { project_id = module.orch-project.project_id prefix = var.prefix name = "orc-cs-df-template" - location = var.region - storage_class = "REGIONAL" + location = var.location + storage_class = "MULTI_REGIONAL" encryption_key = try(local.service_encryption_keys.storage, null) force_destroy = !var.deletion_protection } @@ -173,8 +173,8 @@ module "orch-cs-build-staging" { project_id = module.orch-project.project_id prefix = var.prefix name = "orc-cs-build-staging" - location = var.region - storage_class = "REGIONAL" + location = var.location + storage_class = "MULTI_REGIONAL" encryption_key = try(local.service_encryption_keys.storage, null) force_destroy = !var.deletion_protection } diff --git a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf index 164bc8630..7dfb2bdd7 100644 --- a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf +++ b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf @@ -40,7 +40,7 @@ module "transf-project" { source = "../../../modules/project" parent = var.project_config.parent billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null + project_create = var.project_config.project_create prefix = local.use_projects ? null : var.prefix name = ( local.use_projects diff --git a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf index 1765adec9..381f3789a 100644 --- a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf +++ b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf @@ -68,7 +68,7 @@ module "dwh-lnd-project" { source = "../../../modules/project" parent = var.project_config.parent billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null + project_create = var.project_config.project_create prefix = local.use_projects ? null : var.prefix name = ( local.use_projects @@ -88,7 +88,7 @@ module "dwh-cur-project" { source = "../../../modules/project" parent = var.project_config.parent billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null + project_create = var.project_config.project_create prefix = local.use_projects ? null : var.prefix name = ( local.use_projects @@ -108,7 +108,7 @@ module "dwh-conf-project" { source = "../../../modules/project" parent = var.project_config.parent billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null + project_create = var.project_config.project_create prefix = local.use_projects ? null : var.prefix name = ( local.use_projects diff --git a/blueprints/data-solutions/data-platform-foundations/06-common.tf b/blueprints/data-solutions/data-platform-foundations/06-common.tf index 16d976c06..992a6c03c 100644 --- a/blueprints/data-solutions/data-platform-foundations/06-common.tf +++ b/blueprints/data-solutions/data-platform-foundations/06-common.tf @@ -50,7 +50,7 @@ module "common-project" { source = "../../../modules/project" parent = var.project_config.parent billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null + project_create = var.project_config.project_create prefix = local.use_projects ? null : var.prefix name = ( local.use_projects diff --git a/blueprints/data-solutions/data-platform-foundations/07-exposure.tf b/blueprints/data-solutions/data-platform-foundations/07-exposure.tf index bf09c336f..ad4f39d1a 100644 --- a/blueprints/data-solutions/data-platform-foundations/07-exposure.tf +++ b/blueprints/data-solutions/data-platform-foundations/07-exposure.tf @@ -18,7 +18,7 @@ module "exp-project" { source = "../../../modules/project" parent = var.project_config.parent billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null + project_create = var.project_config.project_create prefix = local.use_projects ? null : var.prefix name = ( local.use_projects diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index c73b0a31b..1e06da7ee 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -281,17 +281,17 @@ terraform destroy |---|---|:---:|:---:|:---:| | [organization_domain](variables.tf#L166) | Organization domain. | string | ✓ | | | [prefix](variables.tf#L171) | Prefix used for resource names. | string | ✓ | | -| [project_config](variables.tf#L180) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | +| [project_config](variables.tf#L180) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | | [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {…} | | [data_catalog_tags](variables.tf#L106) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {…} | | [deletion_protection](variables.tf#L120) | Prevent Terraform from destroying data storage resources (storage buckets, GKE clusters, CloudSQL instances) in this blueprint. When this field is set in Terraform state, a terraform destroy or terraform apply that would delete data storage resources will fail. | bool | | false | | [groups](variables.tf#L127) | User groups. | map(string) | | {…} | | [location](variables.tf#L137) | Location used for multi-regional resources. | string | | "eu" | | [network_config](variables.tf#L143) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | -| [project_services](variables.tf#L214) | List of core services enabled on all projects. | list(string) | | […] | -| [project_suffix](variables.tf#L225) | Suffix used only for project ids. | string | | null | -| [region](variables.tf#L231) | Region used for regional resources. | string | | "europe-west1" | -| [service_encryption_keys](variables.tf#L237) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | +| [project_services](variables.tf#L215) | List of core services enabled on all projects. | list(string) | | […] | +| [project_suffix](variables.tf#L226) | Suffix used only for project ids. | string | | null | +| [region](variables.tf#L232) | Region used for regional resources. | string | | "europe-west1" | +| [service_encryption_keys](variables.tf#L238) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | ## Outputs diff --git a/blueprints/data-solutions/data-platform-foundations/main.tf b/blueprints/data-solutions/data-platform-foundations/main.tf index 66653c77d..bdf5c279b 100644 --- a/blueprints/data-solutions/data-platform-foundations/main.tf +++ b/blueprints/data-solutions/data-platform-foundations/main.tf @@ -74,7 +74,7 @@ locals { ] ]) : "${binding.role}-${binding.member}" => binding } - use_projects = var.project_config.billing_account_id == null + use_projects = !var.project_config.project_create use_shared_vpc = var.network_config != null } diff --git a/blueprints/data-solutions/data-platform-foundations/terraform.tfvars.sample b/blueprints/data-solutions/data-platform-foundations/terraform.tfvars.sample deleted file mode 100644 index 9eb74da4b..000000000 --- a/blueprints/data-solutions/data-platform-foundations/terraform.tfvars.sample +++ /dev/null @@ -1,6 +0,0 @@ -prefix = "dat-plat" -project_config = { - parent = "folders/1111111111" - billing_account_id = "1111111-2222222-33333333" -} -organization_domain = "domain.com" diff --git a/blueprints/data-solutions/data-platform-foundations/variables.tf b/blueprints/data-solutions/data-platform-foundations/variables.tf index e3e72950c..e98a9a264 100644 --- a/blueprints/data-solutions/data-platform-foundations/variables.tf +++ b/blueprints/data-solutions/data-platform-foundations/variables.tf @@ -181,6 +181,7 @@ variable "project_config" { description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format." type = object({ billing_account_id = optional(string, null) + project_create = optional(bool, true) parent = string project_ids = optional(object({ drop = string diff --git a/fast/stages/3-data-platform/dev/README.md b/fast/stages/3-data-platform/dev/README.md index 38bf2f773..51c80f419 100644 --- a/fast/stages/3-data-platform/dev/README.md +++ b/fast/stages/3-data-platform/dev/README.md @@ -185,23 +185,24 @@ You can find examples in the `[demo](../../../../blueprints/data-solutions/data- |---|---|:---:|:---:|:---:|:---:| | [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap | | [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap | -| [folder_ids](variables.tf#L108) | Folder to be used for the networking resources in folders/nnnn format. | object({…}) | ✓ | | 1-resman | -| [host_project_ids](variables.tf#L126) | Shared VPC project ids. | object({…}) | ✓ | | 2-networking | -| [organization](variables.tf#L156) | Organization details. | object({…}) | ✓ | | 00-globals | -| [prefix](variables.tf#L172) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals | -| [composer_config](variables.tf#L38) | Cloud Composer configuration options. | object({…}) | | {…} | | -| [data_catalog_tags](variables.tf#L87) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {…} | | -| [deletion_protection](variables.tf#L101) | Prevent Terraform from destroying data storage resources (storage buckets, GKE clusters, CloudSQL instances) in this blueprint. When this field is set in Terraform state, a terraform destroy or terraform apply that would delete data storage resources will fail. | bool | | true | | -| [groups-dp](variables.tf#L116) | Data Platform groups. | map(string) | | {…} | | -| [location](variables.tf#L134) | Location used for multi-regional resources. | string | | "eu" | | -| [network_config_composer](variables.tf#L140) | Network configurations to use for Composer. | object({…}) | | {…} | | -| [outputs_location](variables.tf#L166) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [project_services](variables.tf#L182) | List of core services enabled on all projects. | list(string) | | […] | | -| [project_suffix](variables.tf#L193) | Suffix used only for project ids. | string | | null | | -| [region](variables.tf#L199) | Region used for regional resources. | string | | "europe-west1" | | -| [service_encryption_keys](variables.tf#L205) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | -| [subnet_self_links](variables.tf#L217) | Shared VPC subnet self links. | object({…}) | | null | 2-networking | -| [vpc_self_links](variables.tf#L226) | Shared VPC self links. | object({…}) | | null | 2-networking | +| [folder_ids](variables.tf#L148) | Folder to be used for the networking resources in folders/nnnn format. | object({…}) | ✓ | | 1-resman | +| [host_project_ids](variables.tf#L166) | Shared VPC project ids. | object({…}) | ✓ | | 2-networking | +| [organization](variables.tf#L196) | Organization details. | object({…}) | ✓ | | 00-globals | +| [prefix](variables.tf#L212) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals | +| [composer_config](variables.tf#L38) | Cloud Composer config. | object({…}) | | {…} | | +| [data_catalog_tags](variables.tf#L127) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {…} | | +| [deletion_protection](variables.tf#L141) | Prevent Terraform from destroying data storage resources (storage buckets, GKE clusters, CloudSQL instances) in this blueprint. When this field is set in Terraform state, a terraform destroy or terraform apply that would delete data storage resources will fail. | bool | | true | | +| [groups_dp](variables.tf#L156) | Data Platform groups. | map(string) | | {…} | | +| [location](variables.tf#L174) | Location used for multi-regional resources. | string | | "eu" | | +| [network_config_composer](variables.tf#L180) | Network configurations to use for Composer. | object({…}) | | {…} | | +| [outputs_location](variables.tf#L206) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | +| [project_config](variables.tf#L222) | Provide projects configuration. | object({…}) | | {} | | +| [project_services](variables.tf#L242) | List of core services enabled on all projects. | list(string) | | […] | | +| [project_suffix](variables.tf#L253) | Suffix used only for project ids. | string | | null | | +| [region](variables.tf#L259) | Region used for regional resources. | string | | "europe-west1" | | +| [service_encryption_keys](variables.tf#L265) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | +| [subnet_self_links](variables.tf#L277) | Shared VPC subnet self links. | object({…}) | | null | 2-networking | +| [vpc_self_links](variables.tf#L286) | Shared VPC self links. | object({…}) | | null | 2-networking | ## Outputs diff --git a/fast/stages/3-data-platform/dev/main.tf b/fast/stages/3-data-platform/dev/main.tf index f536070ed..b27a575ff 100644 --- a/fast/stages/3-data-platform/dev/main.tf +++ b/fast/stages/3-data-platform/dev/main.tf @@ -23,9 +23,11 @@ module "data-platform" { data_catalog_tags = var.data_catalog_tags project_config = { billing_account_id = var.billing_account.id + project_create = var.project_config.project_create parent = var.folder_ids.data-platform-dev + project_ids = var.project_config.project_ids } - groups = var.groups-dp + groups = var.groups_dp location = var.location network_config = { host_project = var.host_project_ids.dev-spoke-0 diff --git a/fast/stages/3-data-platform/dev/variables.tf b/fast/stages/3-data-platform/dev/variables.tf index ac5ee4b35..512f94b01 100644 --- a/fast/stages/3-data-platform/dev/variables.tf +++ b/fast/stages/3-data-platform/dev/variables.tf @@ -36,51 +36,91 @@ variable "billing_account" { } variable "composer_config" { - description = "Cloud Composer configuration options." + description = "Cloud Composer config." type = object({ disable_deployment = optional(bool) - environment_size = string - software_config = object({ - airflow_config_overrides = optional(any) - pypi_packages = optional(any) - env_variables = optional(map(string)) - image_version = string - cloud_data_lineage_integration = optional(bool, true) - }) - workloads_config = object({ - scheduler = object( - { - cpu = number - memory_gb = number - storage_gb = number - count = number - } - ) - web_server = object( - { - cpu = number - memory_gb = number - storage_gb = number - } - ) - worker = object( - { - cpu = number - memory_gb = number - storage_gb = number - min_count = number - max_count = number - } - ) - }) + environment_size = optional(string, "ENVIRONMENT_SIZE_SMALL") + software_config = optional( + object({ + airflow_config_overrides = optional(any) + pypi_packages = optional(any) + env_variables = optional(map(string)) + image_version = string + cloud_data_lineage_integration = optional(bool, true) + }), + { image_version = "composer-2-airflow-2" } + ) + workloads_config = optional( + object({ + scheduler = optional( + object({ + cpu = number + memory_gb = number + storage_gb = number + count = number + }), + { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + count = 1 + } + ) + web_server = optional( + object({ + cpu = number + memory_gb = number + storage_gb = number + }), + { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + } + ) + worker = optional( + object({ + cpu = number + memory_gb = number + storage_gb = number + min_count = number + max_count = number + }), + { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + min_count = 1 + max_count = 3 + } + ) + })) }) default = { environment_size = "ENVIRONMENT_SIZE_SMALL" software_config = { - image_version = "composer-2-airflow-2" - cloud_data_lineage_integration = true + image_version = "composer-2-airflow-2" + } + workloads_config = { + scheduler = { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + count = 1 + } + web_server = { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + } + worker = { + cpu = 0.5 + memory_gb = 1.875 + storage_gb = 1 + min_count = 1 + max_count = 3 + } } - workloads_config = null } } @@ -113,7 +153,7 @@ variable "folder_ids" { }) } -variable "groups-dp" { +variable "groups_dp" { description = "Data Platform groups." type = map(string) default = { @@ -179,6 +219,26 @@ variable "prefix" { } } +variable "project_config" { + description = "Provide projects configuration." + type = object({ + project_create = optional(bool, true) + project_ids = optional(object({ + drop = string + load = string + orc = string + trf = string + dwh-lnd = string + dwh-cur = string + dwh-conf = string + common = string + exp = string + }) + ) + }) + default = {} +} + variable "project_services" { description = "List of core services enabled on all projects." type = list(string)