diff --git a/examples/data-solutions/data-platform-foundations/02-load.tf b/examples/data-solutions/data-platform-foundations/02-load.tf index 927400e69..a9d8b5c75 100644 --- a/examples/data-solutions/data-platform-foundations/02-load.tf +++ b/examples/data-solutions/data-platform-foundations/02-load.tf @@ -83,13 +83,13 @@ module "lod-prj" { } module "lod-vpc" { - count = var.network_config.network != null ? 0 : 1 + count = var.network_config.network_self_link != null ? 0 : 1 source = "../../../modules/net-vpc" project_id = module.lod-prj.project_id name = "${local.prefix_lod}-vpc" subnets = [ { - ip_cidr_range = var.network_config.vpc_subnet.load.range + ip_cidr_range = "10.10.0.0/24" name = "${local.prefix_lod}-subnet" region = var.location_config.region secondary_ip_range = {} @@ -98,18 +98,18 @@ module "lod-vpc" { } module "lod-vpc-firewall" { - count = var.network_config.network != null ? 0 : 1 + count = var.network_config.network_self_link != null ? 0 : 1 source = "../../../modules/net-vpc-firewall" project_id = module.lod-prj.project_id network = local._networks.load.network_name - admin_ranges = [local._networks.load.subnet_range] + admin_ranges = ["10.10.0.0/24"] } module "lod-nat" { - count = var.network_config.enable_cloud_nat ? 1 : 0 + count = var.network_config.network_self_link != null ? 0 : 1 source = "../../../modules/net-cloudnat" project_id = module.lod-prj.project_id region = var.location_config.region name = "${local.prefix_lod}-default" - router_network = local._networks.load.network_name + router_network = module.lod-vpc[0].name } diff --git a/examples/data-solutions/data-platform-foundations/03-composer.tf b/examples/data-solutions/data-platform-foundations/03-composer.tf index 667f3f684..c5e17d7c6 100644 --- a/examples/data-solutions/data-platform-foundations/03-composer.tf +++ b/examples/data-solutions/data-platform-foundations/03-composer.tf @@ -31,63 +31,65 @@ module "orc-sa-cmp-0" { resource "google_composer_environment" "orc-cmp-0" { name = "${local.prefix_orc}-cmp-0" - region = var.composer_config.region + region = var.location_config.region provider = google-beta project = module.orc-prj.project_id config { node_count = var.composer_config.node_count node_config { - zone = "${var.composer_config.region}-b" + zone = "${var.location_config.region}-b" service_account = module.orc-sa-cmp-0.email network = local._networks.orchestration.network subnetwork = local._networks.orchestration.subnet tags = ["composer-worker", "http-server", "https-server"] ip_allocation_policy { - use_ip_aliases = var.composer_config.ip_allocation_policy.use_ip_aliases - cluster_secondary_range_name = var.composer_config.ip_allocation_policy.cluster_secondary_range_name - services_secondary_range_name = var.composer_config.ip_allocation_policy.services_secondary_range_name + use_ip_aliases = "true" + cluster_secondary_range_name = try(var.network_config.composer_secondary_ranges.pods, "pods") + services_secondary_range_name = try(var.network_config.composer_secondary_ranges.services, "services") } } software_config { - image_version = "composer-1.17.5-airflow-2.1.4" - env_variables = { - DTL_L0_PRJ = module.dtl-0-prj.project_id - DTL_L0_BQ_DATASET = module.dtl-0-bq-0.dataset_id - DTL_L0_GCS = module.dtl-0-cs-0.url - DTL_L1_PRJ = module.dtl-1-prj.project_id - DTL_L1_BQ_DATASET = module.dtl-1-bq-0.dataset_id - DTL_L1_GCS = module.dtl-1-cs-0.url - DTL_L2_PRJ = module.dtl-2-prj.project_id - DTL_L2_BQ_DATASET = module.dtl-2-bq-0.dataset_id - DTL_L2_GCS = module.dtl-2-cs-0.url - DTL_PLG_PRJ = module.dtl-plg-prj.project_id - DTL_PLG_BQ_DATASET = module.dtl-plg-bq-0.dataset_id - DTL_PLG_GCS = module.dtl-plg-cs-0.url - GCP_REGION = var.composer_config.region - LND_PRJ = module.lnd-prj.project_id - LND_BQ = module.lnd-bq-0.dataset_id - LND_GCS = module.lnd-cs-0.url - LND_PS = module.lnd-ps-0.id - LOD_PRJ = module.lod-prj.project_id - LOD_GCS_STAGING = module.lod-cs-df-0.url - LOD_NET_VPC = local._networks.load.network - LOD_NET_SUBNET = local._networks.load.subnet - LOD_SA_DF = module.lod-sa-df-0.email - ORC_PRJ = module.orc-prj.project_id - ORC_GCS = module.orc-cs-0.url - TRF_PRJ = module.trf-prj.project_id - TRF_GCS_STAGING = module.trf-cs-df-0.url - TRF_NET_VPC = local._networks.transformation.network - TRF_NET_SUBNET = local._networks.transformation.subnet - TRF_SA_DF = module.trf-sa-df-0.email - TRF_SA_BQ = module.trf-sa-bq-0.email - } + image_version = var.composer_config.airflow_version + env_variables = merge( + var.composer_config.env_variables, { + DTL_L0_PRJ = module.dtl-0-prj.project_id + DTL_L0_BQ_DATASET = module.dtl-0-bq-0.dataset_id + DTL_L0_GCS = module.dtl-0-cs-0.url + DTL_L1_PRJ = module.dtl-1-prj.project_id + DTL_L1_BQ_DATASET = module.dtl-1-bq-0.dataset_id + DTL_L1_GCS = module.dtl-1-cs-0.url + DTL_L2_PRJ = module.dtl-2-prj.project_id + DTL_L2_BQ_DATASET = module.dtl-2-bq-0.dataset_id + DTL_L2_GCS = module.dtl-2-cs-0.url + DTL_PLG_PRJ = module.dtl-plg-prj.project_id + DTL_PLG_BQ_DATASET = module.dtl-plg-bq-0.dataset_id + DTL_PLG_GCS = module.dtl-plg-cs-0.url + GCP_REGION = var.location_config.region + LND_PRJ = module.lnd-prj.project_id + LND_BQ = module.lnd-bq-0.dataset_id + LND_GCS = module.lnd-cs-0.url + LND_PS = module.lnd-ps-0.id + LOD_PRJ = module.lod-prj.project_id + LOD_GCS_STAGING = module.lod-cs-df-0.url + LOD_NET_VPC = local._networks.load.network + LOD_NET_SUBNET = local._networks.load.subnet + LOD_SA_DF = module.lod-sa-df-0.email + ORC_PRJ = module.orc-prj.project_id + ORC_GCS = module.orc-cs-0.url + TRF_PRJ = module.trf-prj.project_id + TRF_GCS_STAGING = module.trf-cs-df-0.url + TRF_NET_VPC = local._networks.transformation.network + TRF_NET_SUBNET = local._networks.transformation.subnet + TRF_SA_DF = module.trf-sa-df-0.email + TRF_SA_BQ = module.trf-sa-bq-0.email + } + ) } private_environment_config { enable_private_endpoint = "true" - master_ipv4_cidr_block = var.composer_config.ip_range_gke_master - cloud_sql_ipv4_cidr_block = var.composer_config.ip_range_cloudsql - web_server_ipv4_cidr_block = var.composer_config.ip_range_web_server + cloud_sql_ipv4_cidr_block = try(var.network_config.composer_ip_ranges.gke_master, "10.20.10.0/24") + master_ipv4_cidr_block = try(var.network_config.composer_ip_ranges.cloudsql, "10.20.11.0/28") + web_server_ipv4_cidr_block = try(var.network_config.composer_ip_ranges.web_server, "10.20.11.16/28") } dynamic "encryption_config" { diff --git a/examples/data-solutions/data-platform-foundations/03-orchestration.tf b/examples/data-solutions/data-platform-foundations/03-orchestration.tf index 3706684a1..0d23263ca 100644 --- a/examples/data-solutions/data-platform-foundations/03-orchestration.tf +++ b/examples/data-solutions/data-platform-foundations/03-orchestration.tf @@ -79,11 +79,13 @@ module "orc-prj" { project_create = var.project_create != null prefix = var.project_create == null ? null : var.prefix # additive IAM bindings avoid disrupting bindings in existing project - iam = var.project_create != null ? local.iam_orc : {} - iam_additive = var.project_create == null ? local.iam_orc : {} - group_iam = local.group_iam_orc - oslogin = false - policy_boolean = var.composer_config.project_policy_boolean + iam = var.project_create != null ? local.iam_orc : {} + iam_additive = var.project_create == null ? local.iam_orc : {} + group_iam = local.group_iam_orc + oslogin = false + policy_boolean = { + "constraints/compute.requireOsLogin" = false + } services = concat( var.project_services, [ @@ -111,28 +113,26 @@ module "orc-prj" { } module "orc-vpc" { - count = var.network_config.network != null ? 0 : 1 + count = var.network_config.network_self_link != null ? 0 : 1 source = "../../../modules/net-vpc" project_id = module.orc-prj.project_id name = "${local.prefix_orc}-vpc" subnets = [ { - ip_cidr_range = var.network_config.vpc_subnet.orchestration.range - name = "${local.prefix_orc}-subnet" - region = var.location_config.region - secondary_ip_range = {} + ip_cidr_range = "10.10.0.0/24" + name = "${local.prefix_orc}-subnet" + region = var.location_config.region secondary_ip_range = { - pods = var.network_config.vpc_subnet.orchestration.secondary_range.pods - services = var.network_config.vpc_subnet.orchestration.secondary_range.services + pods = "10.10.8.0/22" + services = "10.10.12.0/24" } } ] } -#TODO Check with Simo/Ludo resource "google_project_iam_binding" "composer_shared_vpc_agent" { - count = var.network_config.network != null ? 1 : 0 - project = var.network_config.host_project + count = var.network_config.network_self_link != null ? 1 : 0 + project = local._shared_vpc_project role = "roles/composer.sharedVpcAgent" members = [ "serviceAccount:${module.orc-prj.service_accounts.robots.composer}" @@ -140,8 +140,8 @@ resource "google_project_iam_binding" "composer_shared_vpc_agent" { } resource "google_project_iam_binding" "gke_host_service_agent_user" { - count = var.network_config.network != null ? 1 : 0 - project = var.network_config.host_project + count = var.network_config.network_self_link != null ? 1 : 0 + project = local._shared_vpc_project role = "roles/container.hostServiceAgentUser" members = [ "serviceAccount:${module.orc-prj.service_accounts.robots.container-engine}" @@ -149,8 +149,8 @@ resource "google_project_iam_binding" "gke_host_service_agent_user" { } resource "google_project_iam_binding" "composer_network_user_agent" { - count = var.network_config.network != null ? 1 : 0 - project = var.network_config.host_project + count = var.network_config.network_self_link != null ? 1 : 0 + project = local._shared_vpc_project role = "roles/compute.networkUser" members = [ module.orc-sa-cmp-0.iam_email, @@ -164,15 +164,15 @@ resource "google_project_iam_binding" "composer_network_user_agent" { } module "orc-vpc-firewall" { - count = var.network_config.network != null ? 0 : 1 + count = var.network_config.network_self_link != null ? 0 : 1 source = "../../../modules/net-vpc-firewall" project_id = module.orc-prj.project_id network = local._networks.orchestration.network_name - admin_ranges = [local._networks.orchestration.subnet_range] + admin_ranges = ["10.10.0.0/24"] } module "orc-nat" { - count = var.network_config.enable_cloud_nat ? 1 : 0 + count = var.network_config.network_self_link != null ? 0 : 1 source = "../../../modules/net-cloudnat" project_id = module.orc-prj.project_id region = var.location_config.region diff --git a/examples/data-solutions/data-platform-foundations/04-transformation.tf b/examples/data-solutions/data-platform-foundations/04-transformation.tf index 6b3e14a2d..90c5aac78 100644 --- a/examples/data-solutions/data-platform-foundations/04-transformation.tf +++ b/examples/data-solutions/data-platform-foundations/04-transformation.tf @@ -77,13 +77,13 @@ module "trf-prj" { } module "trf-vpc" { - count = var.network_config.network != null ? 0 : 1 + count = var.network_config.network_self_link != null ? 0 : 1 source = "../../../modules/net-vpc" project_id = module.trf-prj.project_id name = "${local.prefix_trf}-vpc" subnets = [ { - ip_cidr_range = var.network_config.vpc_subnet.transformation.range + ip_cidr_range = "10.10.0.0/24" name = "${local.prefix_trf}-subnet" region = var.location_config.region secondary_ip_range = {} @@ -92,18 +92,18 @@ module "trf-vpc" { } module "trf-vpc-firewall" { - count = var.network_config.network != null ? 0 : 1 + count = var.network_config.network_self_link != null ? 0 : 1 source = "../../../modules/net-vpc-firewall" project_id = module.trf-prj.project_id network = local._networks.transformation.network_name - admin_ranges = [local._networks.transformation.subnet_range] + admin_ranges = ["10.10.0.0/24"] } module "trf-nat" { - count = var.network_config.enable_cloud_nat ? 1 : 0 + count = var.network_config.network_self_link != null ? 0 : 1 source = "../../../modules/net-cloudnat" project_id = module.trf-prj.project_id region = var.location_config.region name = "${local.prefix_trf}-default" - router_network = local._networks.transformation.network_name + router_network = module.trf-vpc[0].name } diff --git a/examples/data-solutions/data-platform-foundations/README.md b/examples/data-solutions/data-platform-foundations/README.md index d9be14e01..7e0350d11 100644 --- a/examples/data-solutions/data-platform-foundations/README.md +++ b/examples/data-solutions/data-platform-foundations/README.md @@ -8,7 +8,7 @@ The following diagram is a high-level reference of the resources created and man ![Data Platform architecture overview](./images/overview_diagram.png "Data Platform architecture overview") -A demo pipeline is also part of this example: it can be built and run on top of the foundational infrastructure to verify or test the setup quickly. +A demo Airflow pipeline is also part of this example: it can be built and run on top of the foundational infrastructure to verify or test the setup quickly. ## Design overview and choices @@ -21,7 +21,7 @@ The approach adapts to different high-level requirements: - least privilege principle - rely on service account impersonation -The code in this example doesn't address Organization-level configurations (Organization policy, VPC-SC, centralized logs). We expect those to be managed by automation stages external to this script like those in [FAST](../../../fast). +The code in this example doesn't address Organization-level configurations (Organization policy, VPC-SC, centralized logs). We expect those elements to be managed by automation stages external to this script like those in [FAST](../../../fast). ### Project structure diff --git a/examples/data-solutions/data-platform-foundations/main.tf b/examples/data-solutions/data-platform-foundations/main.tf index 5911dfbda..f451660b0 100644 --- a/examples/data-solutions/data-platform-foundations/main.tf +++ b/examples/data-solutions/data-platform-foundations/main.tf @@ -17,31 +17,28 @@ locals { _networks = { load = { - #TODO Fix Network name logic - network_name = element(split("/", var.network_config.network != null ? var.network_config.network : module.lod-vpc[0].self_link), length(split("/", var.network_config.network != null ? var.network_config.network : module.lod-vpc[0].self_link)) - 1) - network = var.network_config.network != null ? var.network_config.network : module.lod-vpc[0].self_link - subnet = var.network_config.network != null ? var.network_config.vpc_subnet_self_link.load : module.lod-vpc[0].subnet_self_links["${var.composer_config.region}/${local.prefix_lod}-subnet"] - subnet_range = try(var.network_config.vpc_subnet.load.range, null) + network_name = element(split("/", var.network_config.network_self_link != null ? var.network_config.network_self_link : module.lod-vpc[0].self_link), length(split("/", var.network_config.network_self_link != null ? var.network_config.network_self_link : module.lod-vpc[0].self_link)) - 1) + network = var.network_config.network_self_link != null ? var.network_config.network_self_link : module.lod-vpc[0].self_link + subnet = var.network_config.network_self_link != null ? var.network_config.vpc_subnet_self_link.load : module.lod-vpc[0].subnet_self_links["${var.location_config.region}/${local.prefix_lod}-subnet"] } orchestration = { #TODO Fix Network name logic - network_name = element(split("/", var.network_config.network != null ? var.network_config.network : module.orc-vpc[0].self_link), length(split("/", var.network_config.network != null ? var.network_config.network : module.orc-vpc[0].self_link)) - 1) - network = var.network_config.network != null ? var.network_config.network : module.orc-vpc[0].self_link - subnet = var.network_config.network != null ? var.network_config.vpc_subnet_self_link.orchestration : module.orc-vpc[0].subnet_self_links["${var.composer_config.region}/${local.prefix_orc}-subnet"] - subnet_range = try(var.network_config.vpc_subnet.orchestration.range, null) + network_name = element(split("/", var.network_config.network_self_link != null ? var.network_config.network_self_link : module.orc-vpc[0].self_link), length(split("/", var.network_config.network_self_link != null ? var.network_config.network_self_link : module.orc-vpc[0].self_link)) - 1) + network = var.network_config.network_self_link != null ? var.network_config.network_self_link : module.orc-vpc[0].self_link + subnet = var.network_config.network_self_link != null ? var.network_config.vpc_subnet_self_link.orchestration : module.orc-vpc[0].subnet_self_links["${var.location_config.region}/${local.prefix_orc}-subnet"] } transformation = { #TODO Fix Network name logic - network_name = element(split("/", var.network_config.network != null ? var.network_config.network : module.trf-vpc[0].self_link), length(split("/", var.network_config.network != null ? var.network_config.network : module.trf-vpc[0].self_link)) - 1) - network = var.network_config.network != null ? var.network_config.network : module.trf-vpc[0].self_link - subnet = var.network_config.network != null ? var.network_config.vpc_subnet_self_link.transformation : module.trf-vpc[0].subnet_self_links["${var.composer_config.region}/${local.prefix_trf}-subnet"] - subnet_range = try(var.network_config.vpc_subnet.transformation.range, null) + network_name = element(split("/", var.network_config.network_self_link != null ? var.network_config.network_self_link : module.trf-vpc[0].self_link), length(split("/", var.network_config.network_self_link != null ? var.network_config.network_self_link : module.trf-vpc[0].self_link)) - 1) + network = var.network_config.network_self_link != null ? var.network_config.network_self_link : module.trf-vpc[0].self_link + subnet = var.network_config.network_self_link != null ? var.network_config.vpc_subnet_self_link.transformation : module.trf-vpc[0].subnet_self_links["${var.location_config.region}/${local.prefix_trf}-subnet"] } } - _shared_vpc_service_config = var.network_config.network != null ? { + _shared_vpc_project = try(regex("projects/([a-z0-9-]{6,30})", var.network_config.network_self_link), null) + _shared_vpc_service_config = var.network_config.network_self_link != null ? { attach = true - host_project = var.network_config.host_project + host_project = local._shared_vpc_project } : null groups = { for k, v in var.groups : k => "${v}@${var.organization.domain}" } diff --git a/examples/data-solutions/data-platform-foundations/variables.tf b/examples/data-solutions/data-platform-foundations/variables.tf index cb27ab116..752061e60 100644 --- a/examples/data-solutions/data-platform-foundations/variables.tf +++ b/examples/data-solutions/data-platform-foundations/variables.tf @@ -19,43 +19,12 @@ variable "composer_config" { type = object({ node_count = number airflow_version = string - environment = map(string) + env_variables = map(string) }) default = { node_count = 3 - airflow_version = "TODO" - environment = {} - } -} -variable "composer_config" { - type = object({ - node_count = number - #TODO Move to network - ip_range_cloudsql = string - ip_range_gke_master = string - ip_range_web_server = string - #TODO hardcoded - project_policy_boolean = map(bool) - region = string - ip_allocation_policy = object({ - use_ip_aliases = string - cluster_secondary_range_name = string - services_secondary_range_name = string - }) - #TODO Add Env variables, Airflow version - }) - default = { - node_count = 3 - ip_range_cloudsql = "10.20.10.0/24" - ip_range_gke_master = "10.20.11.0/28" - ip_range_web_server = "10.20.11.16/28" - project_policy_boolean = null - region = "europe-west1" - ip_allocation_policy = { - use_ip_aliases = "true" - cluster_secondary_range_name = "pods" - services_secondary_range_name = "services" - } + airflow_version = "composer-1.17.5-airflow-2.1.4" + env_variables = {} } } @@ -76,11 +45,9 @@ variable "groups" { } variable "network_config" { - description = "Network configurations to use. Specify a shared VPC to use, if null networks will be created in projects." + description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values." type = object({ - #TODO hardcoded Cloud NAT network_self_link = string - #TODO hardcoded VPC ranges subnet_self_links = object({ load = string transformation = string @@ -96,29 +63,12 @@ variable "network_config" { services = string }) }) - default = { - enable_cloud_nat = false - host_project = null - network = null - vpc_subnet = { - load = { - range = "10.10.0.0/24" - secondary_range = null - } - transformation = { - range = "10.10.0.0/24" - secondary_range = null - } - orchestration = { - range = "10.10.0.0/24" - secondary_range = { - pods = "10.10.8.0/22" - services = "10.10.12.0/24" - } - } - } - vpc_subnet_self_link = null + default = { + network_self_link = null + subnet_self_links = null + composer_ip_ranges = null + composer_secondary_ranges = null } }