diff --git a/examples/data-solutions/gcs-to-bq-with-least-privileges/README.md b/examples/data-solutions/gcs-to-bq-with-least-privileges/README.md index 8804fcc95..1043b1bf0 100644 --- a/examples/data-solutions/gcs-to-bq-with-least-privileges/README.md +++ b/examples/data-solutions/gcs-to-bq-with-least-privileges/README.md @@ -74,6 +74,16 @@ $ terraform apply You should see the output of the Terraform script with resources created and some command pre-created for you to run the example following steps below. +### Virtual Private Cloud (VPC) design + +As is often the case in real-world configurations, this example accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. + +If the `network_config` variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration). + +When `network_config` variable is configured, the identity running the Terraform script need to have the following roles: + - `roles/compute.xpnAdmin` on the host project folder or org + - `roles/resourcemanager.projectIamAdmin` on the host project, either with no conditions or with a condition allowing [delegated role grants](https://medium.com/google-cloud/managing-gcp-service-usage-through-delegated-role-grants-a843610f2226#:~:text=Delegated%20role%20grants%20is%20a,setIamPolicy%20permission%20on%20a%20resource.) for `roles/compute.networkUser`, `roles/composer.sharedVpcAgent`, `roles/container.hostServiceAgentUser` + ## Test your environment with Cloud Dataflow We assume all those steps are run using a user listed on `data_eng_principals`. You can authenticate as the user using the following command: @@ -131,13 +141,14 @@ bq query --use_legacy_sql=false 'SELECT * FROM `PROJECT.datalake.person` LIMIT 1 | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [prefix](variables.tf#L26) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | string | ✓ | | -| [project_id](variables.tf#L40) | Project id, references existing project if `project_create` is null. | string | ✓ | | +| [prefix](variables.tf#L36) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | string | ✓ | | +| [project_id](variables.tf#L50) | Project id, references existing project if `project_create` is null. | string | ✓ | | | [cmek_encryption](variables.tf#L15) | Flag to enable CMEK on GCP resources created. | bool | | false | | [data_eng_principals](variables.tf#L21) | Groups with Service Account Token creator role on service accounts in IAM format, eg 'group:group@domain.com'. | list(string) | | [] | -| [project_create](variables.tf#L31) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | -| [region](variables.tf#L45) | The region where resources will be deployed. | string | | "europe-west1" | -| [vpc_subnet_range](variables.tf#L51) | Ip range used for the VPC subnet created for the example. | string | | "10.0.0.0/20" | +| [network_config](variables.tf#L27) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | +| [project_create](variables.tf#L41) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | +| [region](variables.tf#L55) | The region where resources will be deployed. | string | | "europe-west1" | +| [vpc_subnet_range](variables.tf#L61) | Ip range used for the VPC subnet created for the example. | string | | "10.0.0.0/20" | ## Outputs diff --git a/examples/data-solutions/gcs-to-bq-with-least-privileges/main.tf b/examples/data-solutions/gcs-to-bq-with-least-privileges/main.tf index 8c8486440..80105017c 100644 --- a/examples/data-solutions/gcs-to-bq-with-least-privileges/main.tf +++ b/examples/data-solutions/gcs-to-bq-with-least-privileges/main.tf @@ -76,6 +76,29 @@ locals { "serviceAccount:${module.project.service_accounts.robots.dataflow}" ] } + network_subnet_selflink = try( + module.vpc[0].subnets["${var.region}/subnet"].self_link, + var.network_config.subnet_self_link + ) + shared_vpc_bindings = { + "roles/compute.networkUser" = [ + "robot-df", "sa-df-worker" + ] + } + # reassemble in a format suitable for for_each + shared_vpc_bindings_map = { + for binding in flatten([ + for role, members in local.shared_vpc_bindings : [ + for member in members : { role = role, member = member } + ] + ]) : "${binding.role}-${binding.member}" => binding + } + shared_vpc_project = try(var.network_config.host_project, null) + shared_vpc_role_members = { + robot-df = "serviceAccount:${module.project.service_accounts.robots.dataflow}" + sa-df-worker = module.service-account-df.iam_email + } + use_shared_vpc = var.network_config != null } module "project" { @@ -100,7 +123,19 @@ module "project" { # additive IAM bindings avoid disrupting bindings in existing project iam = var.project_create != null ? local.iam : {} iam_additive = var.project_create == null ? local.iam : {} + shared_vpc_service_config = local.shared_vpc_project == null ? null : { + attach = true + host_project = local.shared_vpc_project + service_identity_iam = {} + } service_config = { disable_on_destroy = false, disable_dependent_services = false } } + +resource "google_project_iam_member" "shared_vpc" { + for_each = local.use_shared_vpc ? local.shared_vpc_bindings_map : {} + project = var.network_config.host_project + role = each.value.role + member = lookup(local.shared_vpc_role_members, each.value.member) +} diff --git a/examples/data-solutions/gcs-to-bq-with-least-privileges/outputs.tf b/examples/data-solutions/gcs-to-bq-with-least-privileges/outputs.tf index 7ffd06340..141e424bc 100644 --- a/examples/data-solutions/gcs-to-bq-with-least-privileges/outputs.tf +++ b/examples/data-solutions/gcs-to-bq-with-least-privileges/outputs.tf @@ -51,7 +51,7 @@ output "command_02_dataflow" { sa_orch_email = module.service-account-orch.email project_id = module.project.project_id region = var.region - subnet = module.vpc.subnets["${var.region}/subnet"].self_link + subnet = local.network_subnet_selflink gcs_df_stg = format("%s/%s", module.gcs-df-tmp.url, "stg") sa_df_email = module.service-account-df.email cmek_encryption = var.cmek_encryption diff --git a/examples/data-solutions/gcs-to-bq-with-least-privileges/variables.tf b/examples/data-solutions/gcs-to-bq-with-least-privileges/variables.tf index 686339872..026e07b69 100644 --- a/examples/data-solutions/gcs-to-bq-with-least-privileges/variables.tf +++ b/examples/data-solutions/gcs-to-bq-with-least-privileges/variables.tf @@ -23,6 +23,16 @@ variable "data_eng_principals" { type = list(string) default = [] } + +variable "network_config" { + description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values." + type = object({ + host_project = string + subnet_self_link = string + }) + default = null +} + variable "prefix" { description = "Unique prefix used for resource names. Not used for project if 'project_create' is null." type = string diff --git a/examples/data-solutions/gcs-to-bq-with-least-privileges/vpc.tf b/examples/data-solutions/gcs-to-bq-with-least-privileges/vpc.tf index d8cc33244..21c344787 100644 --- a/examples/data-solutions/gcs-to-bq-with-least-privileges/vpc.tf +++ b/examples/data-solutions/gcs-to-bq-with-least-privileges/vpc.tf @@ -14,6 +14,7 @@ module "vpc" { source = "../../../modules/net-vpc" + count = local.use_shared_vpc ? 0 : 1 project_id = module.project.project_id name = "${var.prefix}-vpc" subnets = [ @@ -28,15 +29,17 @@ module "vpc" { module "vpc-firewall" { source = "../../../modules/net-vpc-firewall" + count = local.use_shared_vpc ? 0 : 1 project_id = module.project.project_id - network = module.vpc.name + network = module.vpc[0].name admin_ranges = [var.vpc_subnet_range] } module "nat" { source = "../../../modules/net-cloudnat" + count = local.use_shared_vpc ? 0 : 1 project_id = module.project.project_id region = var.region name = "${var.prefix}-default" - router_network = module.vpc.name + router_network = module.vpc[0].name }