diff --git a/examples/data-solutions/gcs-to-bq-with-least-privileges/README.md b/examples/data-solutions/gcs-to-bq-with-least-privileges/README.md
index 8804fcc95..1043b1bf0 100644
--- a/examples/data-solutions/gcs-to-bq-with-least-privileges/README.md
+++ b/examples/data-solutions/gcs-to-bq-with-least-privileges/README.md
@@ -74,6 +74,16 @@ $ terraform apply
You should see the output of the Terraform script with resources created and some command pre-created for you to run the example following steps below.
+### Virtual Private Cloud (VPC) design
+
+As is often the case in real-world configurations, this example accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable.
+
+If the `network_config` variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration).
+
+When `network_config` variable is configured, the identity running the Terraform script need to have the following roles:
+ - `roles/compute.xpnAdmin` on the host project folder or org
+ - `roles/resourcemanager.projectIamAdmin` on the host project, either with no conditions or with a condition allowing [delegated role grants](https://medium.com/google-cloud/managing-gcp-service-usage-through-delegated-role-grants-a843610f2226#:~:text=Delegated%20role%20grants%20is%20a,setIamPolicy%20permission%20on%20a%20resource.) for `roles/compute.networkUser`, `roles/composer.sharedVpcAgent`, `roles/container.hostServiceAgentUser`
+
## Test your environment with Cloud Dataflow
We assume all those steps are run using a user listed on `data_eng_principals`. You can authenticate as the user using the following command:
@@ -131,13 +141,14 @@ bq query --use_legacy_sql=false 'SELECT * FROM `PROJECT.datalake.person` LIMIT 1
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
-| [prefix](variables.tf#L26) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | string | ✓ | |
-| [project_id](variables.tf#L40) | Project id, references existing project if `project_create` is null. | string | ✓ | |
+| [prefix](variables.tf#L36) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | string | ✓ | |
+| [project_id](variables.tf#L50) | Project id, references existing project if `project_create` is null. | string | ✓ | |
| [cmek_encryption](variables.tf#L15) | Flag to enable CMEK on GCP resources created. | bool | | false |
| [data_eng_principals](variables.tf#L21) | Groups with Service Account Token creator role on service accounts in IAM format, eg 'group:group@domain.com'. | list(string) | | [] |
-| [project_create](variables.tf#L31) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null |
-| [region](variables.tf#L45) | The region where resources will be deployed. | string | | "europe-west1" |
-| [vpc_subnet_range](variables.tf#L51) | Ip range used for the VPC subnet created for the example. | string | | "10.0.0.0/20" |
+| [network_config](variables.tf#L27) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null |
+| [project_create](variables.tf#L41) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null |
+| [region](variables.tf#L55) | The region where resources will be deployed. | string | | "europe-west1" |
+| [vpc_subnet_range](variables.tf#L61) | Ip range used for the VPC subnet created for the example. | string | | "10.0.0.0/20" |
## Outputs
diff --git a/examples/data-solutions/gcs-to-bq-with-least-privileges/main.tf b/examples/data-solutions/gcs-to-bq-with-least-privileges/main.tf
index 8c8486440..80105017c 100644
--- a/examples/data-solutions/gcs-to-bq-with-least-privileges/main.tf
+++ b/examples/data-solutions/gcs-to-bq-with-least-privileges/main.tf
@@ -76,6 +76,29 @@ locals {
"serviceAccount:${module.project.service_accounts.robots.dataflow}"
]
}
+ network_subnet_selflink = try(
+ module.vpc[0].subnets["${var.region}/subnet"].self_link,
+ var.network_config.subnet_self_link
+ )
+ shared_vpc_bindings = {
+ "roles/compute.networkUser" = [
+ "robot-df", "sa-df-worker"
+ ]
+ }
+ # reassemble in a format suitable for for_each
+ shared_vpc_bindings_map = {
+ for binding in flatten([
+ for role, members in local.shared_vpc_bindings : [
+ for member in members : { role = role, member = member }
+ ]
+ ]) : "${binding.role}-${binding.member}" => binding
+ }
+ shared_vpc_project = try(var.network_config.host_project, null)
+ shared_vpc_role_members = {
+ robot-df = "serviceAccount:${module.project.service_accounts.robots.dataflow}"
+ sa-df-worker = module.service-account-df.iam_email
+ }
+ use_shared_vpc = var.network_config != null
}
module "project" {
@@ -100,7 +123,19 @@ module "project" {
# additive IAM bindings avoid disrupting bindings in existing project
iam = var.project_create != null ? local.iam : {}
iam_additive = var.project_create == null ? local.iam : {}
+ shared_vpc_service_config = local.shared_vpc_project == null ? null : {
+ attach = true
+ host_project = local.shared_vpc_project
+ service_identity_iam = {}
+ }
service_config = {
disable_on_destroy = false, disable_dependent_services = false
}
}
+
+resource "google_project_iam_member" "shared_vpc" {
+ for_each = local.use_shared_vpc ? local.shared_vpc_bindings_map : {}
+ project = var.network_config.host_project
+ role = each.value.role
+ member = lookup(local.shared_vpc_role_members, each.value.member)
+}
diff --git a/examples/data-solutions/gcs-to-bq-with-least-privileges/outputs.tf b/examples/data-solutions/gcs-to-bq-with-least-privileges/outputs.tf
index 7ffd06340..141e424bc 100644
--- a/examples/data-solutions/gcs-to-bq-with-least-privileges/outputs.tf
+++ b/examples/data-solutions/gcs-to-bq-with-least-privileges/outputs.tf
@@ -51,7 +51,7 @@ output "command_02_dataflow" {
sa_orch_email = module.service-account-orch.email
project_id = module.project.project_id
region = var.region
- subnet = module.vpc.subnets["${var.region}/subnet"].self_link
+ subnet = local.network_subnet_selflink
gcs_df_stg = format("%s/%s", module.gcs-df-tmp.url, "stg")
sa_df_email = module.service-account-df.email
cmek_encryption = var.cmek_encryption
diff --git a/examples/data-solutions/gcs-to-bq-with-least-privileges/variables.tf b/examples/data-solutions/gcs-to-bq-with-least-privileges/variables.tf
index 686339872..026e07b69 100644
--- a/examples/data-solutions/gcs-to-bq-with-least-privileges/variables.tf
+++ b/examples/data-solutions/gcs-to-bq-with-least-privileges/variables.tf
@@ -23,6 +23,16 @@ variable "data_eng_principals" {
type = list(string)
default = []
}
+
+variable "network_config" {
+ description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values."
+ type = object({
+ host_project = string
+ subnet_self_link = string
+ })
+ default = null
+}
+
variable "prefix" {
description = "Unique prefix used for resource names. Not used for project if 'project_create' is null."
type = string
diff --git a/examples/data-solutions/gcs-to-bq-with-least-privileges/vpc.tf b/examples/data-solutions/gcs-to-bq-with-least-privileges/vpc.tf
index d8cc33244..21c344787 100644
--- a/examples/data-solutions/gcs-to-bq-with-least-privileges/vpc.tf
+++ b/examples/data-solutions/gcs-to-bq-with-least-privileges/vpc.tf
@@ -14,6 +14,7 @@
module "vpc" {
source = "../../../modules/net-vpc"
+ count = local.use_shared_vpc ? 0 : 1
project_id = module.project.project_id
name = "${var.prefix}-vpc"
subnets = [
@@ -28,15 +29,17 @@ module "vpc" {
module "vpc-firewall" {
source = "../../../modules/net-vpc-firewall"
+ count = local.use_shared_vpc ? 0 : 1
project_id = module.project.project_id
- network = module.vpc.name
+ network = module.vpc[0].name
admin_ranges = [var.vpc_subnet_range]
}
module "nat" {
source = "../../../modules/net-cloudnat"
+ count = local.use_shared_vpc ? 0 : 1
project_id = module.project.project_id
region = var.region
name = "${var.prefix}-default"
- router_network = module.vpc.name
+ router_network = module.vpc[0].name
}