diff --git a/CHANGELOG.md b/CHANGELOG.md index b4b50f105..71adc7a32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ All notable changes to this project will be documented in this file. ### FAST +- [[#3384](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/3384)] Add support for universe to fast project factory stage ([ludoo](https://github.com/ludoo)) - [[#3383](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/3383)] Support universe in fast security stage ([ludoo](https://github.com/ludoo)) - [[#3381](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/3381)] Fix typo in fast stage 0 provider template ([ludoo](https://github.com/ludoo)) - [[#3379](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/3379)] Allow FAST stage 0 provider template to work with universe ([ludoo](https://github.com/ludoo)) @@ -27,6 +28,7 @@ All notable changes to this project will be documented in this file. ### MODULES +- [[#3388](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/3388)] Add support for context to bigquery module ([ludoo](https://github.com/ludoo)) - [[#3377](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/3377)] feat(bigquery-dataset): add optional schema support for views ([weather2602](https://github.com/weather2602)) - [[#3380](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/3380)] Lightly refactor service agents locals in project module ([ludoo](https://github.com/ludoo)) - [[#3378](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/3378)] Allow forcing jit service agents generation for universe in project and project factory modules ([ludoo](https://github.com/ludoo)) diff --git a/fast/stages/2-project-factory/README.md b/fast/stages/2-project-factory/README.md index 08a57b92e..8bab44a80 100644 --- a/fast/stages/2-project-factory/README.md +++ b/fast/stages/2-project-factory/README.md @@ -498,6 +498,7 @@ Pattern-based files make specific assumptions: | [stage_name](variables.tf#L57) | FAST stage name. Used to separate output files across different factories. | string | | "2-project-factory" | | | [subnet_self_links](variables-fast.tf#L118) | Shared VPC subnet IDs. | map(map(string)) | | {} | 2-networking | | [tag_values](variables-fast.tf#L126) | FAST-managed resource manager tag values. | map(string) | | {} | 0-org-setup | +| [universe](variables-fast.tf#L134) | GCP universe where to deploy projects. The prefix will be prepended to the project id. | object({…}) | | null | 0-org-setup | ## Outputs diff --git a/fast/stages/2-project-factory/main.tf b/fast/stages/2-project-factory/main.tf index 011113726..e1f7eb18d 100644 --- a/fast/stages/2-project-factory/main.tf +++ b/fast/stages/2-project-factory/main.tf @@ -43,11 +43,16 @@ locals { local.defaults.projects.merges[k], v ) } - overrides = { - for k, v in var.data_overrides : k => try( - local.defaults.projects.overrides[k], v - ) - } + overrides = merge( + { + for k, v in var.data_overrides : k => try( + local.defaults.projects.overrides[k], v + ) + }, + { + universe = var.universe + } + ) } subnet_self_links = flatten([ for net, subnets in var.subnet_self_links : [ diff --git a/fast/stages/2-project-factory/variables-fast.tf b/fast/stages/2-project-factory/variables-fast.tf index b39b799f4..b227f7747 100644 --- a/fast/stages/2-project-factory/variables-fast.tf +++ b/fast/stages/2-project-factory/variables-fast.tf @@ -130,3 +130,16 @@ variable "tag_values" { nullable = false default = {} } + +variable "universe" { + # tfdoc:variable:source 0-org-setup + description = "GCP universe where to deploy projects. The prefix will be prepended to the project id." + type = object({ + domain = string + prefix = string + forced_jit_service_identities = optional(list(string), []) + unavailable_services = optional(list(string), []) + unavailable_service_identities = optional(list(string), []) + }) + default = null +} diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index a32e25409..5e4cb780a 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -353,26 +353,27 @@ module "bigquery-dataset" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [id](variables.tf#L98) | Dataset id. | string | ✓ | | -| [project_id](variables.tf#L162) | Id of the project where datasets will be created. | string | ✓ | | +| [id](variables.tf#L111) | Dataset id. | string | ✓ | | +| [project_id](variables.tf#L175) | Id of the project where datasets will be created. | string | ✓ | | | [access](variables.tf#L17) | Map of access rules with role and identity type. Keys are arbitrary and must match those in the `access_identities` variable, types are `domain`, `group`, `special_group`, `user`, `view`. | map(object({…})) | | {} | | [access_identities](variables.tf#L33) | Map of access identities used for basic access roles. View identities have the format 'project_id\|dataset_id\|table_id'. | map(string) | | {} | | [authorized_datasets](variables.tf#L39) | An array of datasets to be authorized on the dataset. | list(object({…})) | | [] | | [authorized_routines](variables.tf#L48) | An array of routines to be authorized on the dataset. | list(object({…})) | | [] | | [authorized_views](variables.tf#L58) | An array of views to be authorized on the dataset. | list(object({…})) | | [] | -| [dataset_access](variables.tf#L68) | Set access in the dataset resource instead of using separate resources. | bool | | false | -| [description](variables.tf#L74) | Optional description. | string | | "Terraform managed." | -| [encryption_key](variables.tf#L80) | Self link of the KMS key that will be used to protect destination table. | string | | null | -| [friendly_name](variables.tf#L86) | Dataset friendly name. | string | | null | -| [iam](variables.tf#L92) | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | map(list(string)) | | {} | -| [labels](variables.tf#L103) | Dataset labels. | map(string) | | {} | -| [location](variables.tf#L109) | Dataset location. | string | | "EU" | -| [materialized_views](variables.tf#L115) | Materialized views definitions. | map(object({…})) | | {} | -| [options](variables.tf#L148) | Dataset options. | object({…}) | | {} | -| [routines](variables.tf#L167) | Routine definitions. | map(object({…})) | | {} | -| [tables](variables.tf#L205) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…})) | | {} | -| [tag_bindings](variables.tf#L290) | Tag bindings for this dataset, in key => tag value id format. | map(string) | | {} | -| [views](variables.tf#L297) | View definitions. | map(object({…})) | | {} | +| [context](variables.tf#L68) | Context-specific interpolations. | object({…}) | | {} | +| [dataset_access](variables.tf#L81) | Set access in the dataset resource instead of using separate resources. | bool | | false | +| [description](variables.tf#L87) | Optional description. | string | | "Terraform managed." | +| [encryption_key](variables.tf#L93) | Self link of the KMS key that will be used to protect destination table. | string | | null | +| [friendly_name](variables.tf#L99) | Dataset friendly name. | string | | null | +| [iam](variables.tf#L105) | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | map(list(string)) | | {} | +| [labels](variables.tf#L116) | Dataset labels. | map(string) | | {} | +| [location](variables.tf#L122) | Dataset location. | string | | "EU" | +| [materialized_views](variables.tf#L128) | Materialized views definitions. | map(object({…})) | | {} | +| [options](variables.tf#L161) | Dataset options. | object({…}) | | {} | +| [routines](variables.tf#L180) | Routine definitions. | map(object({…})) | | {} | +| [tables](variables.tf#L218) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…})) | | {} | +| [tag_bindings](variables.tf#L303) | Tag bindings for this dataset, in key => tag value id format. | map(string) | | {} | +| [views](variables.tf#L310) | View definitions. | map(object({…})) | | {} | ## Outputs diff --git a/modules/bigquery-dataset/main.tf b/modules/bigquery-dataset/main.tf index 35810965f..2ed2f9927 100644 --- a/modules/bigquery-dataset/main.tf +++ b/modules/bigquery-dataset/main.tf @@ -20,7 +20,30 @@ locals { access_special = { for k, v in var.access : k => v if v.type == "special_group" } access_user = { for k, v in var.access : k => v if v.type == "user" } access_view = { for k, v in var.access : k => v if v.type == "view" } - + authorized_datasets = { + for dataset in var.authorized_datasets : + "${dataset["project_id"]}_${dataset["dataset_id"]}" => dataset + } + authorized_routines = { + for routine in var.authorized_routines : + "${routine["project_id"]}_${routine["dataset_id"]}_${routine["routine_id"]}" => routine + } + authorized_views = merge( + { + for access_key, view in local.identities_view : + "${view["project_id"]}_${view["dataset_id"]}_${view["table_id"]}" => view + }, + { + for view in var.authorized_views : + "${view["project_id"]}_${view["dataset_id"]}_${view["table_id"]}" => view + } + ) + ctx = { + for k, v in var.context : k => { + for kk, vv in v : "${local.ctx_p}${k}:${kk}" => vv + } if k != "condition_vars" + } + ctx_p = "$" identities_view = { for k, v in local.access_view : k => try( zipmap( @@ -30,23 +53,21 @@ locals { { project_id = null, dataset_id = null, table_id = null } ) } - - authorized_views = merge( - { for access_key, view in local.identities_view : "${view["project_id"]}_${view["dataset_id"]}_${view["table_id"]}" => view }, - { for view in var.authorized_views : "${view["project_id"]}_${view["dataset_id"]}_${view["table_id"]}" => view }) - authorized_datasets = { for dataset in var.authorized_datasets : "${dataset["project_id"]}_${dataset["dataset_id"]}" => dataset } - authorized_routines = { for routine in var.authorized_routines : "${routine["project_id"]}_${routine["dataset_id"]}_${routine["routine_id"]}" => routine } - + location = lookup( + local.ctx.locations, var.location, var.location + ) + project_id = lookup( + local.ctx.project_ids, var.project_id, var.project_id + ) } resource "google_bigquery_dataset" "default" { - project = var.project_id - dataset_id = var.id - friendly_name = var.friendly_name - description = var.description - labels = var.labels - location = var.location - + project = local.project_id + dataset_id = var.id + friendly_name = var.friendly_name + description = var.description + labels = var.labels + location = local.location delete_contents_on_destroy = var.options.delete_contents_on_destroy default_collation = var.options.default_collation default_table_expiration_ms = var.options.default_table_expiration_ms @@ -61,7 +82,6 @@ resource "google_bigquery_dataset" "default" { domain = try(var.access_identities[access.key]) } } - dynamic "access" { for_each = var.dataset_access ? local.access_group : {} content { @@ -69,7 +89,6 @@ resource "google_bigquery_dataset" "default" { group_by_email = try(var.access_identities[access.key]) } } - dynamic "access" { for_each = var.dataset_access ? local.access_special : {} content { @@ -77,7 +96,6 @@ resource "google_bigquery_dataset" "default" { special_group = try(var.access_identities[access.key]) } } - dynamic "access" { for_each = var.dataset_access ? local.access_user : {} content { @@ -85,42 +103,44 @@ resource "google_bigquery_dataset" "default" { user_by_email = try(var.access_identities[access.key]) } } - dynamic "access" { for_each = var.dataset_access ? local.authorized_views : {} content { view { - project_id = each.value.project_id + project_id = lookup( + local.ctx.project_ids, each.value.project_id, each.value.project_id + ) dataset_id = each.value.dataset_id table_id = each.value.table_id } } } - dynamic "access" { for_each = var.dataset_access ? local.authorized_datasets : {} content { dataset { dataset { - project_id = each.value.project_id + project_id = lookup( + local.ctx.project_ids, each.value.project_id, each.value.project_id + ) dataset_id = each.value.dataset_id } target_types = ["VIEWS"] } } } - dynamic "access" { for_each = var.dataset_access ? local.authorized_routines : {} content { routine { - project_id = each.value.project_id + project_id = lookup( + local.ctx.project_ids, each.value.project_id, each.value.project_id + ) dataset_id = each.value.dataset_id routine_id = each.value.routine_id } } } - dynamic "default_encryption_configuration" { for_each = var.encryption_key == null ? [] : [""] content { @@ -132,7 +152,7 @@ resource "google_bigquery_dataset" "default" { resource "google_bigquery_dataset_access" "domain" { for_each = var.dataset_access ? {} : local.access_domain provider = google-beta - project = var.project_id + project = local.project_id dataset_id = google_bigquery_dataset.default.dataset_id role = each.value.role domain = try(var.access_identities[each.key]) @@ -141,7 +161,7 @@ resource "google_bigquery_dataset_access" "domain" { resource "google_bigquery_dataset_access" "group_by_email" { for_each = var.dataset_access ? {} : local.access_group provider = google-beta - project = var.project_id + project = local.project_id dataset_id = google_bigquery_dataset.default.dataset_id role = each.value.role group_by_email = try(var.access_identities[each.key]) @@ -150,7 +170,7 @@ resource "google_bigquery_dataset_access" "group_by_email" { resource "google_bigquery_dataset_access" "special_group" { for_each = var.dataset_access ? {} : local.access_special provider = google-beta - project = var.project_id + project = local.project_id dataset_id = google_bigquery_dataset.default.dataset_id role = each.value.role special_group = try(var.access_identities[each.key]) @@ -159,7 +179,7 @@ resource "google_bigquery_dataset_access" "special_group" { resource "google_bigquery_dataset_access" "user_by_email" { for_each = var.dataset_access ? {} : local.access_user provider = google-beta - project = var.project_id + project = local.project_id dataset_id = google_bigquery_dataset.default.dataset_id role = each.value.role user_by_email = try(var.access_identities[each.key]) @@ -167,10 +187,12 @@ resource "google_bigquery_dataset_access" "user_by_email" { resource "google_bigquery_dataset_access" "authorized_views" { for_each = var.dataset_access ? {} : local.authorized_views - project = var.project_id + project = local.project_id dataset_id = google_bigquery_dataset.default.dataset_id view { - project_id = each.value.project_id + project_id = lookup( + local.ctx.project_ids, each.value.project_id, each.value.project_id + ) dataset_id = each.value.dataset_id table_id = each.value.table_id } @@ -178,11 +200,13 @@ resource "google_bigquery_dataset_access" "authorized_views" { resource "google_bigquery_dataset_access" "authorized_datasets" { for_each = var.dataset_access ? {} : local.authorized_datasets - project = var.project_id + project = local.project_id dataset_id = google_bigquery_dataset.default.dataset_id dataset { dataset { - project_id = each.value.project_id + project_id = lookup( + local.ctx.project_ids, each.value.project_id, each.value.project_id + ) dataset_id = each.value.dataset_id } target_types = ["VIEWS"] @@ -191,10 +215,12 @@ resource "google_bigquery_dataset_access" "authorized_datasets" { resource "google_bigquery_dataset_access" "authorized_routines" { for_each = var.dataset_access ? {} : local.authorized_routines - project = var.project_id + project = local.project_id dataset_id = google_bigquery_dataset.default.dataset_id routine { - project_id = each.value.project_id + project_id = lookup( + local.ctx.project_ids, each.value.project_id, each.value.project_id + ) dataset_id = each.value.dataset_id routine_id = each.value.routine_id } @@ -202,16 +228,18 @@ resource "google_bigquery_dataset_access" "authorized_routines" { resource "google_bigquery_dataset_iam_binding" "bindings" { for_each = var.iam - project = var.project_id + project = local.project_id dataset_id = google_bigquery_dataset.default.dataset_id - role = each.key - members = each.value + role = lookup(local.ctx.custom_roles, each.key, each.key) + members = [ + for v in each.value : lookup(local.ctx.iam_principals, v, v) + ] } resource "google_bigquery_table" "default" { provider = google-beta for_each = var.tables - project = var.project_id + project = local.project_id dataset_id = google_bigquery_dataset.default.dataset_id table_id = each.key friendly_name = each.value.friendly_name @@ -347,7 +375,7 @@ resource "google_bigquery_table" "default" { resource "google_bigquery_table" "views" { depends_on = [google_bigquery_table.default] for_each = var.views - project = var.project_id + project = local.project_id dataset_id = google_bigquery_dataset.default.dataset_id table_id = each.key friendly_name = each.value.friendly_name @@ -365,7 +393,7 @@ resource "google_bigquery_table" "views" { resource "google_bigquery_table" "materialized_view" { depends_on = [google_bigquery_table.default] for_each = var.materialized_views - project = var.project_id + project = local.project_id dataset_id = google_bigquery_dataset.default.dataset_id table_id = each.key friendly_name = each.value.friendly_name @@ -407,7 +435,7 @@ resource "google_bigquery_table" "materialized_view" { resource "google_bigquery_routine" "default" { for_each = var.routines - project = var.project_id + project = local.project_id dataset_id = google_bigquery_dataset.default.dataset_id routine_id = each.key description = each.value.description diff --git a/modules/bigquery-dataset/tags.tf b/modules/bigquery-dataset/tags.tf index 55e9d33fe..6f6d75e86 100644 --- a/modules/bigquery-dataset/tags.tf +++ b/modules/bigquery-dataset/tags.tf @@ -17,6 +17,6 @@ resource "google_tags_location_tag_binding" "binding" { for_each = var.tag_bindings parent = "//bigquery.googleapis.com/${google_bigquery_dataset.default.id}" - tag_value = each.value - location = var.location + tag_value = lookup(local.ctx.tag_values, each.value, each.value) + location = lookup(local.ctx.locations, var.location, var.location) } diff --git a/modules/bigquery-dataset/variables.tf b/modules/bigquery-dataset/variables.tf index 77acff86d..ff6394cb0 100644 --- a/modules/bigquery-dataset/variables.tf +++ b/modules/bigquery-dataset/variables.tf @@ -65,6 +65,19 @@ variable "authorized_views" { default = [] } +variable "context" { + description = "Context-specific interpolations." + type = object({ + custom_roles = optional(map(string), {}) + iam_principals = optional(map(string), {}) + locations = optional(map(string), {}) + project_ids = optional(map(string), {}) + tag_values = optional(map(string), {}) + }) + default = {} + nullable = false +} + variable "dataset_access" { description = "Set access in the dataset resource instead of using separate resources." type = bool diff --git a/modules/project-factory/projects-bigquery.tf b/modules/project-factory/projects-bigquery.tf index 650261b0c..7f88337f8 100644 --- a/modules/project-factory/projects-bigquery.tf +++ b/modules/project-factory/projects-bigquery.tf @@ -33,12 +33,20 @@ module "bigquery-datasets" { for_each = { for k in local.projects_bigquery_datasets : "${k.project_key}/${k.id}" => k } - project_id = module.projects[each.value.project_key].project_id - id = each.value.id + project_id = module.projects[each.value.project_key].project_id + id = each.value.id + context = merge(local.ctx, { + project_ids = local.ctx_project_ids + iam_principals = merge( + local.ctx.iam_principals, + local.projects_sas_iam_emails, + local.automation_sas_iam_emails + ) + }) friendly_name = each.value.friendly_name location = coalesce( local.data_defaults.overrides.bigquery_location, lookup(each.value, "location", null), local.data_defaults.defaults.bigquery_location ) -} \ No newline at end of file +} diff --git a/tests/modules/bigquery_dataset/context.tfvars b/tests/modules/bigquery_dataset/context.tfvars new file mode 100644 index 000000000..b4805a158 --- /dev/null +++ b/tests/modules/bigquery_dataset/context.tfvars @@ -0,0 +1,37 @@ +context = { + custom_roles = { + myrole_one = "organizations/366118655033/roles/myRoleOne" + myrole_two = "organizations/366118655033/roles/myRoleTwo" + myrole_three = "organizations/366118655033/roles/myRoleThree" + myrole_four = "organizations/366118655033/roles/myRoleFour" + } + iam_principals = { + mygroup = "group:test-group@example.com" + mysa = "serviceAccount:test@test-project.iam.gserviceaccount.com" + myuser = "user:test-user@example.com" + myuser2 = "user:test-user2@example.com" + } + locations = { + ew8 = "europe-west8" + } + project_ids = { + test = "foo-test-0" + } + tag_values = { + "test/one" = "tagValues/1234567890" + } +} +project_id = "$project_ids:test" +id = "dataset_0" +location = "$locations:ew8" +iam = { + "$custom_roles:myrole_one" = [ + "$iam_principals:myuser" + ] + "roles/viewer" = [ + "$iam_principals:mysa" + ] +} +tag_bindings = { + foo = "$tag_values:test/one" +} diff --git a/tests/modules/bigquery_dataset/context.yaml b/tests/modules/bigquery_dataset/context.yaml new file mode 100644 index 000000000..a9f3d17f0 --- /dev/null +++ b/tests/modules/bigquery_dataset/context.yaml @@ -0,0 +1,60 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + google_bigquery_dataset.default: + dataset_id: dataset_0 + default_encryption_configuration: [] + default_partition_expiration_ms: null + default_table_expiration_ms: null + delete_contents_on_destroy: false + description: Terraform managed. + effective_labels: + goog-terraform-provisioned: 'true' + external_catalog_dataset_options: [] + external_dataset_reference: [] + friendly_name: null + labels: null + location: europe-west8 + max_time_travel_hours: '168' + project: foo-test-0 + resource_tags: null + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null + google_bigquery_dataset_iam_binding.bindings["$custom_roles:myrole_one"]: + condition: [] + dataset_id: dataset_0 + members: + - user:test-user@example.com + project: foo-test-0 + role: organizations/366118655033/roles/myRoleOne + google_bigquery_dataset_iam_binding.bindings["roles/viewer"]: + condition: [] + dataset_id: dataset_0 + members: + - serviceAccount:test@test-project.iam.gserviceaccount.com + project: foo-test-0 + role: roles/viewer + google_tags_location_tag_binding.binding["foo"]: + location: europe-west8 + tag_value: tagValues/1234567890 + timeouts: null + +counts: + google_bigquery_dataset: 1 + google_bigquery_dataset_iam_binding: 2 + google_tags_location_tag_binding: 1 + modules: 0 + resources: 4 diff --git a/tests/modules/bigquery_dataset/tftest.yaml b/tests/modules/bigquery_dataset/tftest.yaml new file mode 100644 index 000000000..27105ccff --- /dev/null +++ b/tests/modules/bigquery_dataset/tftest.yaml @@ -0,0 +1,17 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module: modules/bigquery-dataset +tests: + context: