diff --git a/.agents/rules/dev-workflow.md b/.agents/rules/dev-workflow.md new file mode 100644 index 000000000..62b697a9c --- /dev/null +++ b/.agents/rules/dev-workflow.md @@ -0,0 +1,10 @@ +--- +trigger: always_on +--- + +# Always make sure edited code passes linting checks + +- run `tools/tfdoc.py` if variable or output definitions changed +- run `terraform fmt` on any edited Terraform file, and hcl examples in README files +- a schema change should be reflected in all the other places that use the same schema, those are documented in `tools/duplicate-diff.py` +- always make sure both example and module (or stage) level tests run for all the modules/stages where code was edited \ No newline at end of file diff --git a/.agents/rules/fast-factory-driven.md b/.agents/rules/fast-factory-driven.md new file mode 100644 index 000000000..8334715f6 --- /dev/null +++ b/.agents/rules/fast-factory-driven.md @@ -0,0 +1,18 @@ +--- +trigger: always_on +--- + +# Use factory datasets for resource configuration + +**FAST stages should generally not implement factories, but leverage those defined in modules and their associated schemas.** + +Stages in the FAST folder are split between Terraform code and datasets. + +Code is used to implement and wire together "factories" that implement resource management, while the actual description of resources and their relationships is implemented via YAML configurations read by those factories. + +- YAML configurations are grouped in "datasets" which implement a complete design for the stage +- each factory has a reference JSON schema used to describe and validate the YAML data +- factories are generally implemented in the underlying modules, not in FAST stages +- modules deal with one specific resource set (eg an instance and its disks, a project and its org policies, IAM, etc.) and generally implement a single factory +- the project and VPC factory modules are the exception, as they are designed as "macro modules" to support interrelated creation of resources pertaining to a larger scope +- modules that do not manage "sets" of resources (e.g. one project, one folder, one dataset, etc.) typically do not have an associated factory, or only do for sub-resources (e.g. rules in a single policy), those factories are either implemented in the "macro modules" or directly in FAST diff --git a/.agents/rules/modules-as-resource-managers.md b/.agents/rules/modules-as-resource-managers.md new file mode 100644 index 000000000..cf0d72926 --- /dev/null +++ b/.agents/rules/modules-as-resource-managers.md @@ -0,0 +1,9 @@ +--- +trigger: always_on +--- + +Modules are designed to be containers for all aspects related to uage of a resource type. The pattern is a module is focused on a specific resource (e.g. folder, project, vpc, etc.) and implements all the functionality needed to create/manage that resources so that it is ready for user consumption. This includes: IAM, sub resources (eg subnets and routes for a network), org policies where applicable, etc. + +Unrelated resources like a dataset for a project should never be part of the same module, except in the two "aggregation modules" (project and vpc factory) where that makes sense to allow consumers to create baseline infrastructure ready to receive application-level resources. + +Never, ever break this boundary as a first approach, and always, always check in with the user if this looks like the only plan of action, as the criteria and constraints that led to the plan might need to be revised instead. \ No newline at end of file diff --git a/fast/stages/0-org-setup/schemas/project.schema.json b/fast/stages/0-org-setup/schemas/project.schema.json index 8d437b134..f1801d92d 100644 --- a/fast/stages/0-org-setup/schemas/project.schema.json +++ b/fast/stages/0-org-setup/schemas/project.schema.json @@ -244,6 +244,39 @@ }, "encryption_key": { "type": "string" + }, + "iam": { + "$ref": "#/$defs/iam" + }, + "iam_bindings": { + "$ref": "#/$defs/iam_bindings" + }, + "iam_bindings_additive": { + "$ref": "#/$defs/iam_bindings_additive" + }, + "iam_by_principals": { + "$ref": "#/$defs/iam_by_principals" + }, + "options": { + "type": "object", + "additionalProperties": false, + "properties": { + "default_table_expiration_ms": { + "type": "number" + }, + "default_partition_expiration_ms": { + "type": "number" + }, + "delete_contents_on_destroy": { + "type": "boolean" + }, + "max_time_travel_hours": { + "type": "number" + } + } + }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" } } } @@ -348,6 +381,9 @@ "iam_bindings_additive": { "$ref": "#/$defs/iam_bindings_additive" }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" + }, "keys": { "type": "object", "additionalProperties": false, @@ -651,6 +687,9 @@ }, "iam_sa_roles": { "$ref": "#/$defs/iam_sa_roles" + }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" } } } @@ -756,15 +795,6 @@ } } }, - "tag_bindings": { - "type": "object", - "additionalProperties": false, - "patternProperties": { - "^[a-z0-9_-]+$": { - "type": "string" - } - } - }, "tags": { "type": "object", "additionalProperties": { @@ -813,6 +843,15 @@ } } }, + "tag_bindings": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "string" + } + } + }, "universe": { "type": "object", "additionalProperties": false, @@ -1192,6 +1231,9 @@ }, "enable_object_retention": { "type": "boolean" + }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" } } }, @@ -1804,6 +1846,15 @@ } } } + }, + "tag_bindings": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "string" + } + } } } } \ No newline at end of file diff --git a/fast/stages/2-networking/schemas/project.schema.json b/fast/stages/2-networking/schemas/project.schema.json index 8d437b134..f1801d92d 100644 --- a/fast/stages/2-networking/schemas/project.schema.json +++ b/fast/stages/2-networking/schemas/project.schema.json @@ -244,6 +244,39 @@ }, "encryption_key": { "type": "string" + }, + "iam": { + "$ref": "#/$defs/iam" + }, + "iam_bindings": { + "$ref": "#/$defs/iam_bindings" + }, + "iam_bindings_additive": { + "$ref": "#/$defs/iam_bindings_additive" + }, + "iam_by_principals": { + "$ref": "#/$defs/iam_by_principals" + }, + "options": { + "type": "object", + "additionalProperties": false, + "properties": { + "default_table_expiration_ms": { + "type": "number" + }, + "default_partition_expiration_ms": { + "type": "number" + }, + "delete_contents_on_destroy": { + "type": "boolean" + }, + "max_time_travel_hours": { + "type": "number" + } + } + }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" } } } @@ -348,6 +381,9 @@ "iam_bindings_additive": { "$ref": "#/$defs/iam_bindings_additive" }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" + }, "keys": { "type": "object", "additionalProperties": false, @@ -651,6 +687,9 @@ }, "iam_sa_roles": { "$ref": "#/$defs/iam_sa_roles" + }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" } } } @@ -756,15 +795,6 @@ } } }, - "tag_bindings": { - "type": "object", - "additionalProperties": false, - "patternProperties": { - "^[a-z0-9_-]+$": { - "type": "string" - } - } - }, "tags": { "type": "object", "additionalProperties": { @@ -813,6 +843,15 @@ } } }, + "tag_bindings": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "string" + } + } + }, "universe": { "type": "object", "additionalProperties": false, @@ -1192,6 +1231,9 @@ }, "enable_object_retention": { "type": "boolean" + }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" } } }, @@ -1804,6 +1846,15 @@ } } } + }, + "tag_bindings": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "string" + } + } } } } \ No newline at end of file diff --git a/fast/stages/2-project-factory/schemas/project.schema.json b/fast/stages/2-project-factory/schemas/project.schema.json index 8d437b134..f1801d92d 100644 --- a/fast/stages/2-project-factory/schemas/project.schema.json +++ b/fast/stages/2-project-factory/schemas/project.schema.json @@ -244,6 +244,39 @@ }, "encryption_key": { "type": "string" + }, + "iam": { + "$ref": "#/$defs/iam" + }, + "iam_bindings": { + "$ref": "#/$defs/iam_bindings" + }, + "iam_bindings_additive": { + "$ref": "#/$defs/iam_bindings_additive" + }, + "iam_by_principals": { + "$ref": "#/$defs/iam_by_principals" + }, + "options": { + "type": "object", + "additionalProperties": false, + "properties": { + "default_table_expiration_ms": { + "type": "number" + }, + "default_partition_expiration_ms": { + "type": "number" + }, + "delete_contents_on_destroy": { + "type": "boolean" + }, + "max_time_travel_hours": { + "type": "number" + } + } + }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" } } } @@ -348,6 +381,9 @@ "iam_bindings_additive": { "$ref": "#/$defs/iam_bindings_additive" }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" + }, "keys": { "type": "object", "additionalProperties": false, @@ -651,6 +687,9 @@ }, "iam_sa_roles": { "$ref": "#/$defs/iam_sa_roles" + }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" } } } @@ -756,15 +795,6 @@ } } }, - "tag_bindings": { - "type": "object", - "additionalProperties": false, - "patternProperties": { - "^[a-z0-9_-]+$": { - "type": "string" - } - } - }, "tags": { "type": "object", "additionalProperties": { @@ -813,6 +843,15 @@ } } }, + "tag_bindings": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "string" + } + } + }, "universe": { "type": "object", "additionalProperties": false, @@ -1192,6 +1231,9 @@ }, "enable_object_retention": { "type": "boolean" + }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" } } }, @@ -1804,6 +1846,15 @@ } } } + }, + "tag_bindings": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "string" + } + } } } } \ No newline at end of file diff --git a/fast/stages/2-security/schemas/project.schema.json b/fast/stages/2-security/schemas/project.schema.json index 8d437b134..f1801d92d 100644 --- a/fast/stages/2-security/schemas/project.schema.json +++ b/fast/stages/2-security/schemas/project.schema.json @@ -244,6 +244,39 @@ }, "encryption_key": { "type": "string" + }, + "iam": { + "$ref": "#/$defs/iam" + }, + "iam_bindings": { + "$ref": "#/$defs/iam_bindings" + }, + "iam_bindings_additive": { + "$ref": "#/$defs/iam_bindings_additive" + }, + "iam_by_principals": { + "$ref": "#/$defs/iam_by_principals" + }, + "options": { + "type": "object", + "additionalProperties": false, + "properties": { + "default_table_expiration_ms": { + "type": "number" + }, + "default_partition_expiration_ms": { + "type": "number" + }, + "delete_contents_on_destroy": { + "type": "boolean" + }, + "max_time_travel_hours": { + "type": "number" + } + } + }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" } } } @@ -348,6 +381,9 @@ "iam_bindings_additive": { "$ref": "#/$defs/iam_bindings_additive" }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" + }, "keys": { "type": "object", "additionalProperties": false, @@ -651,6 +687,9 @@ }, "iam_sa_roles": { "$ref": "#/$defs/iam_sa_roles" + }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" } } } @@ -756,15 +795,6 @@ } } }, - "tag_bindings": { - "type": "object", - "additionalProperties": false, - "patternProperties": { - "^[a-z0-9_-]+$": { - "type": "string" - } - } - }, "tags": { "type": "object", "additionalProperties": { @@ -813,6 +843,15 @@ } } }, + "tag_bindings": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "string" + } + } + }, "universe": { "type": "object", "additionalProperties": false, @@ -1192,6 +1231,9 @@ }, "enable_object_retention": { "type": "boolean" + }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" } } }, @@ -1804,6 +1846,15 @@ } } } + }, + "tag_bindings": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "string" + } + } } } } \ No newline at end of file diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index cd9433052..633ba60db 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -55,8 +55,14 @@ module "bigquery-dataset" { iam = { "roles/bigquery.dataOwner" = ["user:user1@example.org"] } + iam_bindings = { + reader_user = { + role = "roles/bigquery.dataViewer" + members = ["user:user2@example.org"] + } + } } -# tftest modules=1 resources=2 inventory=iam.yaml +# tftest modules=1 resources=3 inventory=iam.yaml ``` ## Authorized Views, Datasets, and Routines @@ -353,27 +359,30 @@ module "bigquery-dataset" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [id](variables.tf#L112) | Dataset id. | string | ✓ | | -| [project_id](variables.tf#L176) | Id of the project where datasets will be created. | string | ✓ | | +| [id](variables.tf#L111) | Dataset id. | string | ✓ | | +| [project_id](variables.tf#L175) | Id of the project where datasets will be created. | string | ✓ | | | [access](variables.tf#L17) | Map of access rules with role and identity type. Keys are arbitrary and must match those in the `access_identities` variable, types are `domain`, `group`, `special_group`, `user`, `view`. | map(object({…})) | | {} | | [access_identities](variables.tf#L33) | Map of access identities used for basic access roles. View identities have the format 'project_id\|dataset_id\|table_id'. | map(string) | | {} | | [authorized_datasets](variables.tf#L39) | An array of datasets to be authorized on the dataset. | list(object({…})) | | [] | | [authorized_routines](variables.tf#L48) | An array of routines to be authorized on the dataset. | list(object({…})) | | [] | | [authorized_views](variables.tf#L58) | An array of views to be authorized on the dataset. | list(object({…})) | | [] | -| [context](variables.tf#L68) | Context-specific interpolations. | object({…}) | | {} | -| [dataset_access](variables.tf#L82) | Set access in the dataset resource instead of using separate resources. | bool | | false | -| [description](variables.tf#L88) | Optional description. | string | | "Terraform managed." | -| [encryption_key](variables.tf#L94) | Self link of the KMS key that will be used to protect destination table. | string | | null | -| [friendly_name](variables.tf#L100) | Dataset friendly name. | string | | null | -| [iam](variables.tf#L106) | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | map(list(string)) | | {} | -| [labels](variables.tf#L117) | Dataset labels. | map(string) | | {} | -| [location](variables.tf#L123) | Dataset location. | string | | "EU" | -| [materialized_views](variables.tf#L129) | Materialized views definitions. | map(object({…})) | | {} | -| [options](variables.tf#L162) | Dataset options. | object({…}) | | {} | -| [routines](variables.tf#L181) | Routine definitions. | map(object({…})) | | {} | -| [tables](variables.tf#L220) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…})) | | {} | -| [tag_bindings](variables.tf#L305) | Tag bindings for this dataset, in key => tag value id format. | map(string) | | {} | -| [views](variables.tf#L312) | View definitions. | map(object({…})) | | {} | +| [context](variables.tf#L68) | Context-specific interpolations. | object({…}) | | {} | +| [dataset_access](variables.tf#L83) | Set access in the dataset resource instead of using separate resources. | bool | | false | +| [description](variables.tf#L89) | Optional description. | string | | "Terraform managed." | +| [encryption_key](variables.tf#L95) | Self link of the KMS key that will be used to protect destination table. | string | | null | +| [friendly_name](variables.tf#L101) | Dataset friendly name. | string | | null | +| [iam](variables-iam.tf#L17) | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | map(list(string)) | | {} | +| [iam_bindings](variables-iam.tf#L23) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_bindings_additive](variables-iam.tf#L38) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_by_principals](variables-iam.tf#L53) | Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid errors. Merged internally with the `iam` variable. | map(list(string)) | | {} | +| [labels](variables.tf#L116) | Dataset labels. | map(string) | | {} | +| [location](variables.tf#L122) | Dataset location. | string | | "EU" | +| [materialized_views](variables.tf#L128) | Materialized views definitions. | map(object({…})) | | {} | +| [options](variables.tf#L161) | Dataset options. | object({…}) | | {} | +| [routines](variables.tf#L180) | Routine definitions. | map(object({…})) | | {} | +| [tables](variables.tf#L219) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…})) | | {} | +| [tag_bindings](variables.tf#L304) | Tag bindings for this dataset, in key => tag value id format. | map(string) | | {} | +| [views](variables.tf#L311) | View definitions. | map(object({…})) | | {} | ## Outputs diff --git a/modules/bigquery-dataset/iam.tf b/modules/bigquery-dataset/iam.tf new file mode 100644 index 000000000..045666d9a --- /dev/null +++ b/modules/bigquery-dataset/iam.tf @@ -0,0 +1,82 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description IAM bindings. + +locals { + _iam_principal_roles = distinct(flatten(values(var.iam_by_principals))) + _iam_principals = { + for r in local._iam_principal_roles : r => [ + for k, v in var.iam_by_principals : + k if try(index(v, r), null) != null + ] + } + iam = { + for role in distinct(concat(keys(var.iam), keys(local._iam_principals))) : + role => concat( + try(var.iam[role], []), + try(local._iam_principals[role], []) + ) + } +} + +resource "google_bigquery_dataset_iam_binding" "authoritative" { + for_each = local.iam + project = local.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + role = lookup(local.ctx.custom_roles, each.key, each.key) + members = [ + for v in each.value : lookup(local.ctx.iam_principals, v, v) + ] +} + +resource "google_bigquery_dataset_iam_binding" "bindings" { + for_each = var.iam_bindings + project = local.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + role = lookup(local.ctx.custom_roles, each.value.role, each.value.role) + members = [ + for v in each.value.members : lookup(local.ctx.iam_principals, v, v) + ] + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = templatestring( + each.value.condition.expression, var.context.condition_vars + ) + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_bigquery_dataset_iam_member" "bindings" { + for_each = var.iam_bindings_additive + project = local.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + role = lookup(local.ctx.custom_roles, each.value.role, each.value.role) + member = lookup(local.ctx.iam_principals, each.value.member, each.value.member) + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = templatestring( + each.value.condition.expression, var.context.condition_vars + ) + title = each.value.condition.title + description = each.value.condition.description + } + } +} diff --git a/modules/bigquery-dataset/main.tf b/modules/bigquery-dataset/main.tf index 20a5fd520..83e4fd952 100644 --- a/modules/bigquery-dataset/main.tf +++ b/modules/bigquery-dataset/main.tf @@ -231,15 +231,6 @@ resource "google_bigquery_dataset_access" "authorized_routines" { } } -resource "google_bigquery_dataset_iam_binding" "bindings" { - for_each = var.iam - project = local.project_id - dataset_id = google_bigquery_dataset.default.dataset_id - role = lookup(local.ctx.custom_roles, each.key, each.key) - members = [ - for v in each.value : lookup(local.ctx.iam_principals, v, v) - ] -} resource "google_bigquery_table" "default" { provider = google-beta diff --git a/modules/bigquery-dataset/variables-iam.tf b/modules/bigquery-dataset/variables-iam.tf new file mode 100644 index 000000000..3a7045f4e --- /dev/null +++ b/modules/bigquery-dataset/variables-iam.tf @@ -0,0 +1,58 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "iam" { + description = "IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles." + type = map(list(string)) + default = {} +} + +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "iam_by_principals" { + description = "Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid errors. Merged internally with the `iam` variable." + type = map(list(string)) + default = {} + nullable = false +} diff --git a/modules/bigquery-dataset/variables.tf b/modules/bigquery-dataset/variables.tf index dac68063f..58a570c8b 100644 --- a/modules/bigquery-dataset/variables.tf +++ b/modules/bigquery-dataset/variables.tf @@ -68,6 +68,7 @@ variable "authorized_views" { variable "context" { description = "Context-specific interpolations." type = object({ + condition_vars = optional(map(map(string)), {}) custom_roles = optional(map(string), {}) kms_keys = optional(map(string), {}) iam_principals = optional(map(string), {}) @@ -103,11 +104,9 @@ variable "friendly_name" { default = null } -variable "iam" { - description = "IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles." - type = map(list(string)) - default = {} -} + + + variable "id" { description = "Dataset id." diff --git a/modules/project-factory/README.md b/modules/project-factory/README.md index 394cc279a..47a9b7abd 100644 --- a/modules/project-factory/README.md +++ b/modules/project-factory/README.md @@ -442,7 +442,8 @@ module "project-factory" { data_defaults = { billing_account = var.billing_account_id locations = { - storage = "EU" + bigquery = "EU" + storage = "EU" } } # make sure the environment label and stackdriver service are always added @@ -557,7 +558,6 @@ asset_feeds: name: App 0 factories_config: org_policies: data/factories/org-policies - pam_entitlements: app-0-admins: max_request_duration: 3600s @@ -639,6 +639,8 @@ service_accounts: iam_self_roles: - roles/logging.logWriter - roles/monitoring.metricWriter + tag_bindings: + context: $tag_values:context/project-factory # this is just for illustrative/test purposes iam: roles/iam.serviceAccountUser: @@ -672,6 +674,8 @@ billing_budgets: buckets: app-0-bucket-a: location: europe-west8 + tag_bindings: + context: $tag_values:context/gke app-0-bucket-b: location: europe-west8 logging_config: @@ -695,6 +699,12 @@ services: - container.googleapis.com - pubsub.googleapis.com - storage.googleapis.com +datasets: + test_0: + friendly_name: Test Dataset + iam: + roles/bigquery.dataViewer: + - $iam_principals:gcp-devops pubsub_topics: app-0-topic-a: iam: @@ -703,6 +713,14 @@ pubsub_topics: app-0-topic-b: subscriptions: app-0-topic-b-sub: {} +kms: + keyrings: + my-keyring: + location: europe-west1 + keys: + my-key: {} + tag_bindings: + context: $tag_values:context/project-factory tags: my-tag-key-1: values: diff --git a/modules/project-factory/projects-bigquery.tf b/modules/project-factory/projects-bigquery.tf index eb4d8dfc3..fd6590e12 100644 --- a/modules/project-factory/projects-bigquery.tf +++ b/modules/project-factory/projects-bigquery.tf @@ -18,12 +18,26 @@ locals { projects_bigquery_datasets = flatten([ for k, v in local.projects_input : [ for name, opts in lookup(v, "datasets", {}) : { - project_key = k - project_name = v.name - id = name - encryption_key = lookup(opts, "encryption_key", null) - friendly_name = lookup(opts, "friendly_name", null) - location = lookup(opts, "location", null) + project_key = k + project_name = v.name + id = name + encryption_key = lookup(opts, "encryption_key", null) + friendly_name = lookup(opts, "friendly_name", null) + location = lookup(opts, "location", null) + iam = lookup(opts, "iam", {}) + iam_bindings = lookup(opts, "iam_bindings", {}) + iam_bindings_additive = lookup(opts, "iam_bindings_additive", {}) + iam_by_principals = lookup(opts, "iam_by_principals", {}) + tag_bindings = lookup(opts, "tag_bindings", {}) + options = { + default_collation = try(opts.options.default_collation, null) + default_table_expiration_ms = try(opts.options.default_table_expiration_ms, null) + default_partition_expiration_ms = try(opts.options.default_partition_expiration_ms, null) + delete_contents_on_destroy = try(opts.options.delete_contents_on_destroy, null) + is_case_insensitive = try(opts.options.is_case_insensitive, null) + max_time_travel_hours = try(opts.options.max_time_travel_hours, null) + storage_billing_model = try(opts.options.storage_billing_model, null) + } } ] ]) @@ -46,6 +60,8 @@ module "bigquery-datasets" { kms_keys = merge(local.ctx.kms_keys, local.kms_keys, local.kms_autokeys) locations = local.ctx.locations project_ids = local.ctx_project_ids + tag_keys = local.ctx_tag_keys + tag_values = local.ctx_tag_values }) encryption_key = each.value.encryption_key friendly_name = each.value.friendly_name @@ -54,4 +70,10 @@ module "bigquery-datasets" { lookup(each.value, "location", null), local.data_defaults.defaults.locations.bigquery ) + iam = each.value.iam + iam_bindings = each.value.iam_bindings + iam_bindings_additive = each.value.iam_bindings_additive + iam_by_principals = each.value.iam_by_principals + tag_bindings = each.value.tag_bindings + options = each.value.options } diff --git a/modules/project-factory/projects-buckets.tf b/modules/project-factory/projects-buckets.tf index 2aecfafab..fb0e1c47d 100644 --- a/modules/project-factory/projects-buckets.tf +++ b/modules/project-factory/projects-buckets.tf @@ -56,6 +56,7 @@ locals { lifecycle_rules = lookup(opts, "lifecycle_rules", {}) logging_config = lookup(opts, "logging_config", null) enable_object_retention = lookup(opts, "enable_object_retention", null) + tag_bindings = lookup(opts, "tag_bindings", {}) } ] ]) @@ -81,6 +82,8 @@ module "buckets" { locations = local.ctx.locations project_ids = local.ctx_project_ids storage_buckets = local.ctx.storage_buckets + tag_keys = local.ctx_tag_keys + tag_values = local.ctx_tag_values }) iam = each.value.iam iam_bindings = each.value.iam_bindings @@ -101,4 +104,5 @@ module "buckets" { soft_delete_retention = each.value.soft_delete_retention logging_config = each.value.logging_config enable_object_retention = each.value.enable_object_retention + tag_bindings = each.value.tag_bindings } diff --git a/modules/project-factory/projects-kms.tf b/modules/project-factory/projects-kms.tf index 56b3c649f..3550cdb5b 100644 --- a/modules/project-factory/projects-kms.tf +++ b/modules/project-factory/projects-kms.tf @@ -25,6 +25,7 @@ locals { iam = lookup(opts, "iam", {}) iam_bindings = lookup(opts, "iam_bindings", {}) iam_bindings_additive = lookup(opts, "iam_bindings_additive", {}) + tag_bindings = lookup(opts, "tag_bindings", {}) keys = lookup(opts, "keys", {}) } if try(opts.location, null) != null ] @@ -64,6 +65,7 @@ module "kms" { iam = each.value.iam iam_bindings = each.value.iam_bindings iam_bindings_additive = each.value.iam_bindings_additive + tag_bindings = each.value.tag_bindings keys = each.value.keys context = merge(local.ctx, { iam_principals = merge( @@ -75,5 +77,7 @@ module "kms" { ) locations = local.ctx.locations project_ids = local.ctx_project_ids + tag_keys = local.ctx_tag_keys + tag_values = local.ctx_tag_values }) } diff --git a/modules/project-factory/projects-service-accounts.tf b/modules/project-factory/projects-service-accounts.tf index afbdc71ef..6247ee87d 100644 --- a/modules/project-factory/projects-service-accounts.tf +++ b/modules/project-factory/projects-service-accounts.tf @@ -40,6 +40,7 @@ locals { try(local.data_defaults.defaults.service_accounts.iam_self_roles, []), )) iam_storage_roles = try(opts.iam_storage_roles, {}) + tag_bindings = try(opts.tag_bindings, {}) opts = opts } ] @@ -85,6 +86,7 @@ module "service-accounts" { display_name = each.value.display_name context = merge(local.ctx, { project_ids = local.ctx_project_ids + tag_values = local.ctx_tag_values }) iam_project_roles = merge( each.value.iam_project_roles, @@ -92,6 +94,7 @@ module "service-accounts" { "$project_ids:${each.value.project_key}" = each.value.iam_self_roles } ) + tag_bindings = each.value.tag_bindings } module "service_accounts-iam" { diff --git a/modules/project-factory/schemas/project.schema.json b/modules/project-factory/schemas/project.schema.json index 8d437b134..f1801d92d 100644 --- a/modules/project-factory/schemas/project.schema.json +++ b/modules/project-factory/schemas/project.schema.json @@ -244,6 +244,39 @@ }, "encryption_key": { "type": "string" + }, + "iam": { + "$ref": "#/$defs/iam" + }, + "iam_bindings": { + "$ref": "#/$defs/iam_bindings" + }, + "iam_bindings_additive": { + "$ref": "#/$defs/iam_bindings_additive" + }, + "iam_by_principals": { + "$ref": "#/$defs/iam_by_principals" + }, + "options": { + "type": "object", + "additionalProperties": false, + "properties": { + "default_table_expiration_ms": { + "type": "number" + }, + "default_partition_expiration_ms": { + "type": "number" + }, + "delete_contents_on_destroy": { + "type": "boolean" + }, + "max_time_travel_hours": { + "type": "number" + } + } + }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" } } } @@ -348,6 +381,9 @@ "iam_bindings_additive": { "$ref": "#/$defs/iam_bindings_additive" }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" + }, "keys": { "type": "object", "additionalProperties": false, @@ -651,6 +687,9 @@ }, "iam_sa_roles": { "$ref": "#/$defs/iam_sa_roles" + }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" } } } @@ -756,15 +795,6 @@ } } }, - "tag_bindings": { - "type": "object", - "additionalProperties": false, - "patternProperties": { - "^[a-z0-9_-]+$": { - "type": "string" - } - } - }, "tags": { "type": "object", "additionalProperties": { @@ -813,6 +843,15 @@ } } }, + "tag_bindings": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "string" + } + } + }, "universe": { "type": "object", "additionalProperties": false, @@ -1192,6 +1231,9 @@ }, "enable_object_retention": { "type": "boolean" + }, + "tag_bindings": { + "$ref": "#/$defs/tag_bindings" } } }, @@ -1804,6 +1846,15 @@ } } } + }, + "tag_bindings": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "string" + } + } } } } \ No newline at end of file diff --git a/tests/modules/bigquery_dataset/context.yaml b/tests/modules/bigquery_dataset/context.yaml index ef273fb32..6550af94b 100644 --- a/tests/modules/bigquery_dataset/context.yaml +++ b/tests/modules/bigquery_dataset/context.yaml @@ -34,14 +34,14 @@ values: terraform_labels: goog-terraform-provisioned: 'true' timeouts: null - google_bigquery_dataset_iam_binding.bindings["$custom_roles:myrole_one"]: + google_bigquery_dataset_iam_binding.authoritative["$custom_roles:myrole_one"]: condition: [] dataset_id: dataset_0 members: - user:test-user@example.com project: foo-test-0 role: organizations/366118655033/roles/myRoleOne - google_bigquery_dataset_iam_binding.bindings["roles/viewer"]: + google_bigquery_dataset_iam_binding.authoritative["roles/viewer"]: condition: [] dataset_id: dataset_0 members: diff --git a/tests/modules/bigquery_dataset/examples/iam.yaml b/tests/modules/bigquery_dataset/examples/iam.yaml index a8607d97a..0d64f6e29 100644 --- a/tests/modules/bigquery_dataset/examples/iam.yaml +++ b/tests/modules/bigquery_dataset/examples/iam.yaml @@ -15,15 +15,39 @@ values: module.bigquery-dataset.google_bigquery_dataset.default: dataset_id: my_dataset + default_encryption_configuration: [] + default_partition_expiration_ms: null + default_table_expiration_ms: null + delete_contents_on_destroy: false + description: Terraform managed. + effective_labels: + goog-terraform-provisioned: 'true' + external_catalog_dataset_options: [] + external_dataset_reference: [] + friendly_name: null + labels: null + location: EU + max_time_travel_hours: '168' project: my-project - module.bigquery-dataset.google_bigquery_dataset_iam_binding.bindings["roles/bigquery.dataOwner"]: + resource_tags: null + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null + module.bigquery-dataset.google_bigquery_dataset_iam_binding.authoritative["roles/bigquery.dataOwner"]: condition: [] dataset_id: my_dataset members: - user:user1@example.org project: my-project role: roles/bigquery.dataOwner + module.bigquery-dataset.google_bigquery_dataset_iam_binding.bindings["reader_user"]: + condition: [] + dataset_id: my_dataset + members: + - user:user2@example.org + project: my-project + role: roles/bigquery.dataViewer counts: google_bigquery_dataset: 1 - google_bigquery_dataset_iam_binding: 1 + google_bigquery_dataset_iam_binding: 2 diff --git a/tests/modules/project_factory/examples/example.yaml b/tests/modules/project_factory/examples/example.yaml index b38126e39..ccaaa15bc 100644 --- a/tests/modules/project_factory/examples/example.yaml +++ b/tests/modules/project_factory/examples/example.yaml @@ -81,6 +81,33 @@ values: member: serviceAccount:dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com project: test-pf-teams-iac-0 timeouts: null + module.project-factory.module.bigquery-datasets["dev-ta-app0-be/test_0"].google_bigquery_dataset.default: + dataset_id: test_0 + default_encryption_configuration: [] + default_partition_expiration_ms: null + default_table_expiration_ms: null + delete_contents_on_destroy: false + description: Terraform managed. + effective_labels: + goog-terraform-provisioned: 'true' + external_catalog_dataset_options: [] + external_dataset_reference: [] + friendly_name: Test Dataset + labels: null + location: EU + max_time_travel_hours: '168' + project: test-pf-dev-ta-app0-be + resource_tags: null + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null + ? module.project-factory.module.bigquery-datasets["dev-ta-app0-be/test_0"].google_bigquery_dataset_iam_binding.authoritative["roles/bigquery.dataViewer"] + : condition: [] + dataset_id: test_0 + members: + - group:gcp-devops@example.org + project: test-pf-dev-ta-app0-be + role: roles/bigquery.dataViewer module.project-factory.module.billing-budgets[0].google_billing_budget.default["test-100"]: all_updates_rule: - disable_default_iam_recipients: true @@ -148,6 +175,11 @@ values: uniform_bucket_level_access: true versioning: - enabled: false + module.project-factory.module.buckets["dev-ta-app0-be/app-0-bucket-a"].google_tags_location_tag_binding.binding["context"]: + location: europe-west8 + parent: //storage.googleapis.com/projects/_/buckets/test-pf-dev-ta-app0-be-app-0-bucket-a + tag_value: tagValues/654321 + timeouts: null module.project-factory.module.buckets["dev-ta-app0-be/app-0-bucket-b"].google_storage_bucket.bucket[0]: autoclass: [] cors: [] @@ -288,6 +320,26 @@ values: display_name: App X tags: null timeouts: null + module.project-factory.module.kms["dev-ta-app0-be/my-keyring"].google_kms_crypto_key.default["my-key"]: + effective_labels: + goog-terraform-provisioned: 'true' + labels: null + name: my-key + purpose: ENCRYPT_DECRYPT + rotation_period: null + skip_initial_version_creation: false + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null + module.project-factory.module.kms["dev-ta-app0-be/my-keyring"].google_kms_key_ring.default[0]: + location: europe-west1 + name: my-keyring + project: test-pf-dev-ta-app0-be + timeouts: null + module.project-factory.module.kms["dev-ta-app0-be/my-keyring"].google_tags_location_tag_binding.binding["context"]: + location: europe-west1 + tag_value: $tag_values:context/project-factory + timeouts: null ? module.project-factory.module.projects-iam["dev-ta-app0-be"].google_compute_shared_vpc_service_project.shared_vpc_service[0] : deletion_policy: null host_project: $project_ids:dev-spoke-0 @@ -819,6 +871,9 @@ values: member: serviceAccount:app-0-be@test-pf-dev-ta-app0-be.iam.gserviceaccount.com project: test-pf-dev-ta-app0-be timeouts: null + module.project-factory.module.service-accounts["dev-ta-app0-be/app-0-be"].google_tags_tag_binding.binding["context"]: + tag_value: $tag_values:context/project-factory + timeouts: null ? module.project-factory.module.service-accounts["dev-ta-app0-be/app-0-fe"].google_project_iam_member.project-roles["$project_ids:dev-spoke-0-roles/compute.networkUser"] : condition: [] project: $project_ids:dev-spoke-0 @@ -905,6 +960,8 @@ values: triggers_replace: null counts: + google_bigquery_dataset: 1 + google_bigquery_dataset_iam_binding: 1 google_billing_budget: 1 google_cloud_asset_folder_feed: 1 google_compute_shared_vpc_host_project: 1 @@ -915,7 +972,9 @@ counts: google_folder_iam_binding: 1 google_iam_workload_identity_pool: 1 google_iam_workload_identity_pool_provider: 1 + google_kms_crypto_key: 1 google_kms_crypto_key_iam_member: 2 + google_kms_key_ring: 1 google_monitoring_notification_channel: 1 google_org_policy_policy: 3 google_privileged_access_manager_entitlement: 2 @@ -934,10 +993,11 @@ counts: google_storage_bucket: 3 google_storage_bucket_iam_binding: 2 google_storage_project_service_account: 4 - google_tags_tag_binding: 2 + google_tags_location_tag_binding: 2 + google_tags_tag_binding: 3 google_tags_tag_key: 1 google_tags_tag_value: 2 google_tags_tag_value_iam_binding: 1 - modules: 32 - resources: 111 + modules: 34 + resources: 118 terraform_data: 2