diff --git a/README.md b/README.md index 432c71596..a40a47d0d 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Currently available modules: - **foundational** - [billing account](./modules/billing-account), [Cloud Identity group](./modules/cloud-identity-group/), [folder](./modules/folder), [service accounts](./modules/iam-service-account), [logging bucket](./modules/logging-bucket), [organization](./modules/organization), [project](./modules/project), [projects-data-source](./modules/projects-data-source) - **networking** - [DNS](./modules/dns), [DNS Response Policy](./modules/dns-response-policy/), [Cloud Endpoints](./modules/endpoints), [address reservation](./modules/net-address), [NAT](./modules/net-cloudnat), [VLAN Attachment](./modules/net-vlan-attachment/), [External Application LB](./modules/net-lb-app-ext/), [External Passthrough Network LB](./modules/net-lb-ext), [External Regional Application Load Balancer](./modules/net-lb-app-ext-regional/), [Firewall policy](./modules/net-firewall-policy), [Internal Application LB](./modules/net-lb-app-int), [Cross-region Internal Application LB](./modules/net-lb-app-int-cross-region), [Internal Passthrough Network LB](./modules/net-lb-int), [Internal Proxy Network LB](./modules/net-lb-proxy-int), [IPSec over Interconnect](./modules/net-ipsec-over-interconnect), [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC peering](./modules/net-vpc-peering), [VPN dynamic](./modules/net-vpn-dynamic), [HA VPN](./modules/net-vpn-ha), [VPN static](./modules/net-vpn-static), [Service Directory](./modules/service-directory), [Secure Web Proxy](./modules/net-swp) - **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [COS container](./modules/cloud-config-container/cos-generic-metadata/) (coredns, mysql, onprem, squid), [GKE cluster](./modules/gke-cluster-standard), [GKE hub](./modules/gke-hub), [GKE nodepool](./modules/gke-nodepool), [GCVE private cloud](./modules/gcve-private-cloud) -- **data** - [BigQuery dataset](./modules/bigquery-dataset), [Bigtable instance](./modules/bigtable-instance), [Dataplex](./modules/dataplex), [Dataplex DataScan](./modules/dataplex-datascan/), [Cloud SQL instance](./modules/cloudsql-instance), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag), [Datafusion](./modules/datafusion), [Dataproc](./modules/dataproc), [GCS](./modules/gcs), [Pub/Sub](./modules/pubsub), [Dataform Repository](./modules/dataform-repository/) +- **data** - [BigQuery dataset](./modules/bigquery-dataset), [Bigtable instance](./modules/bigtable-instance), [Dataplex](./modules/dataplex), [Dataplex DataScan](./modules/dataplex-datascan/), [Cloud SQL instance](./modules/cloudsql-instance), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag), [Data Catalog Tag Template](./modules/data-catalog-tag-template), [Datafusion](./modules/datafusion), [Dataproc](./modules/dataproc), [GCS](./modules/gcs), [Pub/Sub](./modules/pubsub), [Dataform Repository](./modules/dataform-repository/) - **development** - [API Gateway](./modules/api-gateway), [Apigee](./modules/apigee), [Artifact Registry](./modules/artifact-registry), [Container Registry](./modules/container-registry), [Cloud Source Repository](./modules/source-repository), [Workstation cluster](./modules/workstation-cluster) - **security** - [Binauthz](./modules/binauthz/), [KMS](./modules/kms), [SecretManager](./modules/secret-manager), [VPC Service Control](./modules/vpc-sc) - **serverless** - [Cloud Function v1](./modules/cloud-function-v1), [Cloud Function v2](./modules/cloud-function-v2), [Cloud Run](./modules/cloud-run), [Cloud Run v2](./modules/cloud-run-v2) diff --git a/modules/README.md b/modules/README.md index cf5750959..2fee21135 100644 --- a/modules/README.md +++ b/modules/README.md @@ -77,15 +77,16 @@ These modules are used in the examples included in this repository. If you are u - [BigQuery dataset](./bigquery-dataset) - [Bigtable instance](./bigtable-instance) -- [Dataplex](./dataplex) -- [Dataplex DataScan](./dataplex-datascan/) - [Cloud SQL instance](./cloudsql-instance) - [Data Catalog Policy Tag](./data-catalog-policy-tag) +- [Data Catalog Tag Template](./data-catalog-tag-template) +- [Dataform Repository](./dataform-repository/) - [Datafusion](./datafusion) +- [Dataplex](./dataplex) +- [Dataplex DataScan](./dataplex-datascan/) - [Dataproc](./dataproc) - [GCS](./gcs) - [Pub/Sub](./pubsub) -- [Dataform Repository](./dataform-repository/) ## Development diff --git a/modules/data-catalog-tag-template/README.md b/modules/data-catalog-tag-template/README.md new file mode 100644 index 000000000..cc49be3ed --- /dev/null +++ b/modules/data-catalog-tag-template/README.md @@ -0,0 +1,221 @@ +# Google Cloud Data Catalog Tag Template Module + +This module allows managing [Data Catalog Tag Templates](https://cloud.google.com/data-catalog/docs/tags-and-tag-templates). + +## Examples + +### Simple Tag Template + +```hcl +module "data-catalog-tag-template" { + source = "./fabric/modules/data-catalog-tag-template" + project_id = "my-project" + tag_templates = { + demo_var = { + tag_template_id = "my_template" + region = "europe-west1" + display_name = "Demo Tag Template" + fields = { + source = { + display_name = "Source of data asset" + type = { + primitive_type = "STRING" + } + is_required = true + } + } + } + } +} +# tftest modules=1 resources=1 +``` + +### Tag Template with IAM + +```hcl +module "data-catalog-tag-template" { + source = "./fabric/modules/data-catalog-tag-template" + project_id = "my-project" + tag_templates = { + demo_var = { + tag_template_id = "my_template" + region = "europe-west1" + display_name = "Demo Tag Template" + fields = { + source = { + display_name = "Source of data asset" + type = { + primitive_type = "STRING" + } + is_required = true + } + } + } + } + iam = { + "roles/datacatalog.tagTemplateOwner" = ["group:data-governance@example.com"] + "roles/datacatalog.tagTemplateUser" = ["group:data-product-eng@example.com"] + } +} +# tftest modules=1 resources=3 +``` + +```hcl +module "data-catalog-tag-template" { + source = "./fabric/modules/data-catalog-tag-template" + project_id = var.project_id + tag_templates = { + demo_var = { + tag_template_id = "my_template" + region = "europe-west1" + display_name = "Demo Tag Template" + fields = { + source = { + display_name = "Source of data asset" + type = { + primitive_type = "STRING" + } + is_required = true + } + } + } + } + iam_bindings = { + admin-with-delegated_roles = { + role = "roles/datacatalog.tagTemplateOwner" + members = ["group:data-governance@example.com"] + condition = { + title = "delegated-role-grants" + expression = format( + "api.getAttribute('iam.googleapis.com/modifiedGrantsByRole', []).hasOnly([%s])", + join(",", formatlist("'%s'", + [ + "roles/datacatalog.tagTemplateOwner" + ] + )) + ) + } + } + } +} +# tftest modules=1 resources=2 +``` + +```hcl +module "data-catalog-tag-template" { + source = "./fabric/modules/data-catalog-tag-template" + project_id = var.project_id + tag_templates = { + demo_var = { + tag_template_id = "my_template" + region = "europe-west1" + display_name = "Demo Tag Template" + fields = { + source = { + display_name = "Source of data asset" + type = { + primitive_type = "STRING" + } + is_required = true + } + } + } + } + iam_bindings_additive = { + admin-with-delegated_roles = { + role = "roles/datacatalog.tagTemplateOwner" + member = "group:data-governance@example.com" + condition = { + title = "delegated-role-grants" + expression = format( + "api.getAttribute('iam.googleapis.com/modifiedGrantsByRole', []).hasOnly([%s])", + join(",", formatlist("'%s'", + [ + "roles/datacatalog.tagTemplateOwner" + ] + )) + ) + } + } + } +} +# tftest modules=1 resources=2 +``` + +### Factory + +Similarly to other modules, a rules factory (see [Resource Factories](../../blueprints/factories/)) is also included here to allow tag template management via descriptive configuration files. + +Factory configuration is via one optional attributes in the `factory_config_path` variable specifying the path where tag template files are stored. + +Factory tag templates are merged with rules declared in code, with the latter taking precedence where both use the same key. + +The name of the file will be used as `tag_template_id` field. + +This is an example of a simple factory: + +```hcl +module "data-catalog-tag-template" { + source = "./fabric/modules/data-catalog-tag-template" + project_id = "my-project" + tag_templates = { + demo_var = { + tag_template_id = "my_template" + region = "europe-west1" + display_name = "Demo Tag Template" + fields = { + source = { + display_name = "Source of data asset" + type = { + primitive_type = "STRING" + } + is_required = true + } + } + } + } + factories_config = { + tag_templates = "data" + } +} +# tftest modules=1 resources=2 files=demo_tag +``` + +```yaml +# tftest-file id=demo_tag path=data/demo.yaml + +region: europe-west2 +display_name: Demo Tag Template +fields: + - field_id: source + display_name: Source of data asset + type: + primitive_type: STRING + is_required: true + - field_id: pii_type + display_name: PII type + type: + enum_type: + - EMAIL + - SOCIAL SECURITY NUMBER + - NONE +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [project_id](variables.tf#L62) | Id of the project where Tag Templates will be created. | string | ✓ | | +| [factories_config](variables.tf#L17) | Paths to data files and folders that enable factory functionality. | object({…}) | | {} | +| [iam](variables.tf#L26) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_bindings](variables.tf#L32) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L47) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [tag_templates](variables.tf#L67) | Tag templates definitions in the form {TAG_TEMPLATE_ID => TEMPLATE_DEFINITION}. | map(object({…})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [data_catalog_tag_template_ids](outputs.tf#L17) | Data catalog tag template ids. | | +| [data_catalog_tag_templates](outputs.tf#L22) | Data catalog tag templates. | | + diff --git a/modules/data-catalog-tag-template/iam.tf b/modules/data-catalog-tag-template/iam.tf new file mode 100644 index 000000000..67e99a8eb --- /dev/null +++ b/modules/data-catalog-tag-template/iam.tf @@ -0,0 +1,96 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description IAM bindings + +locals { + iam_template_map = { + for binding in flatten([ + for role, members in var.iam : [ + for template_k, template_v in google_data_catalog_tag_template.tag_template : { + template = template_v, + role = role, + members = members + } + ] + ]) : "${binding.template.tag_template_id}-${binding.role}" => binding + } + + iam_bindings_template_map = { + for binding in flatten([ + for iam_bindings_k, iam_bindings_v in var.iam_bindings : [ + for template_k, template_v in google_data_catalog_tag_template.tag_template : { + template = template_v, + iam_bindings_key = iam_bindings_k, + role = iam_bindings_v.role, + member = iam_bindings_v.members, + condition = iam_bindings_v.condition + } + ] + ]) : "${binding.template.tag_template_id}-${binding.iam_bindings_key}" => binding + } + + iam_bindings_additive_template_map = { + for binding in flatten([ + for iam_bindings_k, iam_bindings_v in var.iam_bindings_additive : [ + for template_k, template_v in google_data_catalog_tag_template.tag_template : { + template = template_v, + iam_bindings_k = iam_bindings_k, + role = iam_bindings_v.role, + member = iam_bindings_v.member, + condition = iam_bindings_v.condition + } + ] + ]) : "${binding.template.tag_template_id}-${binding.iam_bindings_k}" => binding + } +} + +resource "google_data_catalog_tag_template_iam_binding" "authoritative" { + for_each = local.iam_template_map + tag_template = each.value.template.id + role = each.value.role + members = each.value.members +} + +resource "google_data_catalog_tag_template_iam_binding" "bindings" { + for_each = local.iam_bindings_template_map + tag_template = each.value.template.id + role = each.value.role + members = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_data_catalog_tag_template_iam_member" "bindings" { + for_each = local.iam_bindings_additive_template_map + tag_template = each.value.template.id + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} diff --git a/modules/data-catalog-tag-template/main.tf b/modules/data-catalog-tag-template/main.tf new file mode 100644 index 000000000..94efc66f6 --- /dev/null +++ b/modules/data-catalog-tag-template/main.tf @@ -0,0 +1,57 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _factory_tag_template = { + for f in try(fileset(var.factories_config.tag_templates, "*.yaml"), []) : + trimsuffix(f, ".yaml") => yamldecode(file("${var.factories_config.tag_templates}/${f}")) + } + + factory_tag_template = merge(local._factory_tag_template, var.tag_templates) +} + +resource "google_data_catalog_tag_template" "tag_template" { + for_each = local.factory_tag_template + project = var.project_id + tag_template_id = each.key + region = each.value.region + display_name = try(each.value.display_name, null) + + dynamic "fields" { + for_each = each.value.fields + content { + field_id = fields.key + display_name = try(fields.value["display_name"], null) + is_required = try(fields.value["is_required"], false) + type { + primitive_type = try(fields.value["type"].primitive_type, null) + dynamic "enum_type" { + for_each = try(fields.value["type"].enum_type != null, false) ? ["1"] : [] + content { + dynamic "allowed_values" { + for_each = fields.value["type"].enum_type + content { + display_name = allowed_values.value + } + } + } + } + } + } + } + + force_delete = try(each.value.force_delete, false) +} diff --git a/modules/data-catalog-tag-template/outputs.tf b/modules/data-catalog-tag-template/outputs.tf new file mode 100644 index 000000000..f319a35ef --- /dev/null +++ b/modules/data-catalog-tag-template/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "data_catalog_tag_template_ids" { + description = "Data catalog tag template ids." + value = { for k, v in google_data_catalog_tag_template.tag_template : v.tag_template_id => v.id } +} + +output "data_catalog_tag_templates" { + description = "Data catalog tag templates." + value = { for k, v in google_data_catalog_tag_template.tag_template : v.tag_template_id => v } +} diff --git a/modules/data-catalog-tag-template/variables.tf b/modules/data-catalog-tag-template/variables.tf new file mode 100644 index 000000000..0d141b41e --- /dev/null +++ b/modules/data-catalog-tag-template/variables.tf @@ -0,0 +1,89 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "factories_config" { + description = "Paths to data files and folders that enable factory functionality." + type = object({ + tag_templates = optional(string) + }) + nullable = false + default = {} +} + +variable "iam" { + description = "IAM bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} +} + +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "project_id" { + description = "Id of the project where Tag Templates will be created." + type = string +} + +variable "tag_templates" { + description = "Tag templates definitions in the form {TAG_TEMPLATE_ID => TEMPLATE_DEFINITION}." + type = map(object({ + display_name = optional(string) + force_delete = optional(bool, false) + region = string + fields = map(object({ + display_name = optional(string) + description = optional(string) + type = object({ + primitive_type = optional(string) + enum_type = optional(list(object({ + allowed_values = object({ + display_name = string + }) + })), null) + }) + is_required = optional(bool, false) + order = optional(number) + })) + })) + default = {} +} diff --git a/modules/data-catalog-tag-template/versions.tf b/modules/data-catalog-tag-template/versions.tf new file mode 100644 index 000000000..84b66699c --- /dev/null +++ b/modules/data-catalog-tag-template/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.5.1" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 5.11.0, < 6.0.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 5.11.0, < 6.0.0" # tftest + } + } +}