From 5adba9a018d947b531588cc1ec4d5d0f26dc29c4 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 20 Apr 2025 11:25:13 +0200 Subject: [PATCH] New Dataplex Aspect Types module (#3050) * README and tests missing * default location value * tested * READMEs --- README.md | 2 +- fast/stages/3-secops-dev/README.md | 2 +- .../{versions.tf => providers-override.tf} | 19 +- modules/README.md | 1 + modules/dataplex-aspects/README.md | 167 ++++++++++++++++++ modules/dataplex-aspects/iam.tf | 89 ++++++++++ modules/dataplex-aspects/main.tf | 51 ++++++ modules/dataplex-aspects/outputs.tf | 66 +++++++ .../schemas/aspect-type.schema.json | 125 +++++++++++++ modules/dataplex-aspects/variables.tf | 67 +++++++ modules/dataplex/README.md | 10 +- 11 files changed, 580 insertions(+), 19 deletions(-) rename fast/stages/3-secops-dev/{versions.tf => providers-override.tf} (54%) create mode 100644 modules/dataplex-aspects/README.md create mode 100644 modules/dataplex-aspects/iam.tf create mode 100644 modules/dataplex-aspects/main.tf create mode 100644 modules/dataplex-aspects/outputs.tf create mode 100644 modules/dataplex-aspects/schemas/aspect-type.schema.json create mode 100644 modules/dataplex-aspects/variables.tf diff --git a/README.md b/README.md index 58d4907fd..26efe4d40 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Currently available modules: - **process factories** - [project factory](./modules/project-factory/README.md) - **networking** - [DNS](./modules/dns), [DNS Response Policy](./modules/dns-response-policy/), [Cloud Endpoints](./modules/endpoints), [address reservation](./modules/net-address), [NAT](./modules/net-cloudnat), [VLAN Attachment](./modules/net-vlan-attachment/), [External Application LB](./modules/net-lb-app-ext/), [External Passthrough Network LB](./modules/net-lb-ext), [External Regional Application Load Balancer](./modules/net-lb-app-ext-regional/), [Firewall policy](./modules/net-firewall-policy), [Internal Application LB](./modules/net-lb-app-int), [Cross-region Internal Application LB](./modules/net-lb-app-int-cross-region), [Internal Passthrough Network LB](./modules/net-lb-int), [Internal Proxy Network LB](./modules/net-lb-proxy-int), [IPSec over Interconnect](./modules/net-ipsec-over-interconnect), [VPC](./modules/net-vpc), [VPC factory](./modules/net-vpc-factory/README.md), [VPC firewall](./modules/net-vpc-firewall), [VPC peering](./modules/net-vpc-peering), [VPN dynamic](./modules/net-vpn-dynamic), [HA VPN](./modules/net-vpn-ha), [VPN static](./modules/net-vpn-static), [Service Directory](./modules/service-directory), [Secure Web Proxy](./modules/net-swp) - **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [COS container](./modules/cloud-config-container/cos-generic-metadata/) (coredns, mysql, onprem, squid), [GKE cluster](./modules/gke-cluster-standard), [GKE hub](./modules/gke-hub), [GKE nodepool](./modules/gke-nodepool), [GCVE private cloud](./modules/gcve-private-cloud) -- **data** - [AlloyDB instance](./modules/alloydb), [Analytics Hub](./modules/analytics-hub), [BigQuery dataset](./modules/bigquery-dataset), [Biglake Catalog](./modules/biglake-catalog), [Bigtable instance](./modules/bigtable-instance), [Dataplex](./modules/dataplex), [Dataplex DataScan](./modules/dataplex-datascan), [Cloud SQL instance](./modules/cloudsql-instance), [Spanner instance](./modules/spanner-instance), [Firestore](./modules/firestore), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag), [Data Catalog Tag](./modules/data-catalog-tag), [Data Catalog Tag Template](./modules/data-catalog-tag-template), [Datafusion](./modules/datafusion), [Dataproc](./modules/dataproc), [GCS](./modules/gcs), [Pub/Sub](./modules/pubsub), [Dataform Repository](./modules/dataform-repository/), [Looker Core](./modules/looker-core) +- **data** - [AlloyDB instance](./modules/alloydb), [Analytics Hub](./modules/analytics-hub), [BigQuery dataset](./modules/bigquery-dataset), [Biglake Catalog](./modules/biglake-catalog), [Bigtable instance](./modules/bigtable-instance), [Dataplex](./modules/dataplex), [Dataplex Aspect Types](./modules/dataplex-aspects/), [Dataplex DataScan](./modules/dataplex-datascan), [Cloud SQL instance](./modules/cloudsql-instance), [Spanner instance](./modules/spanner-instance), [Firestore](./modules/firestore), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag), [Data Catalog Tag](./modules/data-catalog-tag), [Data Catalog Tag Template](./modules/data-catalog-tag-template), [Datafusion](./modules/datafusion), [Dataproc](./modules/dataproc), [GCS](./modules/gcs), [Pub/Sub](./modules/pubsub), [Dataform Repository](./modules/dataform-repository/), [Looker Core](./modules/looker-core) - **development** - [API Gateway](./modules/api-gateway), [Apigee](./modules/apigee), [Artifact Registry](./modules/artifact-registry), [Container Registry](./modules/container-registry), [Cloud Source Repository](./modules/source-repository), [Secure Source Manager instance](./modules/secure-source-manager-instance), [Workstation cluster](./modules/workstation-cluster) - **security** - [Binauthz](./modules/binauthz/), [Certificate Authority Service (CAS)](./modules/certificate-authority-service), [KMS](./modules/kms), [SecretManager](./modules/secret-manager), [VPC Service Control](./modules/vpc-sc), [Certificate Manager](./modules/certificate-manager/) - **serverless** - [Cloud Function v1](./modules/cloud-function-v1), [Cloud Function v2](./modules/cloud-function-v2), [Cloud Run](./modules/cloud-run), [Cloud Run v2](./modules/cloud-run-v2) diff --git a/fast/stages/3-secops-dev/README.md b/fast/stages/3-secops-dev/README.md index 3b8c2c0c9..884665249 100644 --- a/fast/stages/3-secops-dev/README.md +++ b/fast/stages/3-secops-dev/README.md @@ -204,12 +204,12 @@ Please be aware the Service Account Client ID needed during domain wide delegati |---|---|---|---| | [main.tf](./main.tf) | Module-level locals and resources. | project · secops-rules | google_apikeys_key | | [outputs.tf](./outputs.tf) | Module outputs. | | | +| [providers-override.tf](./providers-override.tf) | None | | | | [secops-providers.tf](./secops-providers.tf) | None | | | | [secops.tf](./secops.tf) | None | | google_chronicle_data_access_label · google_chronicle_data_access_scope | | [secrets.tf](./secrets.tf) | None | secret-manager | | | [variables-fast.tf](./variables-fast.tf) | None | | | | [variables.tf](./variables.tf) | Module variables. | | | -| [versions.tf](./versions.tf) | Version pins. | | | | [workspace.tf](./workspace.tf) | None | iam-service-account | google_service_account_key · restful_resource | ## Variables diff --git a/fast/stages/3-secops-dev/versions.tf b/fast/stages/3-secops-dev/providers-override.tf similarity index 54% rename from fast/stages/3-secops-dev/versions.tf rename to fast/stages/3-secops-dev/providers-override.tf index 5a9b71dff..bb80a1a35 100644 --- a/fast/stages/3-secops-dev/versions.tf +++ b/fast/stages/3-secops-dev/providers-override.tf @@ -12,24 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Fabric release: v39.0.0 - terraform { - required_version = ">= 1.10.2" required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.28.0, < 7.0.0" # tftest + restful = { + source = "magodo/restful" + version = "= 0.21.0" } - google-beta = { - source = "hashicorp/google-beta" - version = ">= 6.28.0, < 7.0.0" # tftest - } - } - provider_meta "google" { - module_name = "google-pso-tool/cloud-foundation-fabric/fast/stages/3-secops-dev:v39.0.0-tf" - } - provider_meta "google-beta" { - module_name = "google-pso-tool/cloud-foundation-fabric/fast/stages/3-secops-dev:v39.0.0-tf" } } diff --git a/modules/README.md b/modules/README.md index 05f9d636d..dfc2ecb3c 100644 --- a/modules/README.md +++ b/modules/README.md @@ -92,6 +92,7 @@ These modules are used in the examples included in this repository. If you are u - [Dataform Repository](./dataform-repository/) - [Datafusion](./datafusion) - [Dataplex](./dataplex) +- [Dataplex Aspect Types](./dataplex-aspects/) - [Dataplex DataScan](./dataplex-datascan/) - [Dataproc](./dataproc) - [Firestore](./firestore) diff --git a/modules/dataplex-aspects/README.md b/modules/dataplex-aspects/README.md new file mode 100644 index 000000000..7991da33d --- /dev/null +++ b/modules/dataplex-aspects/README.md @@ -0,0 +1,167 @@ +# Dataplex Aspect Types Module + +This module allows managing [Dataplex Aspect Types](https://cloud.google.com/dataplex/docs/enrich-entries-metadata) and their associated IAM bindings via variables and YAML files defined via a resource factory. + +The module manages Aspect Types for a single location in a single project. To manage them in different locations invoke the module multiple times, or use it with a `for_each` on locations/projects. + + +- [Simple example](#simple-example) +- [Factory example](#factory-example) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Simple example + +This example mirrors the one in the [`google_dataplex_aspect_type`](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/dataplex_aspect_type) resource documentation, but also shows how to manage IAM on the single aspect type. More types can of course be defined by just adding them to the `aspect_types` map. + +```hcl +module "aspect-types" { + source = "./fabric/modules/dataplex-aspects" + project_id = "test-project" + # var.location defaults to "global" + # location = "global" + aspect_types = { + tf-test-template = { + display_name = "Test template." + iam = { + "roles/dataplex.aspectTypeOwner" = ["group:data-owners@example.com"] + } + iam_bindings_additive = { + user = { + role = "roles/dataplex.aspectTypeUser" + member = "serviceAccount:sa-0@test-project.iam.gserviceaccount.com" + } + } + metadata_template = < +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [project_id](variables.tf#L64) | Project id where resources will be created. | string | ✓ | | +| [aspect_types](variables.tf#L17) | Aspect templates. Merged with those defined via the factory. | map(object({…})) | | {} | +| [factories_config](variables.tf#L48) | Paths to folders for the optional factories. | object({…}) | | {} | +| [location](variables.tf#L57) | Location for aspect types. | string | | "global" | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [ids](outputs.tf#L17) | Aspect type IDs. | | +| [names](outputs.tf#L29) | Aspect type names. | | +| [timestamps](outputs.tf#L41) | Aspect type create and update timestamps. | | +| [uids](outputs.tf#L56) | Aspect type gobally unique IDs. | | + diff --git a/modules/dataplex-aspects/iam.tf b/modules/dataplex-aspects/iam.tf new file mode 100644 index 000000000..4d1f59c61 --- /dev/null +++ b/modules/dataplex-aspects/iam.tf @@ -0,0 +1,89 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + iam = flatten([ + for k, v in local.aspect_types : [ + for role, members in v.iam : { + aspect_type_id = k + role = role + members = members + } + ] + ]) + iam_bindings = merge([ + for k, v in local.aspect_types : { + for binding_key, data in v.iam_bindings : + binding_key => { + aspect_type_id = k + role = data.role + members = data.members + condition = data.condition + } + } + ]...) + iam_bindings_additive = merge([ + for k, v in local.aspect_types : { + for binding_key, data in v.iam_bindings_additive : + binding_key => { + aspect_type_id = k + role = data.role + member = data.member + condition = data.condition + } + } + ]...) +} + +resource "google_dataplex_aspect_type_iam_binding" "authoritative" { + for_each = { + for binding in local.iam : + "${binding.aspect_type_id}.${binding.role}" => binding + } + role = each.value.role + aspect_type_id = google_dataplex_aspect_type.default[each.value.aspect_type_id].id + members = each.value.members +} + +resource "google_dataplex_aspect_type_iam_binding" "bindings" { + for_each = local.iam_bindings + role = each.value.role + aspect_type_id = google_dataplex_aspect_type.default[each.value.aspect_type_id].id + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_dataplex_aspect_type_iam_member" "members" { + for_each = local.iam_bindings_additive + aspect_type_id = google_dataplex_aspect_type.default[each.value.aspect_type_id].id + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} diff --git a/modules/dataplex-aspects/main.tf b/modules/dataplex-aspects/main.tf new file mode 100644 index 000000000..1b6df7404 --- /dev/null +++ b/modules/dataplex-aspects/main.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _factory_path = try(pathexpand(var.factories_config.aspect_types), null) + _factory_data_raw = { + for f in try(fileset(local._factory_path, "**/*.yaml"), []) : + trimsuffix(basename(f), ".yaml") => yamldecode(file("${local._factory_path}/${f}")) + } + aspect_types = merge(var.aspect_types, { + for k, v in local._factory_data_raw : k => { + description = lookup(v, "description", null) + display_name = lookup(v, "display_name", null) + iam = lookup(v, "iam", {}) + iam_bindings = { + for ik, iv in lookup(v, "iam_bindings", {}) : + ik => merge({ condition = null }, iv) + } + iam_bindings_additive = { + for ik, iv in lookup(v, "iam_bindings_additive", {}) : + ik => merge({ condition = null }, iv) + } + labels = lookup(v, "labels", {}) + metadata_template = lookup(v, "metadata_template", null) + } + }) +} + +resource "google_dataplex_aspect_type" "default" { + for_each = local.aspect_types + project = var.project_id + location = var.location + aspect_type_id = each.key + description = each.value.description + display_name = each.value.display_name + labels = each.value.labels + metadata_template = each.value.metadata_template +} diff --git a/modules/dataplex-aspects/outputs.tf b/modules/dataplex-aspects/outputs.tf new file mode 100644 index 000000000..28ece3720 --- /dev/null +++ b/modules/dataplex-aspects/outputs.tf @@ -0,0 +1,66 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "ids" { + description = "Aspect type IDs." + value = { + for k, v in google_dataplex_aspect_type.default : k => v.id + } + depends_on = [ + google_dataplex_aspect_type_iam_binding.authoritative, + google_dataplex_aspect_type_iam_binding.bindings, + google_dataplex_aspect_type_iam_member.members + ] +} + +output "names" { + description = "Aspect type names." + value = { + for k, v in google_dataplex_aspect_type.default : k => v.name + } + depends_on = [ + google_dataplex_aspect_type_iam_binding.authoritative, + google_dataplex_aspect_type_iam_binding.bindings, + google_dataplex_aspect_type_iam_member.members + ] +} + +output "timestamps" { + description = "Aspect type create and update timestamps." + value = { + for k, v in google_dataplex_aspect_type.default : k => { + create = v.create_time + update = v.update_time + } + } + depends_on = [ + google_dataplex_aspect_type_iam_binding.authoritative, + google_dataplex_aspect_type_iam_binding.bindings, + google_dataplex_aspect_type_iam_member.members + ] +} + +output "uids" { + description = "Aspect type gobally unique IDs." + value = { + for k, v in google_dataplex_aspect_type.default : k => v.uid + } + depends_on = [ + google_dataplex_aspect_type_iam_binding.authoritative, + google_dataplex_aspect_type_iam_binding.bindings, + google_dataplex_aspect_type_iam_member.members + ] +} diff --git a/modules/dataplex-aspects/schemas/aspect-type.schema.json b/modules/dataplex-aspects/schemas/aspect-type.schema.json new file mode 100644 index 000000000..5148fdd9e --- /dev/null +++ b/modules/dataplex-aspects/schemas/aspect-type.schema.json @@ -0,0 +1,125 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Dataplex Aspect Type", + "type": "object", + "additionalProperties": false, + "properties": { + "description": { + "type": "string" + }, + "display_name": { + "type": "string" + }, + "labels": { + "type": "object" + }, + "metadata_template": { + "type": "string" + }, + "iam": { + "$ref": "#/$defs/iam" + }, + "iam_bindings": { + "$ref": "#/$defs/iam_bindings" + }, + "iam_bindings_additive": { + "$ref": "#/$defs/iam_bindings_additive" + } + }, + "$defs": { + "iam": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^roles/": { + "type": "array", + "items": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|[a-z])" + } + } + } + }, + "iam_bindings": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "members": { + "type": "array", + "items": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|[a-z])" + } + }, + "role": { + "type": "string", + "pattern": "^roles/" + }, + "condition": { + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "title" + ], + "properties": { + "expression": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + } + } + } + } + } + } + }, + "iam_bindings_additive": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "member": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|[a-z])" + }, + "role": { + "type": "string", + "pattern": "^roles/" + }, + "condition": { + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "title" + ], + "properties": { + "expression": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/modules/dataplex-aspects/variables.tf b/modules/dataplex-aspects/variables.tf new file mode 100644 index 000000000..9c43b37be --- /dev/null +++ b/modules/dataplex-aspects/variables.tf @@ -0,0 +1,67 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "aspect_types" { + description = "Aspect templates. Merged with those defined via the factory." + type = map(object({ + description = optional(string) + display_name = optional(string) + labels = optional(map(string), {}) + metadata_template = optional(string) + iam = optional(map(list(string)), {}) + iam_bindings = optional(map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + iam_bindings_additive = optional(map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + })) + nullable = false + default = {} +} + +variable "factories_config" { + description = "Paths to folders for the optional factories." + type = object({ + aspect_types = optional(string) + }) + nullable = false + default = {} +} + +variable "location" { + description = "Location for aspect types." + type = string + nullable = false + default = "global" +} + +variable "project_id" { + description = "Project id where resources will be created." + type = string +} diff --git a/modules/dataplex/README.md b/modules/dataplex/README.md index bc16be747..d67bda400 100644 --- a/modules/dataplex/README.md +++ b/modules/dataplex/README.md @@ -1,6 +1,14 @@ # Dataplex instance with lake, zone & assets -This module manages the creation of Dataplex instance along with lake, zone & assets in single regions. +This module manages the creation of a Dataplex instance along with lake, zone & assets in single regions. + + +- [Simple example](#simple-example) +- [IAM](#iam) +- [TODO](#todo) +- [Variables](#variables) +- [Outputs](#outputs) + ## Simple example