diff --git a/.gitignore b/.gitignore index 21f3a2f50..8f632a30e 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,5 @@ node_modules fast/**/globals.auto.tfvars.json cloud_sql_proxy env/ +.jetskicli +.agents diff --git a/fast/addons/2-networking-test/README.md b/fast/addons/2-networking-test/README.md index f0313e5da..5edf969d5 100644 --- a/fast/addons/2-networking-test/README.md +++ b/fast/addons/2-networking-test/README.md @@ -111,6 +111,6 @@ terraform apply | name | description | sensitive | consumers | |---|---|:---:|---| | [instance_addresses](outputs.tf#L17) | Instance names and addresses. | | | -| [instance_ssh](outputs.tf#L24) | Instance SSH commands. | | | -| [service_account_emails](outputs.tf#L33) | Service account emails. | | | +| [instance_ssh](outputs.tf#L24) | Instance SSH commands. | ✓ | | +| [service_account_emails](outputs.tf#L34) | Service account emails. | | | diff --git a/fast/addons/2-networking-test/outputs.tf b/fast/addons/2-networking-test/outputs.tf index 69b880c93..a778f6331 100644 --- a/fast/addons/2-networking-test/outputs.tf +++ b/fast/addons/2-networking-test/outputs.tf @@ -23,6 +23,7 @@ output "instance_addresses" { output "instance_ssh" { description = "Instance SSH commands." + sensitive = true value = { for k, v in module.instances : k => ( "gcloud compute ssh ${k} --project ${nonsensitive(v.instance.project)} --zone ${nonsensitive(v.instance.zone)}" diff --git a/fast/stages/0-org-setup/schemas/project.schema.json b/fast/stages/0-org-setup/schemas/project.schema.json index d65fa4a58..94c224062 100644 --- a/fast/stages/0-org-setup/schemas/project.schema.json +++ b/fast/stages/0-org-setup/schemas/project.schema.json @@ -111,6 +111,9 @@ "type": "object", "additionalProperties": false, "properties": { + "display_name": { + "type": "string" + }, "description": { "type": "string" }, @@ -297,9 +300,15 @@ "type": "object", "additionalProperties": false, "properties": { + "aspect_types": { + "type": "string" + }, "custom_roles": { "type": "string" }, + "data_catalog_taxonomy": { + "type": "string" + }, "observability": { "type": "string" }, diff --git a/fast/stages/0-org-setup/schemas/project.schema.md b/fast/stages/0-org-setup/schemas/project.schema.md index a14fb866d..31998ca8a 100644 --- a/fast/stages/0-org-setup/schemas/project.schema.md +++ b/fast/stages/0-org-setup/schemas/project.schema.md @@ -37,6 +37,7 @@
*additional properties: false* - **`^[a-z0-9-]+$`**: *object*
*additional properties: false* + - **display_name**: *string* - **description**: *string* - **prefix**: *string* - **iam**: *reference([iam](#refs-iam))* @@ -95,7 +96,9 @@
*enum: ['PREVENT', 'DELETE', 'ABANDON']* - **factories_config**: *object*
*additional properties: false* + - **aspect_types**: *string* - **custom_roles**: *string* + - **data_catalog_taxonomy**: *string* - **observability**: *string* - **org_policies**: *string* - **quotas**: *string* diff --git a/fast/stages/2-networking/schemas/project.schema.json b/fast/stages/2-networking/schemas/project.schema.json index d65fa4a58..94c224062 100644 --- a/fast/stages/2-networking/schemas/project.schema.json +++ b/fast/stages/2-networking/schemas/project.schema.json @@ -111,6 +111,9 @@ "type": "object", "additionalProperties": false, "properties": { + "display_name": { + "type": "string" + }, "description": { "type": "string" }, @@ -297,9 +300,15 @@ "type": "object", "additionalProperties": false, "properties": { + "aspect_types": { + "type": "string" + }, "custom_roles": { "type": "string" }, + "data_catalog_taxonomy": { + "type": "string" + }, "observability": { "type": "string" }, diff --git a/fast/stages/2-networking/schemas/project.schema.md b/fast/stages/2-networking/schemas/project.schema.md index a14fb866d..31998ca8a 100644 --- a/fast/stages/2-networking/schemas/project.schema.md +++ b/fast/stages/2-networking/schemas/project.schema.md @@ -37,6 +37,7 @@
*additional properties: false* - **`^[a-z0-9-]+$`**: *object*
*additional properties: false* + - **display_name**: *string* - **description**: *string* - **prefix**: *string* - **iam**: *reference([iam](#refs-iam))* @@ -95,7 +96,9 @@
*enum: ['PREVENT', 'DELETE', 'ABANDON']* - **factories_config**: *object*
*additional properties: false* + - **aspect_types**: *string* - **custom_roles**: *string* + - **data_catalog_taxonomy**: *string* - **observability**: *string* - **org_policies**: *string* - **quotas**: *string* diff --git a/fast/stages/2-project-factory/README.md b/fast/stages/2-project-factory/README.md index a4ce8909a..47f5a724d 100644 --- a/fast/stages/2-project-factory/README.md +++ b/fast/stages/2-project-factory/README.md @@ -9,6 +9,7 @@ - [Organization IAM](#organization-iam) - [Parent folder](#parent-folder) - [Factory configuration](#factory-configuration) + - [Data Platform Dataset](#data-platform-dataset) - [Stage provider and Terraform variables](#stage-provider-and-terraform-variables) - [Managing folders and projects](#managing-folders-and-projects) - [Project defaults and overrides](#project-defaults-and-overrides) @@ -149,6 +150,10 @@ tag_bindings: The `data` folder in this stage contains factory files that can be used as examples to implement the team-based design shown above. Before running `terraform apply` check the YAML files, as project names and other attributes will need basic editing to match your desired setup. +### Data Platform Dataset + +A specialized dataset for Data Platform workloads is available in [`datasets/data-platform`](./datasets/data-platform/). It includes configuration for projects, VPCs, tags, and taxonomies tailored for data analytics. To use it, set the `factories_config.dataset` variable to `datasets/data-platform`. + ### Stage provider and Terraform variables As all other FAST stages, the [mechanism](../0-org-setup/README.md#output-files-and-cross-stage-variables) used to pass variable values and pre-built provider files from one stage to the next is also leveraged here. @@ -473,6 +478,7 @@ Pattern-based files make specific assumptions: | [variables-fast.tf](./variables-fast.tf) | None | | | | [variables-projects.tf](./variables-projects.tf) | None | | | | [variables.tf](./variables.tf) | Module variables. | | | +| [vpcs.tf](./vpcs.tf) | VPC factory. | net-vpc-factory | | ## Variables @@ -487,7 +493,7 @@ Pattern-based files make specific assumptions: | [data_defaults](variables-projects.tf#L17) | Optional default values used when corresponding project or folder data from files are missing. | object({…}) | | {} | | | [data_merges](variables-projects.tf#L93) | Optional values that will be merged with corresponding data from files. Combines with `data_defaults`, file data, and `data_overrides`. | object({…}) | | {} | | | [data_overrides](variables-projects.tf#L112) | Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`. | object({…}) | | {} | | -| [factories_config](variables.tf#L41) | Path to folder with YAML resource description data files. | object({…}) | | {} | | +| [factories_config](variables.tf#L42) | Path to folder with YAML resource description data files. | object({…}) | | {} | | | [folder_ids](variables-fast.tf#L42) | Folders created in the bootstrap stage. | map(string) | | {} | 0-org-setup | | [host_project_ids](variables-fast.tf#L58) | Host project for the shared VPC. | map(string) | | {} | 2-networking | | [iam_principals](variables-fast.tf#L50) | IAM-format principals. | map(string) | | {} | 0-org-setup | @@ -495,7 +501,7 @@ Pattern-based files make specific assumptions: | [perimeters](variables-fast.tf#L84) | Optional VPC-SC perimeter ids. | map(string) | | {} | 1-vpcsc | | [project_ids](variables-fast.tf#L102) | Projects created in the bootstrap stage. | map(string) | | {} | 0-org-setup | | [service_accounts](variables-fast.tf#L110) | Service accounts created in the bootstrap stage. | map(string) | | {} | 0-org-setup | -| [stage_name](variables.tf#L62) | FAST stage name. Used to separate output files across different factories. | string | | "2-project-factory" | | +| [stage_name](variables.tf#L65) | FAST stage name. Used to separate output files across different factories. | string | | "2-project-factory" | | | [subnet_self_links](variables-fast.tf#L118) | Shared VPC subnet IDs. | map(map(string)) | | {} | 2-networking | | [tag_values](variables-fast.tf#L126) | FAST-managed resource manager tag values. | map(string) | | {} | 0-org-setup | | [tag_vars](variables-fast.tf#L134) | FAST-managed resource manager tag key namespaced names. | object({…}) | | {} | 0-org-setup | @@ -506,4 +512,5 @@ Pattern-based files make specific assumptions: | name | description | sensitive | consumers | |---|---|:---:|---| | [projects](outputs.tf#L17) | Attributes for managed projects. | | | +| [vpcs](outputs.tf#L22) | VPCs. | | | diff --git a/fast/stages/2-project-factory/datasets/data-platform/README.md b/fast/stages/2-project-factory/datasets/data-platform/README.md new file mode 100644 index 000000000..80d8898e2 --- /dev/null +++ b/fast/stages/2-project-factory/datasets/data-platform/README.md @@ -0,0 +1,403 @@ +# Data Platform Dataset + +This dataset configures an opinionated Data Platform architecture based on Google Cloud best practices, managed via the Project Factory. + +Its architecture is designed to be reliable, robust, and scalable, facilitating the continuous onboarding of new Data Products (or data workloads). + + +- [Design Overview and Choices](#design-overview-and-choices) + - [Data Platform Architecture](#data-platform-architecture) + - [Folder and Project Structure](#folder-and-project-structure) + - [Central Shared Services (Federated Governance)](#central-shared-services-federated-governance) + - [Data Domains (Domain-Driven Ownership)](#data-domains-domain-driven-ownership) + - [Data Products (DaaP)](#data-products-daap) + - [Delegated Automation Model](#delegated-automation-model) + - [Teams and Personas](#teams-and-personas) + - [Central Data Platform Team](#central-data-platform-team) + - [Data Domain Team](#data-domain-team) + - [Data Product Team](#data-product-team) +- [Stage Prerequisites](#stage-prerequisites) + - [Service Accounts](#service-accounts) + - [Storage](#storage) + - [IAM Bindings](#iam-bindings) + - [Data Platform Folder](#data-platform-folder) + - [Networking](#networking) + - [Security](#security) + - [Organization Level (VPC-SC)](#organization-level-vpc-sc) + - [Output Files and Provider Generation](#output-files-and-provider-generation) + - [Environment File and Linking](#environment-file-and-linking) +- [Customization Guide](#customization-guide) + - [1. IAM Principals and Context](#1-iam-principals-and-context) + - [2. Data Governance Assets](#2-data-governance-assets) + - [3. Adding Domains and Products](#3-adding-domains-and-products) + - [4. VPC-SC Perimeter](#4-vpc-sc-perimeter) +- [Deployment Choices](#deployment-choices) + - [Networking Models](#networking-models) + - [Project-Local VPCs](#project-local-vpcs) +- [Usage](#usage) + + +## Design Overview and Choices + +### Data Platform Architecture + +The following diagram represents the high-level architecture of the Data Platform related projects and their associated resources managed by this dataset: + +

+ High level diagram. +

+ +### Folder and Project Structure + +The dataset manages the following three high-level logical components implemented via GCP folders and projects: + +- "Central Shared Services", a single central project, in which Dataplex Catalog Aspect Types, Policy Tags, and Resource Manager tags a.k.a. "Secure Tags" are defined +- one or more "Data Domains", each composed of a folder with a top-level shared project hosting shared resources such as Composer at the domain level, and an additional sub-folder for hosting data products e.g. "Data Products" +- one or more "Data Products" per domain, each composed of a project, and related resources that are optional + +

+Folder structure. +

+ +#### Central Shared Services (Federated Governance) + +Central Shared Services Project provides the standardized central capabilities to foster federated governance processes. These are implemented via established foundations that enable cross-domain data discovery, data sharing, self-service functionalities, and consistent governance. + +Core, platform-wide capabilities are delivered as shared services managed within a dedicated "Central Shared Services" project. These capabilities include: + +- [Dataplex Catalog Aspect Types](https://cloud.google.com/dataplex/docs/enrich-entries-metadata): Defined in `aspect-types/`. +- [Policy Tags](https://cloud.google.com/bigquery/docs/best-practices-policy-tags): Configured via the `central_project_config.policy_tags` variable (if applicable) or YAML. + +#### Data Domains (Domain-Driven Ownership) + +A Data Domain typically aligns with a business unit (BU) or a distinct function within an enterprise. To support this ownership model, each logical Data Domain is provisioned with its own isolated GCP folder. + +Within each Data Domain, a corresponding Google Cloud "Data Domain" project serves as the primary container for all its specific services and resources. A dedicated Cloud Composer environment is provisioned within this project for orchestrating the domain's data workflows. + +#### Data Products (DaaP) + +Each Data Product within a Data Domain is encapsulated in its own dedicated Google Cloud Project. This separation is key to achieving modularity, scalability, flexibility, and distinct ownership for each product. + +### Delegated Automation Model + +To enable domain-driven ownership and self-service, the dataset implements a multi-tiered automation model using the Project Factory's `automation` feature: + +1. **Central Platform**: Managed by the main Stage 0 service accounts (or the principal running the stage). +2. **Data Domains**: The Domain project (`projects/domain-0/shared-0.yaml`) creates its own automation service accounts (read-only and read-write) in the Central project (`prod-dp-core-0`). +3. **Data Products**: The Product project (`projects/domain-0/product-0.yaml`) uses the Domain's read-write service account for its automation, and stores its state in a bucket within the Domain project. + +This structure allows central teams to bootstrap domains, and then delegate the management of products within a domain to the respective domain team, using their specific service accounts. + +### Teams and Personas + +Effective data mesh operation relies on well-defined roles and responsibilities. + +| Group | Central Shared Services Project | Data Domain Folder | Data Product Project | +| - | :-: | :-: | :-: | +| Central Data Platform Team | `ADMIN` | `Log and Metrics Viewer` | `Log and Metrics Viewer` | +| Data Domain Team | `READ/USAGE` | `ADMIN` | `Log and Metrics Viewer` | +| Data Product Team | `READ/USAGE` | `READ/USAGE` | `ADMIN` | + +#### Central Data Platform Team + +This team defines the overall data platform architecture, establishes shared infrastructure, and enforces central data governance policies and standards across the data mesh. + +#### Data Domain Team + +Aligned with specific business areas (e.g., customer, finance, distribution), this team holds clearly defined ownership of data within that domain. They are responsible for the domain-wide data product roadmap and security. + +#### Data Product Team + +This team is responsible for the end-to-end lifecycle of a specific Data Product. They develop, operate, and maintain their assigned Data Product, including ingestion, transformation, and exposure. + +## Stage Prerequisites + +When using this dataset as an additional project factory (separate from the main `2-project-factory` stage), you need to configure Stage 0 to provision the necessary automation resources and IAM permissions. + +This typically involves adding a new set of service accounts and a dedicated state bucket folder for the data platform automation. + +### Service Accounts + +Add the following service accounts to your Stage 0 configuration (e.g., in `projects/core/iac-0.yaml`): + +```yaml +service_accounts: + iac-dp-ro: + display_name: IaC service account for data platform (read-only). + iac-dp-rw: + display_name: IaC service account for data platform (read-write). +``` + +### Storage + +Configure a dedicated managed folder for the data platform state in the Stage 0 state bucket, and grant access to the service accounts: + +```yaml +buckets: + iac-stage-state: + managed_folders: + 2-data-platform: + iam: + roles/storage.admin: + - $iam_principals:service_accounts/iac-0/iac-dp-rw + $custom_roles:storage_viewer: + - $iam_principals:service_accounts/iac-0/iac-dp-ro +``` + +Also, ensure the data platform service accounts have access to the `iac-outputs` bucket: + +```yaml + iac-outputs: + iam: + roles/storage.admin: + - $iam_principals:service_accounts/iac-0/iac-dp-rw + $custom_roles:storage_viewer: + - $iam_principals:service_accounts/iac-0/iac-dp-ro +``` + +### IAM Bindings + +Grant the necessary permissions to the data platform service accounts on the folders they will manage. + +#### Data Platform Folder + +Since the Data Platform folder is not part of the default dataset, you need to create a new folder configuration file. + +Create a file named `.config.yaml` for the folder. You can place it: +- Directly under the organization by creating `folders/data-platform/.config.yaml` in your Stage 0 dataset directory. +- Within an existing folder, for example under `teams`, by creating `folders/teams/data-platform/.config.yaml`. + +Here is the complete `.config.yaml` content: + +```yaml +name: Data Platform +iam_by_principals: + $iam_principals:service_accounts/iac-0/iac-dp-rw: + - roles/logging.admin + - roles/owner + - roles/resourcemanager.folderAdmin + - roles/resourcemanager.projectCreator + - roles/compute.xpnAdmin + $iam_principals:service_accounts/iac-0/iac-dp-ro: + - roles/viewer + - roles/resourcemanager.folderViewer +``` + +#### Networking + +This configuration is used if you rely on the centralized network stage, and access to share networking resources is required from the data platform. Grant the following roles on the `networking` folder. + +> [!NOTE] +> These prerequisites are only required if you are using the default Shared VPCs model described in the [Deployment Choices](#deployment-choices) section. + +```yaml +iam_bindings: + dp_rw: + members: + - $iam_principals:service_accounts/iac-0/iac-dp-rw + role: $custom_roles:service_project_network_admin + dp_ro: + role: roles/compute.networkViewer + members: + - $iam_principals:service_accounts/iac-0/iac-dp-ro +``` + +And to delegate IAM project administration for networking resources: + +```yaml +iam_bindings: + dp_delegated_iam: + role: roles/resourcemanager.projectIamAdmin + members: + - $iam_principals:service_accounts/iac-0/iac-dp-rw + condition: + title: Data platform delegated IAM grant. + expression: | + api.getAttribute('iam.googleapis.com/modifiedGrantsByRole', []).hasOnly([ + 'roles/compute.networkUser', 'roles/composer.sharedVpcAgent', + 'roles/container.hostServiceAgentUser', 'roles/vpcaccess.user', + '${custom_roles["dns_zone_binder"]}' + ]) +``` + +> [!TIP] +> If your Stage 0 configuration already includes a `project_factory` delegated IAM grant with these roles (e.g., for the main Project Factory stage), you can simply add the data platform service account (`iac-dp-rw`) to the members list of that existing binding instead of creating a new `dp_delegated_iam` binding. This applies to both networking and security configurations if similar delegations are needed. + +#### Security + +This configuration is used if you rely on the centralized security stage, and access to manage security resources (like KMS keys) is required from the data platform. Grant the following roles on the `security` folder (or the specific project hosting the keys): + +```yaml +iam_bindings: + dp_rw_viewer: + role: roles/cloudkms.viewer + members: + - $iam_principals:service_accounts/iac-0/iac-dp-rw + dp_ro_viewer: + role: roles/cloudkms.viewer + members: + - $iam_principals:service_accounts/iac-0/iac-dp-ro +``` + +And to delegate IAM project administration for KMS resources (allow granting encrypt/decrypt roles on keys): + +```yaml +iam_bindings: + dp_delegated_kms: + role: roles/cloudkms.admin + members: + - $iam_principals:service_accounts/iac-0/iac-dp-rw + condition: + title: Data platform delegated KMS grant. + expression: | + api.getAttribute('iam.googleapis.com/modifiedGrantsByRole', []).hasOnly( +[ + 'roles/cloudkms.cryptoKeyEncrypterDecrypter', + 'roles/cloudkms.cryptoKeyEncrypterDecrypterViaDelegation' + ]) && resource.type == 'cloudkms.googleapis.com/CryptoKey' +``` + +#### Organization Level (VPC-SC) + +If you are using VPC-SC and need this stage to manage perimeters or add projects to them, you must grant the following roles at the organization level to the data platform service accounts: + +```yaml +iam_by_principals: + $iam_principals:service_accounts/iac-0/iac-dp-rw: + - roles/accesscontextmanager.policyEditor + $iam_principals:service_accounts/iac-0/iac-dp-ro: + - roles/accesscontextmanager.policyReader +``` + +### Output Files and Provider Generation + +To automatically generate the provider files for this stage, you need to configure the `output_files` section in your Stage 0 `defaults.yaml` file (e.g., `fast/stages/0-org-setup/datasets/classic/defaults.yaml`). + +Add the following entries under `output_files.providers`: + +```yaml + 2-data-platform: + bucket: $storage_buckets:iac-0/iac-stage-state + prefix: 2-data-platform + service_account: $iam_principals:service_accounts/iac-0/iac-dp-rw + 2-data-platform-ro: + bucket: $storage_buckets:iac-0/iac-stage-state + prefix: 2-data-platform + service_account: $iam_principals:service_accounts/iac-0/iac-dp-ro +``` + +This configuration instructs Stage 0 to generate `2-data-platform-providers.tf` and `2-data-platform-ro-providers.tf` files in the outputs folder and/or bucket. + +### Environment File and Linking + +To use these generated files in your custom stage directory (e.g., `custom-stages/2-project-factory-dp`), you need to configure the `.fast-stage.env` file in that directory. + +Set `FAST_STAGE_NAME` to `data-platform`: + +```text +FAST_STAGE_DESCRIPTION="data platform" +FAST_STAGE_LEVEL=2 +FAST_STAGE_NAME=data-platform +FAST_STAGE_DEPS="0-globals 0-org-setup" +FAST_STAGE_OPTIONAL="1-vpcsc 2-networking 2-security" +``` + +This allows the `fast-links.sh` script to correctly identify and link the data platform specific provider and tfvars files. + +To create the links, run the script from inside your stage folder: + +```bash +../../fast/stages/fast-links.sh path/to/your/fast-config or gs://automation_bucket_name +``` + +## Customization Guide + +You can customize the deployment by modifying the YAML files in this directory: + +#### 1. IAM Principals and Context +In `defaults.yaml`, update the `context.iam_principals` map with the actual groups or users for your organization: +- `dp-platform`: Central data platform team. +- `dp-domain-a`: Data domain team members. +- `dp-product-a-0`: Data product team members. +- `data-consumer-bi`: Consumers of public data. + +#### 2. Data Governance Assets +The Central project (`projects/core-0.yaml`) is configured to load data governance assets from the following directories: +- `aspect-types/`: Define Dataplex Catalog Aspect Types. +- `tags/`: Define Resource Manager tags. +- `taxonomies/`: Define Data Catalog taxonomies and policy tags (e.g., `taxonomies/tags.yaml`). + +#### 3. Adding Domains and Products +The provided `domain-0` directory is a template. To add new domains or products: +1. Replicate or modify the folder structure under `projects/`. +2. Update the `parent` and `name` attributes in the new YAML files. +3. Ensure the `automation` block references the correct parent project and service accounts. + +> [!TIP] +> **Project Templates**: For data products that follow a similar configuration, you can use the project templates feature by placing template files in the `project-templates/` directory. This dataset provides a simple example in `project-templates/data-product.yaml`. +> +> Note that the project template feature implements a **shallow merge**, meaning that top-level keys defined in your specific project YAML file will completely overwrite the corresponding top-level keys from the template. + +#### 4. VPC-SC Perimeter +If you are using VPC-SC, you can add projects to a perimeter by specifying it in `defaults.yaml` under `projects.defaults`: + +```yaml +projects: + defaults: + vpc_sc: + perimeter_name: $vpc_sc_perimeters:default + is_dry_run: false +``` + +## Deployment Choices + +The Data Platform dataset allows for flexibility in networking models, supporting both Shared VPCs (standard enterprise pattern) and Project-Local VPCs (isolated workloads). + +### Networking Models + +The Data Platform dataset supports two networking models, which can also be used together: + +1. **Project-Local VPCs (Default)**: By default, this dataset creates project-local VPCs using the VPC factory (reading from the `vpcs` directory). +2. **Shared VPCs**: To attach projects to a Shared VPC managed in the `2-networking` stage, you must: + * Delete the files in the `vpcs` directory (or set `factories_config.paths.vpcs` to a non-existent path). + * Uncomment the `shared_vpc_service_config` block in the project YAML files (e.g., `projects/domain-0/shared-0.yaml`). + +### Project-Local VPCs + +This folder can contain YAML files defining VPCs that are local to specific projects. +The structure should follow the `net-vpc-factory` pattern: + +1. Create a folder for the VPC (e.g., `my-vpc-0`). +2. Inside that folder, create a `.config.yaml` file. +3. Define the VPC configuration in `.config.yaml` (see `net-vpc-factory` module documentation for schema details). + +Example `.config.yaml` (e.g. `domain-0/.config.yaml`): + +```yaml +name: domain-0 +project_id: $project_ids:shared-0 # Use context interpolation for project IDs +subnets: + - name: default + ip_cidr_range: 10.0.0.0/24 + region: $locations:primary +``` + +Note: You must also enable the `vpcs` factory path in your `terraform.tfvars` or `*.auto.tfvars` file if it's not enabled by default: + +```hcl +factories_config = { + paths = { + vpcs = "vpcs" + } +} +``` + +## Usage + +To deploy this dataset, configure your `2-project-factory` stage to use this directory by setting the `dataset` attribute in `factories_config`: + +```hcl +factories_config = { + dataset = "datasets/data-platform" +} +``` diff --git a/fast/stages/2-project-factory/datasets/data-platform/aspect-types/core-0/basic.yaml b/fast/stages/2-project-factory/datasets/data-platform/aspect-types/core-0/basic.yaml new file mode 100644 index 000000000..cbd0c0a79 --- /dev/null +++ b/fast/stages/2-project-factory/datasets/data-platform/aspect-types/core-0/basic.yaml @@ -0,0 +1,46 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../../../schemas/aspect-type.schema.json + +display_name: "Basic template" +metadata_template: | + { + "name": "tf-basic-template", + "type": "record", + "recordFields": [ + { + "name": "source", + "type": "string", + "annotations": { + "displayName": "Source", + "description": "Specifies the source of data." + }, + "index": 1, + "constraints": { + "required": true + } + }, + { + "name": "owner", + "type": "string", + "annotations": { + "displayName": "Owner", + "description": "Specifies the data owner." + }, + "index": 2, + "constraints": {} + } + ] + } diff --git a/fast/stages/2-project-factory/datasets/data-platform/defaults.yaml b/fast/stages/2-project-factory/datasets/data-platform/defaults.yaml new file mode 100644 index 000000000..2c76aab99 --- /dev/null +++ b/fast/stages/2-project-factory/datasets/data-platform/defaults.yaml @@ -0,0 +1,34 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../schemas/defaults.schema.json + +projects: + defaults: + locations: + bigquery: $locations:primary + storage: $locations:primary + merges: + services: + - logging.googleapis.com + - monitoring.googleapis.com + +context: + locations: + primary: europe-west1 + iam_principals: + dp-platform: "group:dp-platform@example.com" + dp-domain-a: "group:dp-domain-a@example.com" + dp-product-a-0: "group:dp-product-a-0@example.com" + data-consumer-bi: "group:data-consumer-bi@example.com" diff --git a/fast/stages/2-project-factory/datasets/data-platform/diagram-data-platform.png b/fast/stages/2-project-factory/datasets/data-platform/diagram-data-platform.png new file mode 100644 index 000000000..c01a73853 Binary files /dev/null and b/fast/stages/2-project-factory/datasets/data-platform/diagram-data-platform.png differ diff --git a/fast/stages/2-project-factory/datasets/data-platform/diagram-folders.png b/fast/stages/2-project-factory/datasets/data-platform/diagram-folders.png new file mode 100644 index 000000000..8616b00d2 Binary files /dev/null and b/fast/stages/2-project-factory/datasets/data-platform/diagram-folders.png differ diff --git a/fast/stages/2-project-factory/datasets/data-platform/folders/domain-0/.config.yaml b/fast/stages/2-project-factory/datasets/data-platform/folders/domain-0/.config.yaml new file mode 100644 index 000000000..efce94474 --- /dev/null +++ b/fast/stages/2-project-factory/datasets/data-platform/folders/domain-0/.config.yaml @@ -0,0 +1,42 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../../../schemas/folder.schema.json + +parent: $folder_ids:data-platform +name: Data Domain 0 + +# iam_bindings: +# bigquery_metadata_viewer: +# members: +# - $iam_principals:dp-platform +# - $iam_principals:dp-domain-a +# - $iam_principals:dp-product-a-0 +# - data-consumer-bi +# role: roles/bigquery.metadataViewer +# condition: +# title: exposure +# description: Expose via secure tag. +# expression: resource.matchTagId('$tag_values:core-0/exposure/public') +# dataplex_catalog_viewer: +# members: +# - $iam_principals:dp-platform +# - $iam_principals:dp-domain-a +# - $iam_principals:dp-product-a-0 +# - data-consumer-bi +# role: roles/dataplex.catalogViewer +# condition: +# title: exposure +# description: Expose via secure tag. +# expression: resource.matchTagId('$tag_values:core-0/exposure/public') diff --git a/fast/stages/2-project-factory/datasets/data-platform/project-templates/data-product.yaml b/fast/stages/2-project-factory/datasets/data-platform/project-templates/data-product.yaml new file mode 100644 index 000000000..4b497002c --- /dev/null +++ b/fast/stages/2-project-factory/datasets/data-platform/project-templates/data-product.yaml @@ -0,0 +1,25 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + - bigquery.googleapis.com + - cloudresourcemanager.googleapis.com + - datacatalog.googleapis.com + - dataplex.googleapis.com + - datastream.googleapis.com + - logging.googleapis.com + - monitoring.googleapis.com + - notebooks.googleapis.com + - storage.googleapis.com + - storage-component.googleapis.com diff --git a/fast/stages/2-project-factory/datasets/data-platform/projects/core-0.yaml b/fast/stages/2-project-factory/datasets/data-platform/projects/core-0.yaml new file mode 100644 index 000000000..729f5618f --- /dev/null +++ b/fast/stages/2-project-factory/datasets/data-platform/projects/core-0.yaml @@ -0,0 +1,36 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../../schemas/project.schema.json + +parent: $folder_ids:data-platform +name: prod-dp-core-0 +services: + - bigquery.googleapis.com + - datacatalog.googleapis.com + - logging.googleapis.com + - monitoring.googleapis.com + - storage.googleapis.com + # Uncomment the following lines to enable a centralized Composer environment + # - composer.googleapis.com +# service_accounts: + # Uncomment the following lines to enable a centralized Composer environment + # composer: + # iam_project_roles: + # roles/composer.worker: + # - $iam_principals:service_accounts/_self_/composer +factories_config: + aspect_types: aspect-types/core-0 + tags: tags/core-0 + data_catalog_taxonomy: taxonomies/tags.yaml diff --git a/fast/stages/2-project-factory/datasets/data-platform/projects/domain-0/product-0.yaml b/fast/stages/2-project-factory/datasets/data-platform/projects/domain-0/product-0.yaml new file mode 100644 index 000000000..73172257a --- /dev/null +++ b/fast/stages/2-project-factory/datasets/data-platform/projects/domain-0/product-0.yaml @@ -0,0 +1,104 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../../../schemas/project.schema.json + +parent: $folder_ids:domain-0 +name: prod-dp-dd0-p0 +services: + - bigquery.googleapis.com + - cloudaicompanion.googleapis.com + - cloudresourcemanager.googleapis.com + - composer.googleapis.com + - datacatalog.googleapis.com + - dataplex.googleapis.com + - datalineage.googleapis.com + - logging.googleapis.com + - monitoring.googleapis.com + - storage.googleapis.com +automation: + project: $project_ids:shared-0 + service_accounts: + iac-ro: + display_name: Product 0/0 (ro) + iam: + roles/iam.serviceAccountTokenCreator: + - $iam_principals:dp-product-a-0 + iac-rw: + display_name: Product 0/0 (rw) + iam: + roles/iam.serviceAccountTokenCreator: + - $iam_principals:dp-product-a-0 + bucket: + name: tf-state + iam: + roles/storage.admin: + - $iam_principals:service_accounts/product-0/automation/iac-rw + roles/storage.objectViewer: + - $iam_principals:service_accounts/product-0/automation/iac-ro + - $iam_principals:dp-product-a-0 + versioning: true + +service_accounts: + processing: + display_name: Processing service account. + iam: + roles/iam.serviceAccountUser: + - $iam_principals:service_accounts/product-0/automation/iac-rw + +datasets: + # TODO: this dataset is exposed to consumers via the 'exposure/public' tag + public: + iam: + roles/bigquery.dataViewer: + - $iam_principals:data-consumer-bi + tag_bindings: + exposure: $tag_values:core-0/exposure/public + + # TODO: this dataset is internal/private + private: {} + +buckets: + # TODO: this bucket is exposed to consumers via the 'exposure/public' tag + public: + iam: + roles/storage.objectViewer: + - $iam_principals:data-consumer-bi + tag_bindings: + exposure: $tag_values:core-0/exposure/public + + # TODO: this bucket is internal/private + private: {} + +iam_by_principals: + $iam_principals:service_accounts/product-0/automation/iac-rw: + - roles/owner + $iam_principals:service_accounts/product-0/automation/iac-ro: + - roles/viewer + $iam_principals:service_accounts/_self_/processing: + - roles/bigquery.dataEditor + - roles/bigquery.jobUser + - roles/dataflow.admin + - roles/dataproc.editor + - roles/dataproc.worker + - roles/iam.serviceAccountUser + - roles/storage.bucketViewer + - roles/storage.objectAdmin + + +# shared_vpc_service_config: +# host_project: $project_ids:net-dev-0 +# service_agent_iam: +# roles/compute.networkUser: +# - $service_agents:dataproc diff --git a/fast/stages/2-project-factory/datasets/data-platform/projects/domain-0/shared-0.yaml b/fast/stages/2-project-factory/datasets/data-platform/projects/domain-0/shared-0.yaml new file mode 100644 index 000000000..53f1bb4d9 --- /dev/null +++ b/fast/stages/2-project-factory/datasets/data-platform/projects/domain-0/shared-0.yaml @@ -0,0 +1,73 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../../../schemas/project.schema.json + +parent: $folder_ids:domain-0 +name: prod-dp-dd-0 +services: + - bigquery.googleapis.com + - datacatalog.googleapis.com + - dataplex.googleapis.com + - datalineage.googleapis.com + - logging.googleapis.com + - monitoring.googleapis.com + - storage.googleapis.com + - composer.googleapis.com +automation: + project: $project_ids:core-0 + service_accounts: + iac-ro: + display_name: Domain 0 (ro) + iam: + roles/iam.serviceAccountTokenCreator: + - $iam_principals:dp-product-a-0 + iac-rw: + display_name: Domain 0 (rw) + iam: + roles/iam.serviceAccountTokenCreator: + - $iam_principals:dp-product-a-0 + bucket: + name: tf-state + iam: + roles/storage.admin: + - $iam_principals:service_accounts/shared-0/automation/iac-rw + roles/storage.objectViewer: + - $iam_principals:service_accounts/shared-0/automation/iac-ro + - $iam_principals:dp-product-a-0 + versioning: true +iam_by_principals: + $iam_principals:service_accounts/shared-0/automation/iac-rw: + - roles/owner + $iam_principals:service_accounts/shared-0/automation/iac-ro: + - roles/viewer + $iam_principals:service_accounts/_self_/composer: + - roles/composer.worker +service_accounts: + composer: {} + +# TODO: Uncomment the following block if you want to use Shared VPC instead of local VPCs. +# shared_vpc_service_config: +# host_project: $project_ids:net-dev-0 +# service_agent_iam: +# roles/composer.sharedVpcAgent: +# - $service_agents:composer +# roles/compute.networkUser: +# - $service_agents:composer +# network_users: +# - $iam_principals:service_accounts/_self_/composer + +# If this project contains a local VPC, it should be configured as a Shared VPC host. +# shared_vpc_host_config: +# enabled: true diff --git a/fast/stages/2-project-factory/datasets/data-platform/tags/core-0/exposure.yaml b/fast/stages/2-project-factory/datasets/data-platform/tags/core-0/exposure.yaml new file mode 100644 index 000000000..1c9f06fe2 --- /dev/null +++ b/fast/stages/2-project-factory/datasets/data-platform/tags/core-0/exposure.yaml @@ -0,0 +1,23 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../../../schemas/tags.schema.json + +description: "Data exposure controls." +# iam: +# "roles/resourcemanager.tagViewer": +# - "group:finance-team@example.com" +values: + public: + description: "Data exposure allowed." diff --git a/fast/stages/2-project-factory/datasets/data-platform/taxonomies/tags.yaml b/fast/stages/2-project-factory/datasets/data-platform/taxonomies/tags.yaml new file mode 100644 index 000000000..71cdad1c2 --- /dev/null +++ b/fast/stages/2-project-factory/datasets/data-platform/taxonomies/tags.yaml @@ -0,0 +1,22 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +description: Taxonomy for data platform. +tags: + high: + description: High sensitivity data. + medium: + description: Medium sensitivity data. + low: + description: Low sensitivity data. diff --git a/fast/stages/2-project-factory/datasets/data-platform/vpcs/README.md b/fast/stages/2-project-factory/datasets/data-platform/vpcs/README.md new file mode 100644 index 000000000..98126df2e --- /dev/null +++ b/fast/stages/2-project-factory/datasets/data-platform/vpcs/README.md @@ -0,0 +1,29 @@ +# Project-Local VPCs + +This folder can contain YAML files defining VPCs that are local to specific projects. +The structure should follow the `net-vpc-factory` pattern: + +1. Create a folder for the VPC (e.g., `my-vpc-0`). +2. Inside that folder, create a `.config.yaml` file. +3. Define the VPC configuration in `.config.yaml`. + +Example `.config.yaml` (see `domain-0/.config.yaml`): + +```yaml +name: domain-0 +project_id: $project_ids:shared-0 # Use context interpolation for project IDs +subnets: + - name: default + ip_cidr_range: 10.0.0.0/24 + region: $locations:primary +``` + +Note: You must also enable the `vpcs` factory path in your `terraform.tfvars` or `*.auto.tfvars` file if it's not enabled by default: + +```hcl +factories_config = { + paths = { + vpcs = "vpcs" + } +} +``` diff --git a/fast/stages/2-project-factory/datasets/data-platform/vpcs/domain-0/.config.yaml b/fast/stages/2-project-factory/datasets/data-platform/vpcs/domain-0/.config.yaml new file mode 100644 index 000000000..15371b0b7 --- /dev/null +++ b/fast/stages/2-project-factory/datasets/data-platform/vpcs/domain-0/.config.yaml @@ -0,0 +1,20 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: domain-0 +project_id: $project_ids:shared-0 +subnets: + - name: example-subnet-0 + ip_cidr_range: 10.0.0.0/24 + region: $locations:primary diff --git a/fast/stages/2-project-factory/main.tf b/fast/stages/2-project-factory/main.tf index 338e16262..65d58c6c6 100644 --- a/fast/stages/2-project-factory/main.tf +++ b/fast/stages/2-project-factory/main.tf @@ -64,6 +64,10 @@ locals { } ) } + vpc_defaults = { + defaults = try(local.defaults.vpcs.defaults, {}) + overrides = try(local.defaults.vpcs.overrides, {}) + } subnet_self_links = flatten([ for net, subnets in var.subnet_self_links : [ for subnet_name, subnet_link in subnets : { diff --git a/fast/stages/2-project-factory/outputs.tf b/fast/stages/2-project-factory/outputs.tf index 1dc0e6c3c..cf6d2dfdc 100644 --- a/fast/stages/2-project-factory/outputs.tf +++ b/fast/stages/2-project-factory/outputs.tf @@ -19,6 +19,11 @@ output "projects" { value = module.factory.projects } +output "vpcs" { + description = "VPCs." + value = module.vpc-factory.vpcs +} + resource "google_storage_bucket_object" "version" { count = fileexists("fast_version.txt") ? 1 : 0 bucket = var.automation.outputs_bucket diff --git a/fast/stages/2-project-factory/schemas/aspect-type.schema.json b/fast/stages/2-project-factory/schemas/aspect-type.schema.json new file mode 100644 index 000000000..49a8f8e31 --- /dev/null +++ b/fast/stages/2-project-factory/schemas/aspect-type.schema.json @@ -0,0 +1,125 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Dataplex Aspect Type", + "type": "object", + "additionalProperties": false, + "properties": { + "description": { + "type": "string" + }, + "display_name": { + "type": "string" + }, + "labels": { + "type": "object" + }, + "metadata_template": { + "type": "string" + }, + "iam": { + "$ref": "#/$defs/iam" + }, + "iam_bindings": { + "$ref": "#/$defs/iam_bindings" + }, + "iam_bindings_additive": { + "$ref": "#/$defs/iam_bindings_additive" + } + }, + "$defs": { + "iam": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^(?:roles/|\\$custom_roles:|organizations/[0-9]+/roles/|([a-z0-9.]+:)?projects/[a-z0-9-]+/roles/)": { + "type": "array", + "items": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:||\\$iam_principals:[a-z0-9_-]+)" + } + } + } + }, + "iam_bindings": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "members": { + "type": "array", + "items": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\\$iam_principals:[a-z0-9_-]+)" + } + }, + "role": { + "type": "string", + "pattern": "^(?:roles/|\\$custom_roles:|organizations/[0-9]+/roles/|([a-z0-9.]+:)?projects/[a-z0-9-]+/roles/)" + }, + "condition": { + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "title" + ], + "properties": { + "expression": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + } + } + } + } + } + } + }, + "iam_bindings_additive": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "member": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\\$iam_principals:[a-z0-9_-]+)" + }, + "role": { + "type": "string", + "pattern": "^(?:roles/|\\$custom_roles:|organizations/[0-9]+/roles/|([a-z0-9.]+:)?projects/[a-z0-9-]+/roles/)" + }, + "condition": { + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "title" + ], + "properties": { + "expression": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + } + } + } + } + } + } + } + } +} diff --git a/fast/stages/2-project-factory/schemas/aspect-type.schema.md b/fast/stages/2-project-factory/schemas/aspect-type.schema.md new file mode 100644 index 000000000..81576d96e --- /dev/null +++ b/fast/stages/2-project-factory/schemas/aspect-type.schema.md @@ -0,0 +1,50 @@ +# Dataplex Aspect Type + + + +## Properties + +*additional properties: false* + +- **description**: *string* +- **display_name**: *string* +- **labels**: *object* +- **metadata_template**: *string* +- **iam**: *reference([iam](#refs-iam))* +- **iam_bindings**: *reference([iam_bindings](#refs-iam_bindings))* +- **iam_bindings_additive**: *reference([iam_bindings_additive](#refs-iam_bindings_additive))* + +## Definitions + +- **iam**: *object* +
*additional properties: false* + - **`^(?:roles/|\$custom_roles:|organizations/[0-9]+/roles/|([a-z0-9.]+:)?projects/[a-z0-9-]+/roles/)`**: *array* + - items: *string* +
*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:||\$iam_principals:[a-z0-9_-]+)* +- **iam_bindings**: *object* +
*additional properties: false* + - **`^[a-z0-9_-]+$`**: *object* +
*additional properties: false* + - **members**: *array* + - items: *string* +
*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\$iam_principals:[a-z0-9_-]+)* + - **role**: *string* +
*pattern: ^(?:roles/|\$custom_roles:|organizations/[0-9]+/roles/|([a-z0-9.]+:)?projects/[a-z0-9-]+/roles/)* + - **condition**: *object* +
*additional properties: false* + - ⁺**expression**: *string* + - ⁺**title**: *string* + - **description**: *string* +- **iam_bindings_additive**: *object* +
*additional properties: false* + - **`^[a-z0-9_-]+$`**: *object* +
*additional properties: false* + - **member**: *string* +
*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\$iam_principals:[a-z0-9_-]+)* + - **role**: *string* +
*pattern: ^(?:roles/|\$custom_roles:|organizations/[0-9]+/roles/|([a-z0-9.]+:)?projects/[a-z0-9-]+/roles/)* + - **condition**: *object* +
*additional properties: false* + - ⁺**expression**: *string* + - ⁺**title**: *string* + - **description**: *string* diff --git a/fast/stages/2-project-factory/schemas/defaults.schema.json b/fast/stages/2-project-factory/schemas/defaults.schema.json index 117783a93..c8bf0d0fa 100644 --- a/fast/stages/2-project-factory/schemas/defaults.schema.json +++ b/fast/stages/2-project-factory/schemas/defaults.schema.json @@ -472,10 +472,231 @@ } } }, + "vpcs": { + "type": "object", + "additionalProperties": false, + "properties": { + "defaults": { + "type": "object", + "additionalProperties": false, + "properties": { + "project_id": { + "type": "string" + }, + "description": { + "type": "string" + }, + "auto_create_subnetworks": { + "type": "boolean" + }, + "delete_default_routes_on_create": { + "type": "boolean" + }, + "mtu": { + "type": "number", + "minimum": 1460, + "maximum": 1500 + }, + "routing_mode": { + "type": "string", + "enum": [ + "GLOBAL", + "REGIONAL" + ] + }, + "firewall_policy_enforcement_order": { + "type": "string", + "enum": [ + "BEFORE_CLASSIC_FIREWALL", + "AFTER_CLASSIC_FIREWALL" + ] + }, + "create_googleapis_routes": { + "type": "object", + "additionalProperties": false, + "properties": { + "directpath": { + "type": "boolean" + }, + "directpath-6": { + "type": "boolean" + }, + "private": { + "type": "boolean" + }, + "private-6": { + "type": "boolean" + }, + "restricted": { + "type": "boolean" + }, + "restricted-6": { + "type": "boolean" + } + } + }, + "dns_policy": { + "type": "object", + "additionalProperties": false, + "properties": { + "inbound": { + "type": "boolean" + }, + "logging": { + "type": "boolean" + }, + "outbound": { + "type": "object", + "additionalProperties": false, + "properties": { + "private_ns": { + "type": "array", + "items": { + "type": "string" + } + }, + "public_ns": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + }, + "ipv6_config": { + "type": "object", + "additionalProperties": false, + "properties": { + "enable_ula_internal": { + "type": "boolean" + }, + "internal_range": { + "type": "string" + } + } + } + } + }, + "overrides": { + "type": "object", + "additionalProperties": false, + "properties": { + "project_id": { + "type": "string" + }, + "description": { + "type": "string" + }, + "auto_create_subnetworks": { + "type": "boolean" + }, + "delete_default_routes_on_create": { + "type": "boolean" + }, + "mtu": { + "type": "number", + "minimum": 1460, + "maximum": 1500 + }, + "routing_mode": { + "type": "string", + "enum": [ + "GLOBAL", + "REGIONAL" + ] + }, + "firewall_policy_enforcement_order": { + "type": "string", + "enum": [ + "BEFORE_CLASSIC_FIREWALL", + "AFTER_CLASSIC_FIREWALL" + ] + }, + "create_googleapis_routes": { + "type": "object", + "additionalProperties": false, + "properties": { + "directpath": { + "type": "boolean" + }, + "directpath-6": { + "type": "boolean" + }, + "private": { + "type": "boolean" + }, + "private-6": { + "type": "boolean" + }, + "restricted": { + "type": "boolean" + }, + "restricted-6": { + "type": "boolean" + } + } + }, + "dns_policy": { + "type": "object", + "additionalProperties": false, + "properties": { + "inbound": { + "type": "boolean" + }, + "logging": { + "type": "boolean" + }, + "outbound": { + "type": "object", + "additionalProperties": false, + "properties": { + "private_ns": { + "type": "array", + "items": { + "type": "string" + } + }, + "public_ns": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + }, + "ipv6_config": { + "type": "object", + "additionalProperties": false, + "properties": { + "enable_ula_internal": { + "type": "boolean" + }, + "internal_range": { + "type": "string" + } + } + } + } + } + } + }, "context": { "type": "object", "additionalProperties": false, "properties": { + "cidr_ranges_sets": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + } + }, "custom_roles": { "type": "object", "additionalProperties": { diff --git a/fast/stages/2-project-factory/schemas/defaults.schema.md b/fast/stages/2-project-factory/schemas/defaults.schema.md index eee6f3b76..689359e72 100644 --- a/fast/stages/2-project-factory/schemas/defaults.schema.md +++ b/fast/stages/2-project-factory/schemas/defaults.schema.md @@ -129,8 +129,78 @@ - **vpc_sc**: *object* - ⁺**perimeter_name**: *string* - **is_dry_run**: *boolean* +- **vpcs**: *object* +
*additional properties: false* + - **defaults**: *object* +
*additional properties: false* + - **project_id**: *string* + - **description**: *string* + - **auto_create_subnetworks**: *boolean* + - **delete_default_routes_on_create**: *boolean* + - **mtu**: *number* + - **routing_mode**: *string* +
*enum: ['GLOBAL', 'REGIONAL']* + - **firewall_policy_enforcement_order**: *string* +
*enum: ['BEFORE_CLASSIC_FIREWALL', 'AFTER_CLASSIC_FIREWALL']* + - **create_googleapis_routes**: *object* +
*additional properties: false* + - **directpath**: *boolean* + - **directpath-6**: *boolean* + - **private**: *boolean* + - **private-6**: *boolean* + - **restricted**: *boolean* + - **restricted-6**: *boolean* + - **dns_policy**: *object* +
*additional properties: false* + - **inbound**: *boolean* + - **logging**: *boolean* + - **outbound**: *object* +
*additional properties: false* + - **private_ns**: *array* + - items: *string* + - **public_ns**: *array* + - items: *string* + - **ipv6_config**: *object* +
*additional properties: false* + - **enable_ula_internal**: *boolean* + - **internal_range**: *string* + - **overrides**: *object* +
*additional properties: false* + - **project_id**: *string* + - **description**: *string* + - **auto_create_subnetworks**: *boolean* + - **delete_default_routes_on_create**: *boolean* + - **mtu**: *number* + - **routing_mode**: *string* +
*enum: ['GLOBAL', 'REGIONAL']* + - **firewall_policy_enforcement_order**: *string* +
*enum: ['BEFORE_CLASSIC_FIREWALL', 'AFTER_CLASSIC_FIREWALL']* + - **create_googleapis_routes**: *object* +
*additional properties: false* + - **directpath**: *boolean* + - **directpath-6**: *boolean* + - **private**: *boolean* + - **private-6**: *boolean* + - **restricted**: *boolean* + - **restricted-6**: *boolean* + - **dns_policy**: *object* +
*additional properties: false* + - **inbound**: *boolean* + - **logging**: *boolean* + - **outbound**: *object* +
*additional properties: false* + - **private_ns**: *array* + - items: *string* + - **public_ns**: *array* + - items: *string* + - **ipv6_config**: *object* +
*additional properties: false* + - **enable_ula_internal**: *boolean* + - **internal_range**: *string* - **context**: *object*
*additional properties: false* + - **cidr_ranges_sets**: *object* +
*additional properties: array* - **custom_roles**: *object*
*additional properties: string* - **email_addresses**: *object* diff --git a/fast/stages/2-project-factory/schemas/project.schema.json b/fast/stages/2-project-factory/schemas/project.schema.json index d65fa4a58..94c224062 100644 --- a/fast/stages/2-project-factory/schemas/project.schema.json +++ b/fast/stages/2-project-factory/schemas/project.schema.json @@ -111,6 +111,9 @@ "type": "object", "additionalProperties": false, "properties": { + "display_name": { + "type": "string" + }, "description": { "type": "string" }, @@ -297,9 +300,15 @@ "type": "object", "additionalProperties": false, "properties": { + "aspect_types": { + "type": "string" + }, "custom_roles": { "type": "string" }, + "data_catalog_taxonomy": { + "type": "string" + }, "observability": { "type": "string" }, diff --git a/fast/stages/2-project-factory/schemas/project.schema.md b/fast/stages/2-project-factory/schemas/project.schema.md index a14fb866d..31998ca8a 100644 --- a/fast/stages/2-project-factory/schemas/project.schema.md +++ b/fast/stages/2-project-factory/schemas/project.schema.md @@ -37,6 +37,7 @@
*additional properties: false* - **`^[a-z0-9-]+$`**: *object*
*additional properties: false* + - **display_name**: *string* - **description**: *string* - **prefix**: *string* - **iam**: *reference([iam](#refs-iam))* @@ -95,7 +96,9 @@
*enum: ['PREVENT', 'DELETE', 'ABANDON']* - **factories_config**: *object*
*additional properties: false* + - **aspect_types**: *string* - **custom_roles**: *string* + - **data_catalog_taxonomy**: *string* - **observability**: *string* - **org_policies**: *string* - **quotas**: *string* diff --git a/fast/stages/2-project-factory/schemas/tags.schema.json b/fast/stages/2-project-factory/schemas/tags.schema.json new file mode 100644 index 000000000..22b5357b4 --- /dev/null +++ b/fast/stages/2-project-factory/schemas/tags.schema.json @@ -0,0 +1,174 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Resource Manager Tags", + "type": "object", + "additionalProperties": false, + "properties": { + "allowed_values_regex": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "id": { + "type": "string" + }, + "network": { + "type": "string" + }, + "iam": { + "$ref": "#/$defs/iam" + }, + "iam_bindings": { + "$ref": "#/$defs/iam_bindings" + }, + "iam_bindings_additive": { + "$ref": "#/$defs/iam_bindings_additive" + }, + "values": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z-][^\\\\'\"/]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "id": { + "type": "string" + }, + "iam": { + "$ref": "#/$defs/iam" + }, + "iam_bindings": { + "$ref": "#/$defs/iam_bindings" + }, + "iam_bindings_additive": { + "$ref": "#/$defs/iam_bindings_additive" + } + } + } + } + } + }, + "allOf": [ + { + "if": { + "required": [ + "allowed_values_regex" + ] + }, + "then": { + "not": { + "required": [ + "values" + ] + } + } + } + ], + "$defs": { + "iam": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^roles/": { + "type": "array", + "items": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\\$iam_principals:[a-z0-9_-]+)" + } + } + } + }, + "iam_bindings": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "members": { + "type": "array", + "items": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\\$iam_principals:[a-z0-9_-]+)" + } + }, + "role": { + "type": "string", + "pattern": "^roles/" + }, + "condition": { + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "title" + ], + "properties": { + "expression": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + } + } + } + } + } + } + }, + "iam_bindings_additive": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "member": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\\$iam_principals:[a-z0-9_-]+)" + }, + "role": { + "type": "string", + "pattern": "^[a-zA-Z0-9_/]+$" + }, + "condition": { + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "title" + ], + "properties": { + "expression": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + } + } + } + } + } + } + } + } +} diff --git a/fast/stages/2-project-factory/schemas/tags.schema.md b/fast/stages/2-project-factory/schemas/tags.schema.md new file mode 100644 index 000000000..592e01ada --- /dev/null +++ b/fast/stages/2-project-factory/schemas/tags.schema.md @@ -0,0 +1,61 @@ +# Resource Manager Tags + + + +## Properties + +*additional properties: false* + +- **allowed_values_regex**: *string* +- **name**: *string* +- **description**: *string* +- **id**: *string* +- **network**: *string* +- **iam**: *reference([iam](#refs-iam))* +- **iam_bindings**: *reference([iam_bindings](#refs-iam_bindings))* +- **iam_bindings_additive**: *reference([iam_bindings_additive](#refs-iam_bindings_additive))* +- **values**: *object* +
*additional properties: false* + - **`^[a-z-][^\\'"/]+$`**: *object* +
*additional properties: false* + - **name**: *string* + - **description**: *string* + - **id**: *string* + - **iam**: *reference([iam](#refs-iam))* + - **iam_bindings**: *reference([iam_bindings](#refs-iam_bindings))* + - **iam_bindings_additive**: *reference([iam_bindings_additive](#refs-iam_bindings_additive))* + +## Definitions + +- **iam**: *object* +
*additional properties: false* + - **`^roles/`**: *array* + - items: *string* +
*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\$iam_principals:[a-z0-9_-]+)* +- **iam_bindings**: *object* +
*additional properties: false* + - **`^[a-z0-9_-]+$`**: *object* +
*additional properties: false* + - **members**: *array* + - items: *string* +
*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\$iam_principals:[a-z0-9_-]+)* + - **role**: *string* +
*pattern: ^roles/* + - **condition**: *object* +
*additional properties: false* + - ⁺**expression**: *string* + - ⁺**title**: *string* + - **description**: *string* +- **iam_bindings_additive**: *object* +
*additional properties: false* + - **`^[a-z0-9_-]+$`**: *object* +
*additional properties: false* + - **member**: *string* +
*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\$iam_principals:[a-z0-9_-]+)* + - **role**: *string* +
*pattern: ^[a-zA-Z0-9_/]+$* + - **condition**: *object* +
*additional properties: false* + - ⁺**expression**: *string* + - ⁺**title**: *string* + - **description**: *string* diff --git a/fast/stages/2-project-factory/variables.tf b/fast/stages/2-project-factory/variables.tf index e637055d3..a23760be1 100644 --- a/fast/stages/2-project-factory/variables.tf +++ b/fast/stages/2-project-factory/variables.tf @@ -17,6 +17,7 @@ variable "context" { description = "Context-specific interpolations." type = object({ + cidr_ranges_sets = optional(map(list(string)), {}) condition_vars = optional(map(map(string)), {}) custom_roles = optional(map(string), {}) email_addresses = optional(map(string), {}) @@ -43,10 +44,12 @@ variable "factories_config" { type = object({ dataset = optional(string, "datasets/classic") paths = optional(object({ - defaults = optional(string, "defaults.yaml") - folders = optional(string, "folders") - projects = optional(string, "projects") - budgets = optional(string) + defaults = optional(string, "defaults.yaml") + folders = optional(string, "folders") + projects = optional(string, "projects") + project_templates = optional(string, "project-templates") + budgets = optional(string) + vpcs = optional(string, "vpcs") }), {}) }) nullable = false diff --git a/fast/stages/2-project-factory/vpcs.tf b/fast/stages/2-project-factory/vpcs.tf new file mode 100644 index 000000000..a6f77425f --- /dev/null +++ b/fast/stages/2-project-factory/vpcs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description VPC factory. + +module "vpc-factory" { + source = "../../../modules/net-vpc-factory" + data_defaults = local.vpc_defaults.defaults + data_overrides = local.vpc_defaults.overrides + factories_config = { + basepath = var.factories_config.dataset + paths = { + vpcs = var.factories_config.paths.vpcs + } + } + context = merge(local.context, { + project_ids = merge( + local.context.project_ids, + module.factory.project_ids + ) + }) +} diff --git a/fast/stages/2-security/schemas/project.schema.json b/fast/stages/2-security/schemas/project.schema.json index d65fa4a58..94c224062 100644 --- a/fast/stages/2-security/schemas/project.schema.json +++ b/fast/stages/2-security/schemas/project.schema.json @@ -111,6 +111,9 @@ "type": "object", "additionalProperties": false, "properties": { + "display_name": { + "type": "string" + }, "description": { "type": "string" }, @@ -297,9 +300,15 @@ "type": "object", "additionalProperties": false, "properties": { + "aspect_types": { + "type": "string" + }, "custom_roles": { "type": "string" }, + "data_catalog_taxonomy": { + "type": "string" + }, "observability": { "type": "string" }, diff --git a/fast/stages/2-security/schemas/project.schema.md b/fast/stages/2-security/schemas/project.schema.md index a14fb866d..31998ca8a 100644 --- a/fast/stages/2-security/schemas/project.schema.md +++ b/fast/stages/2-security/schemas/project.schema.md @@ -37,6 +37,7 @@
*additional properties: false* - **`^[a-z0-9-]+$`**: *object*
*additional properties: false* + - **display_name**: *string* - **description**: *string* - **prefix**: *string* - **iam**: *reference([iam](#refs-iam))* @@ -95,7 +96,9 @@
*enum: ['PREVENT', 'DELETE', 'ABANDON']* - **factories_config**: *object*
*additional properties: false* + - **aspect_types**: *string* - **custom_roles**: *string* + - **data_catalog_taxonomy**: *string* - **observability**: *string* - **org_policies**: *string* - **quotas**: *string* diff --git a/modules/data-catalog-policy-tag/README.md b/modules/data-catalog-policy-tag/README.md index 717e95ab2..42fe07153 100644 --- a/modules/data-catalog-policy-tag/README.md +++ b/modules/data-catalog-policy-tag/README.md @@ -9,6 +9,7 @@ Note: Data Catalog is still in beta, hence this module currently uses the beta p - [Examples](#examples) - [Simple Taxonomy with policy tags](#simple-taxonomy-with-policy-tags) - [Taxonomy with IAM binding](#taxonomy-with-iam-binding) + - [Factory](#factory) - [Variables](#variables) - [Outputs](#outputs) - [TODO](#todo) @@ -76,22 +77,66 @@ module "cmn-dc" { } # tftest modules=1 resources=7 ``` +### Factory + +```hcl +module "taxonomy" { + source = "./fabric/modules/data-catalog-policy-tag" + project_id = "my-project" + name = "taxonomy" + location = "europe-west1" + context = { + iam_principals = { + user-a = "user:a@example.org" + user-b = "user:b@example.org" + } + } + factories_config = { + taxonomy = "data-catalog/taxonomy.yaml" + } +} +# tftest modules=1 resources=6 files=factory +``` + +```yaml +description: taxonomy description +activated_policy_types: + - FINE_GRAINED_ACCESS_CONTROL +iam: + roles/viewer: + - $iam_principals:user-a +tags: + tag-a: + description: tag a description + iam: + roles/viewer: + - $iam_principals:user-b + tag-b: + description: tag b description + iam_bindings: + viewer: + role: roles/viewer + members: + - user:c@example.org +# tftest-file id=factory path=data-catalog/taxonomy.yaml +``` ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [location](variables.tf#L52) | Data Catalog Taxonomy location. | string | ✓ | | -| [name](variables.tf#L58) | Name of this taxonomy. | string | ✓ | | -| [project_id](variables.tf#L64) | GCP project id. | string | ✓ | | +| [location](variables.tf#L61) | Data Catalog Taxonomy location. | string | ✓ | | +| [name](variables.tf#L67) | Name of this taxonomy. | string | ✓ | | +| [project_id](variables.tf#L73) | GCP project id. | string | ✓ | | | [activated_policy_types](variables.tf#L17) | A list of policy types that are activated for this taxonomy. | list(string) | | ["FINE_GRAINED_ACCESS_CONTROL"] | | [context](variables.tf#L32) | Context-specific interpolations. | object({…}) | | {} | | [description](variables.tf#L45) | Description of this taxonomy. | string | | "Taxonomy - Terraform managed" | +| [factories_config](variables.tf#L52) | Paths to folders and files for the optional factories. | object({…}) | | {} | | [iam](variables-iam.tf#L23) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_bindings](variables-iam.tf#L29) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | | [iam_bindings_additive](variables-iam.tf#L44) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | | [iam_by_principals](variables-iam.tf#L17) | Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid cycle errors. Merged internally with the `iam` variable. | map(list(string)) | | {} | -| [tags](variables.tf#L70) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {} | +| [tags](variables.tf#L79) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {} | ## Outputs diff --git a/modules/data-catalog-policy-tag/iam.tf b/modules/data-catalog-policy-tag/iam.tf index ef3fb914a..521de79e3 100644 --- a/modules/data-catalog-policy-tag/iam.tf +++ b/modules/data-catalog-policy-tag/iam.tf @@ -25,14 +25,35 @@ locals { ] } iam = { - for role in distinct(concat(keys(var.iam), keys(local._iam_principals))) : + for role in distinct(concat( + keys(var.iam), + keys(local._iam_principals), + keys(try(local._factory_data.iam, {})) + )) : role => concat( try(var.iam[role], []), - try(local._iam_principals[role], []) + try(local._iam_principals[role], []), + try(local._factory_data.iam[role], []) ) } + iam_bindings = merge( + var.iam_bindings, + { + for k, v in try(local._factory_data.iam_bindings, {}) : k => merge(v, { + condition = try(v.condition, null) + }) + } + ) + iam_bindings_additive = merge( + var.iam_bindings_additive, + { + for k, v in try(local._factory_data.iam_bindings_additive, {}) : k => merge(v, { + condition = try(v.condition, null) + }) + } + ) tags_iam = flatten([ - for k, v in var.tags : [ + for k, v in local.tags : [ for role, members in v.iam : { tag = k role = role @@ -40,6 +61,22 @@ locals { } ] ]) + tags_iam_bindings = merge({}, [ + for k, v in local.tags : { + for bk, bv in v.iam_bindings : "${k}.${bk}" => merge(bv, { + tag = k + key = bk + }) + } + ]...) + tags_iam_bindings_additive = merge({}, [ + for k, v in local.tags : { + for bk, bv in v.iam_bindings_additive : "${k}.${bk}" => merge(bv, { + tag = k + key = bk + }) + } + ]...) } resource "google_data_catalog_taxonomy_iam_binding" "authoritative" { @@ -55,7 +92,7 @@ resource "google_data_catalog_taxonomy_iam_binding" "authoritative" { resource "google_data_catalog_taxonomy_iam_binding" "bindings" { provider = google-beta - for_each = var.iam_bindings + for_each = local.iam_bindings taxonomy = google_data_catalog_taxonomy.default.id role = lookup(local.ctx.custom_roles, each.value.role, each.value.role) members = [ @@ -76,7 +113,7 @@ resource "google_data_catalog_taxonomy_iam_binding" "bindings" { resource "google_data_catalog_taxonomy_iam_member" "bindings" { provider = google-beta - for_each = var.iam_bindings_additive + for_each = local.iam_bindings_additive taxonomy = google_data_catalog_taxonomy.default.id role = lookup(local.ctx.custom_roles, each.value.role, each.value.role) member = lookup(local.ctx.iam_principals, each.value.member, each.value.member) @@ -104,3 +141,42 @@ resource "google_data_catalog_policy_tag_iam_binding" "authoritative" { lookup(local.ctx.iam_principals, v, v) ] } + +resource "google_data_catalog_policy_tag_iam_binding" "bindings" { + provider = google-beta + for_each = local.tags_iam_bindings + policy_tag = google_data_catalog_policy_tag.default[each.value.tag].name + role = lookup(local.ctx.custom_roles, each.value.role, each.value.role) + members = [ + for v in each.value.members : + lookup(local.ctx.iam_principals, v, v) + ] + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = templatestring( + each.value.condition.expression, var.context.condition_vars + ) + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_data_catalog_policy_tag_iam_member" "bindings" { + provider = google-beta + for_each = local.tags_iam_bindings_additive + policy_tag = google_data_catalog_policy_tag.default[each.value.tag].name + role = lookup(local.ctx.custom_roles, each.value.role, each.value.role) + member = lookup(local.ctx.iam_principals, each.value.member, each.value.member) + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = templatestring( + each.value.condition.expression, var.context.condition_vars + ) + title = each.value.condition.title + description = each.value.condition.description + } + } +} diff --git a/modules/data-catalog-policy-tag/main.tf b/modules/data-catalog-policy-tag/main.tf index 189dc9bb0..0cd5b23d1 100644 --- a/modules/data-catalog-policy-tag/main.tf +++ b/modules/data-catalog-policy-tag/main.tf @@ -27,6 +27,27 @@ locals { project_id = var.project_id == null ? null : lookup( local.ctx.project_ids, var.project_id, var.project_id ) + _factory_data = try( + yamldecode(file(pathexpand(var.factories_config.taxonomy))), + {} + ) + description = try(local._factory_data.description, var.description) + tags = merge(var.tags, { + for k, v in try(local._factory_data.tags, {}) : k => { + description = try(v.description, null) + iam = try(v.iam, {}) + iam_bindings = { + for ik, iv in try(v.iam_bindings, {}) : ik => merge(iv, { + condition = try(iv.condition, null) + }) + } + iam_bindings_additive = { + for ik, iv in try(v.iam_bindings_additive, {}) : ik => merge(iv, { + condition = try(iv.condition, null) + }) + } + } + }) } resource "google_data_catalog_taxonomy" "default" { @@ -34,13 +55,13 @@ resource "google_data_catalog_taxonomy" "default" { project = local.project_id region = local.location display_name = var.name - description = var.description + description = local.description activated_policy_types = var.activated_policy_types } resource "google_data_catalog_policy_tag" "default" { provider = google-beta - for_each = var.tags + for_each = local.tags taxonomy = google_data_catalog_taxonomy.default.id display_name = each.key description = coalesce( diff --git a/modules/data-catalog-policy-tag/schemas/policy-tag.schema.json b/modules/data-catalog-policy-tag/schemas/policy-tag.schema.json new file mode 100644 index 000000000..23da50da8 --- /dev/null +++ b/modules/data-catalog-policy-tag/schemas/policy-tag.schema.json @@ -0,0 +1,153 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": false, + "properties": { + "activated_policy_types": { + "type": "array", + "items": { + "type": "string" + } + }, + "description": { + "type": "string" + }, + "iam": { + "$ref": "#/$defs/iam" + }, + "iam_bindings": { + "$ref": "#/$defs/iam_bindings" + }, + "iam_bindings_additive": { + "$ref": "#/$defs/iam_bindings_additive" + }, + "tags": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^.+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "description": { + "type": "string" + }, + "iam": { + "$ref": "#/$defs/iam" + }, + "iam_bindings": { + "$ref": "#/$defs/iam_bindings" + }, + "iam_bindings_additive": { + "$ref": "#/$defs/iam_bindings_additive" + } + } + } + } + } + }, + "$defs": { + "iam": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^(?:roles/|\\$custom_roles:)": { + "type": "array", + "items": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\\$iam_principals:[a-z0-9_-]+)" + } + } + } + }, + "iam_bindings": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "object", + "additionalProperties": false, + "required": [ + "members", + "role" + ], + "properties": { + "members": { + "type": "array", + "items": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\\$iam_principals:[a-z0-9_-]+)" + } + }, + "role": { + "type": "string", + "pattern": "^(?:roles/|\\$custom_roles:)" + }, + "condition": { + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "title" + ], + "properties": { + "expression": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + } + } + } + } + } + } + }, + "iam_bindings_additive": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "object", + "additionalProperties": false, + "required": [ + "member", + "role" + ], + "properties": { + "member": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\\$iam_principals:[a-z0-9_-]+)" + }, + "role": { + "type": "string", + "pattern": "^(?:roles/|\\$custom_roles:)" + }, + "condition": { + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "title" + ], + "properties": { + "expression": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/modules/data-catalog-policy-tag/schemas/policy-tag.schema.md b/modules/data-catalog-policy-tag/schemas/policy-tag.schema.md new file mode 100644 index 000000000..da41b4360 --- /dev/null +++ b/modules/data-catalog-policy-tag/schemas/policy-tag.schema.md @@ -0,0 +1,57 @@ +# None + + + +## Properties + +*additional properties: false* + +- **activated_policy_types**: *array* + - items: *string* +- **description**: *string* +- **iam**: *reference([iam](#refs-iam))* +- **iam_bindings**: *reference([iam_bindings](#refs-iam_bindings))* +- **iam_bindings_additive**: *reference([iam_bindings_additive](#refs-iam_bindings_additive))* +- **tags**: *object* +
*additional properties: false* + - **`^.+$`**: *object* +
*additional properties: false* + - **description**: *string* + - **iam**: *reference([iam](#refs-iam))* + - **iam_bindings**: *reference([iam_bindings](#refs-iam_bindings))* + - **iam_bindings_additive**: *reference([iam_bindings_additive](#refs-iam_bindings_additive))* + +## Definitions + +- **iam**: *object* +
*additional properties: false* + - **`^(?:roles/|\$custom_roles:)`**: *array* + - items: *string* +
*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\$iam_principals:[a-z0-9_-]+)* +- **iam_bindings**: *object* +
*additional properties: false* + - **`^[a-z0-9_-]+$`**: *object* +
*additional properties: false* + - ⁺**members**: *array* + - items: *string* +
*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\$iam_principals:[a-z0-9_-]+)* + - ⁺**role**: *string* +
*pattern: ^(?:roles/|\$custom_roles:)* + - **condition**: *object* +
*additional properties: false* + - ⁺**expression**: *string* + - ⁺**title**: *string* + - **description**: *string* +- **iam_bindings_additive**: *object* +
*additional properties: false* + - **`^[a-z0-9_-]+$`**: *object* +
*additional properties: false* + - ⁺**member**: *string* +
*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\$iam_principals:[a-z0-9_-]+)* + - ⁺**role**: *string* +
*pattern: ^(?:roles/|\$custom_roles:)* + - **condition**: *object* +
*additional properties: false* + - ⁺**expression**: *string* + - ⁺**title**: *string* + - **description**: *string* diff --git a/modules/data-catalog-policy-tag/variables.tf b/modules/data-catalog-policy-tag/variables.tf index 51a16de8d..94e616494 100644 --- a/modules/data-catalog-policy-tag/variables.tf +++ b/modules/data-catalog-policy-tag/variables.tf @@ -49,6 +49,15 @@ variable "description" { default = "Taxonomy - Terraform managed" } +variable "factories_config" { + description = "Paths to folders and files for the optional factories." + type = object({ + taxonomy = optional(string) + }) + nullable = false + default = {} +} + variable "location" { description = "Data Catalog Taxonomy location." type = string @@ -72,6 +81,24 @@ variable "tags" { type = map(object({ description = optional(string) iam = optional(map(list(string)), {}) + iam_bindings = optional(map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + iam_bindings_additive = optional(map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) })) nullable = false default = {} diff --git a/modules/project-factory/README.md b/modules/project-factory/README.md index 6d89af6c7..f95815588 100644 --- a/modules/project-factory/README.md +++ b/modules/project-factory/README.md @@ -766,6 +766,8 @@ iam: - $iam_principals:service_accounts/dev-tb-app0-0/automation/rw "roles/viewer": - $iam_principals:service_accounts/dev-tb-app0-0/automation/ro +factories_config: + data_catalog_taxonomy: data/taxonomies/sample.yaml shared_vpc_host_config: enabled: true service_accounts: @@ -871,6 +873,7 @@ compute.disableSerialPortAccess: | name | description | modules | resources | |---|---|---|---| +| [aspect-types.tf](./aspect-types.tf) | Aspect types resources. | dataplex-aspect-types | | | [automation.tf](./automation.tf) | None | gcs · iam-service-account | | | [budgets.tf](./budgets.tf) | Billing budget factory locals. | billing-account | | | [folders.tf](./folders.tf) | Folder hierarchy factory resources. | folder | | @@ -885,6 +888,7 @@ compute.disableSerialPortAccess: | [projects-pubsub.tf](./projects-pubsub.tf) | None | pubsub | | | [projects-service-accounts.tf](./projects-service-accounts.tf) | None | iam-service-account | | | [projects.tf](./projects.tf) | None | project | terraform_data | +| [taxonomies.tf](./taxonomies.tf) | Taxonomy resources. | data-catalog-policy-tag | | | [variables-billing.tf](./variables-billing.tf) | None | | | | [variables-folders.tf](./variables-folders.tf) | None | | | | [variables-projects.tf](./variables-projects.tf) | None | | | @@ -907,20 +911,20 @@ compute.disableSerialPortAccess: | name | description | sensitive | |---|---|:---:| -| [folder_ids](outputs.tf#L102) | Folder ids. | | -| [iam_principals](outputs.tf#L107) | IAM principals mappings. | | -| [kms_keys](outputs.tf#L112) | KMS key ids. | | -| [log_buckets](outputs.tf#L117) | Log bucket ids. | | -| [project_ids](outputs.tf#L124) | Project ids. | | -| [project_numbers](outputs.tf#L129) | Project numbers. | | -| [projects](outputs.tf#L136) | Project attributes. | | -| [pubsub_topics](outputs.tf#L141) | PubSub topic ids. | | -| [service_account_emails](outputs.tf#L148) | Service account emails. | | -| [service_account_iam_emails](outputs.tf#L155) | Service account IAM-format emails. | | -| [service_account_ids](outputs.tf#L162) | Service account IDs. | | -| [service_accounts](outputs.tf#L169) | Service account emails. | | -| [service_agents](outputs.tf#L174) | Service agent emails. | | -| [storage_buckets](outputs.tf#L185) | Bucket names. | | +| [folder_ids](outputs.tf#L107) | Folder ids. | | +| [iam_principals](outputs.tf#L112) | IAM principals mappings. | | +| [kms_keys](outputs.tf#L117) | KMS key ids. | | +| [log_buckets](outputs.tf#L122) | Log bucket ids. | | +| [project_ids](outputs.tf#L129) | Project ids. | | +| [project_numbers](outputs.tf#L134) | Project numbers. | | +| [projects](outputs.tf#L141) | Project attributes. | | +| [pubsub_topics](outputs.tf#L146) | PubSub topic ids. | | +| [service_account_emails](outputs.tf#L153) | Service account emails. | | +| [service_account_iam_emails](outputs.tf#L160) | Service account IAM-format emails. | | +| [service_account_ids](outputs.tf#L167) | Service account IDs. | | +| [service_accounts](outputs.tf#L174) | Service account emails. | | +| [service_agents](outputs.tf#L179) | Service agent emails. | | +| [storage_buckets](outputs.tf#L190) | Bucket names. | | ## Tests diff --git a/modules/project-factory/aspect-types.tf b/modules/project-factory/aspect-types.tf new file mode 100644 index 000000000..93b9b8eb8 --- /dev/null +++ b/modules/project-factory/aspect-types.tf @@ -0,0 +1,44 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Aspect types resources. + +module "aspect-types" { + source = "../dataplex-aspect-types" + for_each = { + for k, v in local.projects_input : k => v + if try(v.factories_config.aspect_types, null) != null + } + project_id = module.projects[each.key].project_id + factories_config = { + aspect_types = lookup(each.value.factories_config, "aspect_types", null) == null ? null : try(pathexpand( + var.factories_config.basepath == null || startswith(each.value.factories_config.aspect_types, "/") || startswith(each.value.factories_config.aspect_types, ".") + ? each.value.factories_config.aspect_types : + "${var.factories_config.basepath}/${each.value.factories_config.aspect_types}" + ), null) + } + context = merge(local.ctx, { + iam_principals = merge( + local.ctx_iam_principals, + lookup(local.self_sas_iam_emails, each.key, {}), + local.projects_service_agents + ) + project_ids = merge( + local.ctx.project_ids, + { for k, v in module.projects : k => v.project_id } + ) + }) +} diff --git a/modules/project-factory/data/taxonomies/sample.yaml b/modules/project-factory/data/taxonomies/sample.yaml new file mode 100644 index 000000000..cfaf7b57c --- /dev/null +++ b/modules/project-factory/data/taxonomies/sample.yaml @@ -0,0 +1,26 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +description: taxonomy description +activated_policy_types: + - FINE_GRAINED_ACCESS_CONTROL +iam: + roles/viewer: + - $iam_principals:gcp-devops +tags: + tag-a: + description: tag a description + iam: + roles/viewer: + - $iam_principals:gcp-devops diff --git a/modules/project-factory/outputs.tf b/modules/project-factory/outputs.tf index 3ea8d3027..546b410b7 100644 --- a/modules/project-factory/outputs.tf +++ b/modules/project-factory/outputs.tf @@ -23,6 +23,11 @@ locals { } outputs_projects = { for k, v in local.projects_input : k => { + aspect_types = ( + v.factories_config.aspect_types == null + ? {} + : module.aspect-types[k].ids + ) automation = { bucket = try( module.automation-bucket[local._outputs_automation_buckets[k]].name, diff --git a/modules/project-factory/projects-defaults.tf b/modules/project-factory/projects-defaults.tf index 6b3842a84..4a7f5e6a1 100644 --- a/modules/project-factory/projects-defaults.tf +++ b/modules/project-factory/projects-defaults.tf @@ -51,7 +51,9 @@ locals { local.data_defaults.defaults.contacts ) factories_config = { + aspect_types = try(v.factories_config.aspect_types, null) custom_roles = try(v.factories_config.custom_roles, null) + data_catalog_taxonomy = try(v.factories_config.data_catalog_taxonomy, null) observability = try(v.factories_config.observability, null) org_policies = try(v.factories_config.org_policies, null) pam_entitlements = try(v.factories_config.pam_entitlements, null) diff --git a/modules/project-factory/projects.tf b/modules/project-factory/projects.tf index e334597a7..8ad850134 100644 --- a/modules/project-factory/projects.tf +++ b/modules/project-factory/projects.tf @@ -53,18 +53,18 @@ locals { ctx_project_numbers = merge(local.ctx.project_numbers, local.project_numbers) # cross-project tag contexts, keyed on project name ctx_tag_keys = merge(local.ctx.tag_keys, { - for k, v in merge([ + for k, v in merge({}, [ for pk, pv in local.projects_input : { for tk, tv in module.projects[pk].tag_keys : - "${pv.name}/${tk}" => tv.id + "${pk}/${tk}" => tv.id } ]...) : k => v }) ctx_tag_values = merge(local.ctx.tag_values, { - for k, v in merge([ + for k, v in merge({}, [ for pk, pv in local.projects_input : { for tk, tv in module.projects[pk].tag_values : - "${pv.name}/${tk}" => tv.id + "${pk}/${tk}" => tv.id } ]...) : k => v }) @@ -140,13 +140,16 @@ module "projects" { folder_ids = local.ctx_folder_ids }) default_service_account = try(each.value.default_service_account, "keep") + # Exclude factories that are either: + # a) Handled in parallel by calling specific modules (e.g., aspect_types, data_catalog_taxonomy) + # b) Handled in the projects-iam call to leverage expanded context (e.g., org_policies) factories_config = { for k, v in each.value.factories_config : k => try(pathexpand( var.factories_config.basepath == null || startswith(v, "/") || startswith(v, ".") ? v : "${var.factories_config.basepath}/${v}" ), null) - if k != "org_policies" + if !contains(["aspect_types", "data_catalog_taxonomy", "org_policies"], k) } kms_autokeys = try(each.value.kms.autokeys, {}) labels = merge( diff --git a/modules/project-factory/schemas/aspect-type.schema.json b/modules/project-factory/schemas/aspect-type.schema.json new file mode 100644 index 000000000..49a8f8e31 --- /dev/null +++ b/modules/project-factory/schemas/aspect-type.schema.json @@ -0,0 +1,125 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Dataplex Aspect Type", + "type": "object", + "additionalProperties": false, + "properties": { + "description": { + "type": "string" + }, + "display_name": { + "type": "string" + }, + "labels": { + "type": "object" + }, + "metadata_template": { + "type": "string" + }, + "iam": { + "$ref": "#/$defs/iam" + }, + "iam_bindings": { + "$ref": "#/$defs/iam_bindings" + }, + "iam_bindings_additive": { + "$ref": "#/$defs/iam_bindings_additive" + } + }, + "$defs": { + "iam": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^(?:roles/|\\$custom_roles:|organizations/[0-9]+/roles/|([a-z0-9.]+:)?projects/[a-z0-9-]+/roles/)": { + "type": "array", + "items": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:||\\$iam_principals:[a-z0-9_-]+)" + } + } + } + }, + "iam_bindings": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "members": { + "type": "array", + "items": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\\$iam_principals:[a-z0-9_-]+)" + } + }, + "role": { + "type": "string", + "pattern": "^(?:roles/|\\$custom_roles:|organizations/[0-9]+/roles/|([a-z0-9.]+:)?projects/[a-z0-9-]+/roles/)" + }, + "condition": { + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "title" + ], + "properties": { + "expression": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + } + } + } + } + } + } + }, + "iam_bindings_additive": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "member": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\\$iam_principals:[a-z0-9_-]+)" + }, + "role": { + "type": "string", + "pattern": "^(?:roles/|\\$custom_roles:|organizations/[0-9]+/roles/|([a-z0-9.]+:)?projects/[a-z0-9-]+/roles/)" + }, + "condition": { + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "title" + ], + "properties": { + "expression": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + } + } + } + } + } + } + } + } +} diff --git a/modules/project-factory/schemas/aspect-type.schema.md b/modules/project-factory/schemas/aspect-type.schema.md new file mode 100644 index 000000000..81576d96e --- /dev/null +++ b/modules/project-factory/schemas/aspect-type.schema.md @@ -0,0 +1,50 @@ +# Dataplex Aspect Type + + + +## Properties + +*additional properties: false* + +- **description**: *string* +- **display_name**: *string* +- **labels**: *object* +- **metadata_template**: *string* +- **iam**: *reference([iam](#refs-iam))* +- **iam_bindings**: *reference([iam_bindings](#refs-iam_bindings))* +- **iam_bindings_additive**: *reference([iam_bindings_additive](#refs-iam_bindings_additive))* + +## Definitions + +- **iam**: *object* +
*additional properties: false* + - **`^(?:roles/|\$custom_roles:|organizations/[0-9]+/roles/|([a-z0-9.]+:)?projects/[a-z0-9-]+/roles/)`**: *array* + - items: *string* +
*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:||\$iam_principals:[a-z0-9_-]+)* +- **iam_bindings**: *object* +
*additional properties: false* + - **`^[a-z0-9_-]+$`**: *object* +
*additional properties: false* + - **members**: *array* + - items: *string* +
*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\$iam_principals:[a-z0-9_-]+)* + - **role**: *string* +
*pattern: ^(?:roles/|\$custom_roles:|organizations/[0-9]+/roles/|([a-z0-9.]+:)?projects/[a-z0-9-]+/roles/)* + - **condition**: *object* +
*additional properties: false* + - ⁺**expression**: *string* + - ⁺**title**: *string* + - **description**: *string* +- **iam_bindings_additive**: *object* +
*additional properties: false* + - **`^[a-z0-9_-]+$`**: *object* +
*additional properties: false* + - **member**: *string* +
*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\$iam_principals:[a-z0-9_-]+)* + - **role**: *string* +
*pattern: ^(?:roles/|\$custom_roles:|organizations/[0-9]+/roles/|([a-z0-9.]+:)?projects/[a-z0-9-]+/roles/)* + - **condition**: *object* +
*additional properties: false* + - ⁺**expression**: *string* + - ⁺**title**: *string* + - **description**: *string* diff --git a/modules/project-factory/schemas/project.schema.json b/modules/project-factory/schemas/project.schema.json index d65fa4a58..94c224062 100644 --- a/modules/project-factory/schemas/project.schema.json +++ b/modules/project-factory/schemas/project.schema.json @@ -111,6 +111,9 @@ "type": "object", "additionalProperties": false, "properties": { + "display_name": { + "type": "string" + }, "description": { "type": "string" }, @@ -297,9 +300,15 @@ "type": "object", "additionalProperties": false, "properties": { + "aspect_types": { + "type": "string" + }, "custom_roles": { "type": "string" }, + "data_catalog_taxonomy": { + "type": "string" + }, "observability": { "type": "string" }, diff --git a/modules/project-factory/schemas/project.schema.md b/modules/project-factory/schemas/project.schema.md index a14fb866d..31998ca8a 100644 --- a/modules/project-factory/schemas/project.schema.md +++ b/modules/project-factory/schemas/project.schema.md @@ -37,6 +37,7 @@
*additional properties: false* - **`^[a-z0-9-]+$`**: *object*
*additional properties: false* + - **display_name**: *string* - **description**: *string* - **prefix**: *string* - **iam**: *reference([iam](#refs-iam))* @@ -95,7 +96,9 @@
*enum: ['PREVENT', 'DELETE', 'ABANDON']* - **factories_config**: *object*
*additional properties: false* + - **aspect_types**: *string* - **custom_roles**: *string* + - **data_catalog_taxonomy**: *string* - **observability**: *string* - **org_policies**: *string* - **quotas**: *string* diff --git a/modules/project-factory/schemas/taxonomy.schema.json b/modules/project-factory/schemas/taxonomy.schema.json new file mode 100644 index 000000000..23da50da8 --- /dev/null +++ b/modules/project-factory/schemas/taxonomy.schema.json @@ -0,0 +1,153 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": false, + "properties": { + "activated_policy_types": { + "type": "array", + "items": { + "type": "string" + } + }, + "description": { + "type": "string" + }, + "iam": { + "$ref": "#/$defs/iam" + }, + "iam_bindings": { + "$ref": "#/$defs/iam_bindings" + }, + "iam_bindings_additive": { + "$ref": "#/$defs/iam_bindings_additive" + }, + "tags": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^.+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "description": { + "type": "string" + }, + "iam": { + "$ref": "#/$defs/iam" + }, + "iam_bindings": { + "$ref": "#/$defs/iam_bindings" + }, + "iam_bindings_additive": { + "$ref": "#/$defs/iam_bindings_additive" + } + } + } + } + } + }, + "$defs": { + "iam": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^(?:roles/|\\$custom_roles:)": { + "type": "array", + "items": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\\$iam_principals:[a-z0-9_-]+)" + } + } + } + }, + "iam_bindings": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "object", + "additionalProperties": false, + "required": [ + "members", + "role" + ], + "properties": { + "members": { + "type": "array", + "items": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\\$iam_principals:[a-z0-9_-]+)" + } + }, + "role": { + "type": "string", + "pattern": "^(?:roles/|\\$custom_roles:)" + }, + "condition": { + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "title" + ], + "properties": { + "expression": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + } + } + } + } + } + } + }, + "iam_bindings_additive": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "object", + "additionalProperties": false, + "required": [ + "member", + "role" + ], + "properties": { + "member": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\\$iam_principals:[a-z0-9_-]+)" + }, + "role": { + "type": "string", + "pattern": "^(?:roles/|\\$custom_roles:)" + }, + "condition": { + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "title" + ], + "properties": { + "expression": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/modules/project-factory/schemas/taxonomy.schema.md b/modules/project-factory/schemas/taxonomy.schema.md new file mode 100644 index 000000000..da41b4360 --- /dev/null +++ b/modules/project-factory/schemas/taxonomy.schema.md @@ -0,0 +1,57 @@ +# None + + + +## Properties + +*additional properties: false* + +- **activated_policy_types**: *array* + - items: *string* +- **description**: *string* +- **iam**: *reference([iam](#refs-iam))* +- **iam_bindings**: *reference([iam_bindings](#refs-iam_bindings))* +- **iam_bindings_additive**: *reference([iam_bindings_additive](#refs-iam_bindings_additive))* +- **tags**: *object* +
*additional properties: false* + - **`^.+$`**: *object* +
*additional properties: false* + - **description**: *string* + - **iam**: *reference([iam](#refs-iam))* + - **iam_bindings**: *reference([iam_bindings](#refs-iam_bindings))* + - **iam_bindings_additive**: *reference([iam_bindings_additive](#refs-iam_bindings_additive))* + +## Definitions + +- **iam**: *object* +
*additional properties: false* + - **`^(?:roles/|\$custom_roles:)`**: *array* + - items: *string* +
*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\$iam_principals:[a-z0-9_-]+)* +- **iam_bindings**: *object* +
*additional properties: false* + - **`^[a-z0-9_-]+$`**: *object* +
*additional properties: false* + - ⁺**members**: *array* + - items: *string* +
*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\$iam_principals:[a-z0-9_-]+)* + - ⁺**role**: *string* +
*pattern: ^(?:roles/|\$custom_roles:)* + - **condition**: *object* +
*additional properties: false* + - ⁺**expression**: *string* + - ⁺**title**: *string* + - **description**: *string* +- **iam_bindings_additive**: *object* +
*additional properties: false* + - **`^[a-z0-9_-]+$`**: *object* +
*additional properties: false* + - ⁺**member**: *string* +
*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\$iam_principals:[a-z0-9_-]+)* + - ⁺**role**: *string* +
*pattern: ^(?:roles/|\$custom_roles:)* + - **condition**: *object* +
*additional properties: false* + - ⁺**expression**: *string* + - ⁺**title**: *string* + - **description**: *string* diff --git a/modules/project-factory/taxonomies.tf b/modules/project-factory/taxonomies.tf new file mode 100644 index 000000000..d8bb4997a --- /dev/null +++ b/modules/project-factory/taxonomies.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Taxonomy resources. + +module "taxonomies" { + source = "../data-catalog-policy-tag" + for_each = { + for k, v in local.projects_input : k => v + if try(v.factories_config.data_catalog_taxonomy, null) != null + } + project_id = module.projects[each.key].project_id + factories_config = { + taxonomy = lookup(each.value.factories_config, "data_catalog_taxonomy", null) == null ? null : try(pathexpand( + var.factories_config.basepath == null || startswith(each.value.factories_config.data_catalog_taxonomy, "/") || startswith(each.value.factories_config.data_catalog_taxonomy, ".") + ? each.value.factories_config.data_catalog_taxonomy : + "${var.factories_config.basepath}/${each.value.factories_config.data_catalog_taxonomy}" + ), null) + } + name = "taxonomy" + location = try(each.value.locations.storage, "europe-west1") + context = merge(local.ctx, { + iam_principals = merge( + local.ctx_iam_principals, + lookup(local.self_sas_iam_emails, each.key, {}), + local.projects_service_agents + ) + project_ids = merge( + local.ctx.project_ids, + { for k, v in module.projects : k => v.project_id } + ) + }) +} diff --git a/modules/project-factory/variables-projects.tf b/modules/project-factory/variables-projects.tf index 5e228fa5c..fc8aad4e2 100644 --- a/modules/project-factory/variables-projects.tf +++ b/modules/project-factory/variables-projects.tf @@ -240,7 +240,9 @@ variable "projects" { threat_detector_provider = optional(string) }), {}) factories_config = optional(object({ + aspect_types = optional(string) custom_roles = optional(string) + data_catalog_taxonomy = optional(string) observability = optional(string) org_policies = optional(string) pam_entitlements = optional(string) diff --git a/tests/fast/stages/s2_project_factory/data-platform.tfvars b/tests/fast/stages/s2_project_factory/data-platform.tfvars new file mode 100644 index 000000000..c22403d2e --- /dev/null +++ b/tests/fast/stages/s2_project_factory/data-platform.tfvars @@ -0,0 +1,25 @@ +automation = { + outputs_bucket = "fast2-prod-iac-core-outputs" +} +prefix = "testorg" +billing_account = { + id = "000000-111111-222222" +} +folder_ids = { + data-platform = "folders/1234567890" +} +project_ids = { + net-dev-0 = "projects/net-dev-0" +} +tag_values = { + "environment/development" = "tagValues/1234567890" + "environment/production" = "tagValues/2345678901" +} +factories_config = { + dataset = "datasets/data-platform" +} +organization = { + domain = "fast.example.com" + id = 123456789012 + customer_id = "C00000000" +} diff --git a/tests/fast/stages/s2_project_factory/data-platform.yaml b/tests/fast/stages/s2_project_factory/data-platform.yaml new file mode 100644 index 000000000..69d9411fe --- /dev/null +++ b/tests/fast/stages/s2_project_factory/data-platform.yaml @@ -0,0 +1,815 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + google_storage_bucket_object.version[0]: + bucket: fast2-prod-iac-core-outputs + cache_control: null + content_disposition: null + content_encoding: null + content_language: null + contexts: [] + customer_encryption: [] + deletion_policy: null + detect_md5hash: null + event_based_hold: null + force_empty_content_type: null + metadata: null + name: versions/2-project-factory-version.txt + retention: [] + source: fast_version.txt + temporary_hold: null + timeouts: null + module.factory.module.aspect-types["core-0"].google_dataplex_aspect_type.default["basic"]: + aspect_type_id: basic + data_classification: null + description: null + display_name: Basic template + effective_labels: + goog-terraform-provisioned: 'true' + labels: null + location: global + metadata_template: '{"name":"tf-basic-template","recordFields":[{"annotations":{"description":"Specifies + the source of data.","displayName":"Source"},"constraints":{"required":true},"index":1,"name":"source","type":"string"},{"annotations":{"description":"Specifies + the data owner.","displayName":"Owner"},"constraints":{},"index":2,"name":"owner","type":"string"}],"type":"record"}' + project: testorg-prod-dp-core-0 + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null + module.factory.module.automation-bucket["product-0/automation/tf-state"].google_storage_bucket.bucket[0]: + autoclass: [] + cors: [] + custom_placement_config: [] + default_event_based_hold: null + effective_labels: + goog-terraform-provisioned: 'true' + enable_object_retention: null + encryption: [] + force_destroy: false + hierarchical_namespace: [] + ip_filter: [] + labels: null + lifecycle_rule: [] + location: EUROPE-WEST1 + logging: [] + name: testorg-product-0-tf-state + project: testorg-prod-dp-dd-0 + requester_pays: null + retention_policy: [] + storage_class: STANDARD + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null + uniform_bucket_level_access: true + versioning: + - enabled: true + ? module.factory.module.automation-bucket["product-0/automation/tf-state"].google_storage_bucket_iam_binding.authoritative["roles/storage.admin"] + : bucket: testorg-product-0-tf-state + condition: [] + members: + - serviceAccount:product-0-iac-rw@testorg-prod-dp-dd-0.iam.gserviceaccount.com + role: roles/storage.admin + timeouts: null + ? module.factory.module.automation-bucket["product-0/automation/tf-state"].google_storage_bucket_iam_binding.authoritative["roles/storage.objectViewer"] + : bucket: testorg-product-0-tf-state + condition: [] + members: + - group:dp-product-a-0@example.com + - serviceAccount:product-0-iac-ro@testorg-prod-dp-dd-0.iam.gserviceaccount.com + role: roles/storage.objectViewer + timeouts: null + module.factory.module.automation-bucket["shared-0/automation/tf-state"].google_storage_bucket.bucket[0]: + autoclass: [] + cors: [] + custom_placement_config: [] + default_event_based_hold: null + effective_labels: + goog-terraform-provisioned: 'true' + enable_object_retention: null + encryption: [] + force_destroy: false + hierarchical_namespace: [] + ip_filter: [] + labels: null + lifecycle_rule: [] + location: EUROPE-WEST1 + logging: [] + name: testorg-shared-0-tf-state + project: testorg-prod-dp-core-0 + requester_pays: null + retention_policy: [] + storage_class: STANDARD + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null + uniform_bucket_level_access: true + versioning: + - enabled: true + ? module.factory.module.automation-bucket["shared-0/automation/tf-state"].google_storage_bucket_iam_binding.authoritative["roles/storage.admin"] + : bucket: testorg-shared-0-tf-state + condition: [] + members: + - serviceAccount:shared-0-iac-rw@testorg-prod-dp-core-0.iam.gserviceaccount.com + role: roles/storage.admin + timeouts: null + ? module.factory.module.automation-bucket["shared-0/automation/tf-state"].google_storage_bucket_iam_binding.authoritative["roles/storage.objectViewer"] + : bucket: testorg-shared-0-tf-state + condition: [] + members: + - group:dp-product-a-0@example.com + - serviceAccount:shared-0-iac-ro@testorg-prod-dp-core-0.iam.gserviceaccount.com + role: roles/storage.objectViewer + timeouts: null + module.factory.module.automation-service-accounts["product-0/automation/iac-ro"].google_service_account.service_account[0]: + account_id: product-0-iac-ro + create_ignore_already_exists: null + description: null + disabled: false + display_name: Product 0/0 (ro) + email: product-0-iac-ro@testorg-prod-dp-dd-0.iam.gserviceaccount.com + member: serviceAccount:product-0-iac-ro@testorg-prod-dp-dd-0.iam.gserviceaccount.com + project: testorg-prod-dp-dd-0 + timeouts: null + ? module.factory.module.automation-service-accounts["product-0/automation/iac-ro"].google_service_account_iam_binding.authoritative["roles/iam.serviceAccountTokenCreator"] + : condition: [] + members: + - group:dp-product-a-0@example.com + role: roles/iam.serviceAccountTokenCreator + module.factory.module.automation-service-accounts["product-0/automation/iac-rw"].google_service_account.service_account[0]: + account_id: product-0-iac-rw + create_ignore_already_exists: null + description: null + disabled: false + display_name: Product 0/0 (rw) + email: product-0-iac-rw@testorg-prod-dp-dd-0.iam.gserviceaccount.com + member: serviceAccount:product-0-iac-rw@testorg-prod-dp-dd-0.iam.gserviceaccount.com + project: testorg-prod-dp-dd-0 + timeouts: null + ? module.factory.module.automation-service-accounts["product-0/automation/iac-rw"].google_service_account_iam_binding.authoritative["roles/iam.serviceAccountTokenCreator"] + : condition: [] + members: + - group:dp-product-a-0@example.com + role: roles/iam.serviceAccountTokenCreator + module.factory.module.automation-service-accounts["shared-0/automation/iac-ro"].google_service_account.service_account[0]: + account_id: shared-0-iac-ro + create_ignore_already_exists: null + description: null + disabled: false + display_name: Domain 0 (ro) + email: shared-0-iac-ro@testorg-prod-dp-core-0.iam.gserviceaccount.com + member: serviceAccount:shared-0-iac-ro@testorg-prod-dp-core-0.iam.gserviceaccount.com + project: testorg-prod-dp-core-0 + timeouts: null + ? module.factory.module.automation-service-accounts["shared-0/automation/iac-ro"].google_service_account_iam_binding.authoritative["roles/iam.serviceAccountTokenCreator"] + : condition: [] + members: + - group:dp-product-a-0@example.com + role: roles/iam.serviceAccountTokenCreator + module.factory.module.automation-service-accounts["shared-0/automation/iac-rw"].google_service_account.service_account[0]: + account_id: shared-0-iac-rw + create_ignore_already_exists: null + description: null + disabled: false + display_name: Domain 0 (rw) + email: shared-0-iac-rw@testorg-prod-dp-core-0.iam.gserviceaccount.com + member: serviceAccount:shared-0-iac-rw@testorg-prod-dp-core-0.iam.gserviceaccount.com + project: testorg-prod-dp-core-0 + timeouts: null + ? module.factory.module.automation-service-accounts["shared-0/automation/iac-rw"].google_service_account_iam_binding.authoritative["roles/iam.serviceAccountTokenCreator"] + : condition: [] + members: + - group:dp-product-a-0@example.com + role: roles/iam.serviceAccountTokenCreator + module.factory.module.bigquery-datasets["product-0/private"].google_bigquery_dataset.default: + dataset_id: private + default_encryption_configuration: [] + default_partition_expiration_ms: null + default_table_expiration_ms: null + delete_contents_on_destroy: false + description: Terraform managed. + effective_labels: + goog-terraform-provisioned: 'true' + external_catalog_dataset_options: [] + external_dataset_reference: [] + friendly_name: null + labels: null + location: europe-west1 + max_time_travel_hours: '168' + project: testorg-prod-dp-dd0-p0 + resource_tags: null + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null + module.factory.module.bigquery-datasets["product-0/public"].google_bigquery_dataset.default: + dataset_id: public + default_encryption_configuration: [] + default_partition_expiration_ms: null + default_table_expiration_ms: null + delete_contents_on_destroy: false + description: Terraform managed. + effective_labels: + goog-terraform-provisioned: 'true' + external_catalog_dataset_options: [] + external_dataset_reference: [] + friendly_name: null + labels: null + location: europe-west1 + max_time_travel_hours: '168' + project: testorg-prod-dp-dd0-p0 + resource_tags: null + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null + ? module.factory.module.bigquery-datasets["product-0/public"].google_bigquery_dataset_iam_binding.authoritative["roles/bigquery.dataViewer"] + : condition: [] + dataset_id: public + members: + - group:data-consumer-bi@example.com + project: testorg-prod-dp-dd0-p0 + role: roles/bigquery.dataViewer + module.factory.module.bigquery-datasets["product-0/public"].google_tags_location_tag_binding.binding["exposure"]: + location: europe-west1 + timeouts: null + module.factory.module.buckets["product-0/private"].google_storage_bucket.bucket[0]: + autoclass: [] + cors: [] + custom_placement_config: [] + default_event_based_hold: null + effective_labels: + goog-terraform-provisioned: 'true' + enable_object_retention: null + encryption: [] + force_destroy: false + hierarchical_namespace: [] + ip_filter: [] + labels: null + lifecycle_rule: [] + location: EUROPE-WEST1 + logging: [] + name: testorg-prod-dp-dd0-p0-private + project: testorg-prod-dp-dd0-p0 + requester_pays: null + retention_policy: [] + storage_class: STANDARD + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null + uniform_bucket_level_access: true + versioning: + - enabled: false + module.factory.module.buckets["product-0/public"].google_storage_bucket.bucket[0]: + autoclass: [] + cors: [] + custom_placement_config: [] + default_event_based_hold: null + effective_labels: + goog-terraform-provisioned: 'true' + enable_object_retention: null + encryption: [] + force_destroy: false + hierarchical_namespace: [] + ip_filter: [] + labels: null + lifecycle_rule: [] + location: EUROPE-WEST1 + logging: [] + name: testorg-prod-dp-dd0-p0-public + project: testorg-prod-dp-dd0-p0 + requester_pays: null + retention_policy: [] + storage_class: STANDARD + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null + uniform_bucket_level_access: true + versioning: + - enabled: false + ? module.factory.module.buckets["product-0/public"].google_storage_bucket_iam_binding.authoritative["roles/storage.objectViewer"] + : bucket: testorg-prod-dp-dd0-p0-public + condition: [] + members: + - group:data-consumer-bi@example.com + role: roles/storage.objectViewer + timeouts: null + module.factory.module.buckets["product-0/public"].google_tags_location_tag_binding.binding["exposure"]: + location: europe-west1 + parent: //storage.googleapis.com/projects/_/buckets/testorg-prod-dp-dd0-p0-public + timeouts: null + module.factory.module.folder-1["domain-0"].google_folder.folder[0]: + deletion_protection: false + display_name: Data Domain 0 + parent: folders/1234567890 + tags: null + timeouts: null + module.factory.module.projects-iam["product-0"].google_project_iam_binding.authoritative["roles/bigquery.dataEditor"]: + condition: [] + members: + - serviceAccount:processing@testorg-prod-dp-dd0-p0.iam.gserviceaccount.com + project: testorg-prod-dp-dd0-p0 + role: roles/bigquery.dataEditor + module.factory.module.projects-iam["product-0"].google_project_iam_binding.authoritative["roles/bigquery.jobUser"]: + condition: [] + members: + - serviceAccount:processing@testorg-prod-dp-dd0-p0.iam.gserviceaccount.com + project: testorg-prod-dp-dd0-p0 + role: roles/bigquery.jobUser + module.factory.module.projects-iam["product-0"].google_project_iam_binding.authoritative["roles/dataflow.admin"]: + condition: [] + members: + - serviceAccount:processing@testorg-prod-dp-dd0-p0.iam.gserviceaccount.com + project: testorg-prod-dp-dd0-p0 + role: roles/dataflow.admin + module.factory.module.projects-iam["product-0"].google_project_iam_binding.authoritative["roles/dataproc.editor"]: + condition: [] + members: + - serviceAccount:processing@testorg-prod-dp-dd0-p0.iam.gserviceaccount.com + project: testorg-prod-dp-dd0-p0 + role: roles/dataproc.editor + module.factory.module.projects-iam["product-0"].google_project_iam_binding.authoritative["roles/dataproc.worker"]: + condition: [] + members: + - serviceAccount:processing@testorg-prod-dp-dd0-p0.iam.gserviceaccount.com + project: testorg-prod-dp-dd0-p0 + role: roles/dataproc.worker + module.factory.module.projects-iam["product-0"].google_project_iam_binding.authoritative["roles/iam.serviceAccountUser"]: + condition: [] + members: + - serviceAccount:processing@testorg-prod-dp-dd0-p0.iam.gserviceaccount.com + project: testorg-prod-dp-dd0-p0 + role: roles/iam.serviceAccountUser + module.factory.module.projects-iam["product-0"].google_project_iam_binding.authoritative["roles/owner"]: + condition: [] + members: + - serviceAccount:product-0-iac-rw@testorg-prod-dp-dd-0.iam.gserviceaccount.com + project: testorg-prod-dp-dd0-p0 + role: roles/owner + module.factory.module.projects-iam["product-0"].google_project_iam_binding.authoritative["roles/storage.bucketViewer"]: + condition: [] + members: + - serviceAccount:processing@testorg-prod-dp-dd0-p0.iam.gserviceaccount.com + project: testorg-prod-dp-dd0-p0 + role: roles/storage.bucketViewer + module.factory.module.projects-iam["product-0"].google_project_iam_binding.authoritative["roles/storage.objectAdmin"]: + condition: [] + members: + - serviceAccount:processing@testorg-prod-dp-dd0-p0.iam.gserviceaccount.com + project: testorg-prod-dp-dd0-p0 + role: roles/storage.objectAdmin + module.factory.module.projects-iam["product-0"].google_project_iam_binding.authoritative["roles/viewer"]: + condition: [] + members: + - serviceAccount:product-0-iac-ro@testorg-prod-dp-dd-0.iam.gserviceaccount.com + project: testorg-prod-dp-dd0-p0 + role: roles/viewer + module.factory.module.projects-iam["shared-0"].google_project_iam_binding.authoritative["roles/composer.worker"]: + condition: [] + members: + - serviceAccount:composer@testorg-prod-dp-dd-0.iam.gserviceaccount.com + project: testorg-prod-dp-dd-0 + role: roles/composer.worker + module.factory.module.projects-iam["shared-0"].google_project_iam_binding.authoritative["roles/owner"]: + condition: [] + members: + - serviceAccount:shared-0-iac-rw@testorg-prod-dp-core-0.iam.gserviceaccount.com + project: testorg-prod-dp-dd-0 + role: roles/owner + module.factory.module.projects-iam["shared-0"].google_project_iam_binding.authoritative["roles/viewer"]: + condition: [] + members: + - serviceAccount:shared-0-iac-ro@testorg-prod-dp-core-0.iam.gserviceaccount.com + project: testorg-prod-dp-dd-0 + role: roles/viewer + module.factory.module.projects["core-0"].data.google_bigquery_default_service_account.bq_sa[0]: + project: testorg-prod-dp-core-0 + module.factory.module.projects["core-0"].data.google_logging_project_settings.logging_sa[0]: + project: testorg-prod-dp-core-0 + module.factory.module.projects["core-0"].data.google_storage_project_service_account.gcs_sa[0]: + project: testorg-prod-dp-core-0 + user_project: null + module.factory.module.projects["core-0"].google_project.project[0]: + auto_create_network: false + billing_account: 000000-111111-222222 + deletion_policy: DELETE + effective_labels: + goog-terraform-provisioned: 'true' + folder_id: '1234567890' + labels: null + name: testorg-prod-dp-core-0 + org_id: null + project_id: testorg-prod-dp-core-0 + tags: null + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null + module.factory.module.projects["core-0"].google_project_iam_member.service_agents["monitoring-notification"]: + condition: [] + project: testorg-prod-dp-core-0 + role: roles/monitoring.notificationServiceAgent + module.factory.module.projects["core-0"].google_project_service.project_services["bigquery.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-core-0 + service: bigquery.googleapis.com + timeouts: null + module.factory.module.projects["core-0"].google_project_service.project_services["datacatalog.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-core-0 + service: datacatalog.googleapis.com + timeouts: null + module.factory.module.projects["core-0"].google_project_service.project_services["logging.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-core-0 + service: logging.googleapis.com + timeouts: null + module.factory.module.projects["core-0"].google_project_service.project_services["monitoring.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-core-0 + service: monitoring.googleapis.com + timeouts: null + module.factory.module.projects["core-0"].google_project_service.project_services["storage.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-core-0 + service: storage.googleapis.com + timeouts: null + module.factory.module.projects["core-0"].google_project_service_identity.default["monitoring.googleapis.com"]: + project: testorg-prod-dp-core-0 + service: monitoring.googleapis.com + timeouts: null + module.factory.module.projects["core-0"].google_tags_tag_key.default["exposure"]: + allowed_values_regex: null + description: Data exposure controls. + parent: projects/testorg-prod-dp-core-0 + purpose: null + purpose_data: null + short_name: exposure + timeouts: null + module.factory.module.projects["core-0"].google_tags_tag_value.default["exposure/public"]: + description: Data exposure allowed. + short_name: public + timeouts: null + module.factory.module.projects["product-0"].data.google_bigquery_default_service_account.bq_sa[0]: + project: testorg-prod-dp-dd0-p0 + module.factory.module.projects["product-0"].data.google_logging_project_settings.logging_sa[0]: + project: testorg-prod-dp-dd0-p0 + module.factory.module.projects["product-0"].data.google_storage_project_service_account.gcs_sa[0]: + project: testorg-prod-dp-dd0-p0 + user_project: null + module.factory.module.projects["product-0"].google_project.project[0]: + auto_create_network: false + billing_account: 000000-111111-222222 + deletion_policy: DELETE + effective_labels: + goog-terraform-provisioned: 'true' + labels: null + name: testorg-prod-dp-dd0-p0 + org_id: null + project_id: testorg-prod-dp-dd0-p0 + tags: null + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null + module.factory.module.projects["product-0"].google_project_iam_member.service_agents["cloudaicompanion"]: + condition: [] + project: testorg-prod-dp-dd0-p0 + role: roles/cloudaicompanion.serviceAgent + module.factory.module.projects["product-0"].google_project_iam_member.service_agents["cloudcomposer-accounts"]: + condition: [] + project: testorg-prod-dp-dd0-p0 + role: roles/composer.serviceAgent + module.factory.module.projects["product-0"].google_project_iam_member.service_agents["dataplex"]: + condition: [] + project: testorg-prod-dp-dd0-p0 + role: roles/dataplex.serviceAgent + module.factory.module.projects["product-0"].google_project_iam_member.service_agents["monitoring-notification"]: + condition: [] + project: testorg-prod-dp-dd0-p0 + role: roles/monitoring.notificationServiceAgent + module.factory.module.projects["product-0"].google_project_service.project_services["bigquery.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd0-p0 + service: bigquery.googleapis.com + timeouts: null + module.factory.module.projects["product-0"].google_project_service.project_services["cloudaicompanion.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd0-p0 + service: cloudaicompanion.googleapis.com + timeouts: null + module.factory.module.projects["product-0"].google_project_service.project_services["cloudresourcemanager.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd0-p0 + service: cloudresourcemanager.googleapis.com + timeouts: null + module.factory.module.projects["product-0"].google_project_service.project_services["composer.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd0-p0 + service: composer.googleapis.com + timeouts: null + module.factory.module.projects["product-0"].google_project_service.project_services["datacatalog.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd0-p0 + service: datacatalog.googleapis.com + timeouts: null + module.factory.module.projects["product-0"].google_project_service.project_services["datalineage.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd0-p0 + service: datalineage.googleapis.com + timeouts: null + module.factory.module.projects["product-0"].google_project_service.project_services["dataplex.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd0-p0 + service: dataplex.googleapis.com + timeouts: null + module.factory.module.projects["product-0"].google_project_service.project_services["logging.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd0-p0 + service: logging.googleapis.com + timeouts: null + module.factory.module.projects["product-0"].google_project_service.project_services["monitoring.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd0-p0 + service: monitoring.googleapis.com + timeouts: null + module.factory.module.projects["product-0"].google_project_service.project_services["storage.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd0-p0 + service: storage.googleapis.com + timeouts: null + module.factory.module.projects["product-0"].google_project_service_identity.default["cloudaicompanion.googleapis.com"]: + project: testorg-prod-dp-dd0-p0 + service: cloudaicompanion.googleapis.com + timeouts: null + module.factory.module.projects["product-0"].google_project_service_identity.default["composer.googleapis.com"]: + project: testorg-prod-dp-dd0-p0 + service: composer.googleapis.com + timeouts: null + module.factory.module.projects["product-0"].google_project_service_identity.default["dataplex.googleapis.com"]: + project: testorg-prod-dp-dd0-p0 + service: dataplex.googleapis.com + timeouts: null + module.factory.module.projects["product-0"].google_project_service_identity.default["monitoring.googleapis.com"]: + project: testorg-prod-dp-dd0-p0 + service: monitoring.googleapis.com + timeouts: null + module.factory.module.projects["shared-0"].data.google_bigquery_default_service_account.bq_sa[0]: + project: testorg-prod-dp-dd-0 + module.factory.module.projects["shared-0"].data.google_logging_project_settings.logging_sa[0]: + project: testorg-prod-dp-dd-0 + module.factory.module.projects["shared-0"].data.google_storage_project_service_account.gcs_sa[0]: + project: testorg-prod-dp-dd-0 + user_project: null + module.factory.module.projects["shared-0"].google_project.project[0]: + auto_create_network: false + billing_account: 000000-111111-222222 + deletion_policy: DELETE + effective_labels: + goog-terraform-provisioned: 'true' + labels: null + name: testorg-prod-dp-dd-0 + org_id: null + project_id: testorg-prod-dp-dd-0 + tags: null + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null + module.factory.module.projects["shared-0"].google_project_iam_member.service_agents["cloudcomposer-accounts"]: + condition: [] + project: testorg-prod-dp-dd-0 + role: roles/composer.serviceAgent + module.factory.module.projects["shared-0"].google_project_iam_member.service_agents["dataplex"]: + condition: [] + project: testorg-prod-dp-dd-0 + role: roles/dataplex.serviceAgent + module.factory.module.projects["shared-0"].google_project_iam_member.service_agents["monitoring-notification"]: + condition: [] + project: testorg-prod-dp-dd-0 + role: roles/monitoring.notificationServiceAgent + module.factory.module.projects["shared-0"].google_project_service.project_services["bigquery.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd-0 + service: bigquery.googleapis.com + timeouts: null + module.factory.module.projects["shared-0"].google_project_service.project_services["composer.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd-0 + service: composer.googleapis.com + timeouts: null + module.factory.module.projects["shared-0"].google_project_service.project_services["datacatalog.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd-0 + service: datacatalog.googleapis.com + timeouts: null + module.factory.module.projects["shared-0"].google_project_service.project_services["datalineage.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd-0 + service: datalineage.googleapis.com + timeouts: null + module.factory.module.projects["shared-0"].google_project_service.project_services["dataplex.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd-0 + service: dataplex.googleapis.com + timeouts: null + module.factory.module.projects["shared-0"].google_project_service.project_services["logging.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd-0 + service: logging.googleapis.com + timeouts: null + module.factory.module.projects["shared-0"].google_project_service.project_services["monitoring.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd-0 + service: monitoring.googleapis.com + timeouts: null + module.factory.module.projects["shared-0"].google_project_service.project_services["storage.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: testorg-prod-dp-dd-0 + service: storage.googleapis.com + timeouts: null + module.factory.module.projects["shared-0"].google_project_service_identity.default["composer.googleapis.com"]: + project: testorg-prod-dp-dd-0 + service: composer.googleapis.com + timeouts: null + module.factory.module.projects["shared-0"].google_project_service_identity.default["dataplex.googleapis.com"]: + project: testorg-prod-dp-dd-0 + service: dataplex.googleapis.com + timeouts: null + module.factory.module.projects["shared-0"].google_project_service_identity.default["monitoring.googleapis.com"]: + project: testorg-prod-dp-dd-0 + service: monitoring.googleapis.com + timeouts: null + ? module.factory.module.service-accounts-iam["product-0/processing"].google_service_account_iam_binding.authoritative["roles/iam.serviceAccountUser"] + : condition: [] + members: + - serviceAccount:product-0-iac-rw@testorg-prod-dp-dd-0.iam.gserviceaccount.com + role: roles/iam.serviceAccountUser + module.factory.module.service-accounts["product-0/processing"].google_service_account.service_account[0]: + account_id: processing + create_ignore_already_exists: null + description: null + disabled: false + display_name: Processing service account. + email: processing@testorg-prod-dp-dd0-p0.iam.gserviceaccount.com + member: serviceAccount:processing@testorg-prod-dp-dd0-p0.iam.gserviceaccount.com + project: testorg-prod-dp-dd0-p0 + timeouts: null + module.factory.module.service-accounts["shared-0/composer"].google_service_account.service_account[0]: + account_id: composer + create_ignore_already_exists: null + description: null + disabled: false + display_name: Terraform-managed. + email: composer@testorg-prod-dp-dd-0.iam.gserviceaccount.com + member: serviceAccount:composer@testorg-prod-dp-dd-0.iam.gserviceaccount.com + project: testorg-prod-dp-dd-0 + timeouts: null + module.factory.module.taxonomies["core-0"].google_data_catalog_policy_tag.default["high"]: + description: High sensitivity data. + display_name: high + parent_policy_tag: null + timeouts: null + module.factory.module.taxonomies["core-0"].google_data_catalog_policy_tag.default["low"]: + description: Low sensitivity data. + display_name: low + parent_policy_tag: null + timeouts: null + module.factory.module.taxonomies["core-0"].google_data_catalog_policy_tag.default["medium"]: + description: Medium sensitivity data. + display_name: medium + parent_policy_tag: null + timeouts: null + module.factory.module.taxonomies["core-0"].google_data_catalog_taxonomy.default: + activated_policy_types: + - FINE_GRAINED_ACCESS_CONTROL + description: Taxonomy for data platform. + display_name: taxonomy + project: testorg-prod-dp-core-0 + region: europe-west1 + timeouts: null + module.factory.terraform_data.defaults_preconditions: + input: null + output: null + triggers_replace: null + module.factory.terraform_data.project_preconditions: + input: null + output: null + triggers_replace: null + module.vpc-factory.module.vpcs["domain-0"].google_compute_network.network[0]: + auto_create_subnetworks: false + delete_bgp_always_compare_med: false + delete_default_routes_on_create: true + description: Terraform managed + enable_ula_internal_ipv6: null + mtu: 1500 + name: domain-0 + network_firewall_policy_enforcement_order: AFTER_CLASSIC_FIREWALL + network_profile: null + params: [] + project: testorg-prod-dp-dd-0 + routing_mode: GLOBAL + timeouts: null + module.vpc-factory.module.vpcs["domain-0"].google_compute_route.gateway["directpath-googleapis"]: + description: Terraform-managed. + dest_range: 34.126.0.0/18 + name: domain-0-directpath-googleapis + network: domain-0 + next_hop_gateway: default-internet-gateway + next_hop_ilb: null + next_hop_instance: null + next_hop_vpn_tunnel: null + params: [] + priority: 1000 + project: testorg-prod-dp-dd-0 + tags: null + timeouts: null + module.vpc-factory.module.vpcs["domain-0"].google_compute_route.gateway["private-googleapis"]: + description: Terraform-managed. + dest_range: 199.36.153.8/30 + name: domain-0-private-googleapis + network: domain-0 + next_hop_gateway: default-internet-gateway + next_hop_ilb: null + next_hop_instance: null + next_hop_vpn_tunnel: null + params: [] + priority: 1000 + project: testorg-prod-dp-dd-0 + tags: null + timeouts: null + module.vpc-factory.module.vpcs["domain-0"].google_compute_route.gateway["restricted-googleapis"]: + description: Terraform-managed. + dest_range: 199.36.153.4/30 + name: domain-0-restricted-googleapis + network: domain-0 + next_hop_gateway: default-internet-gateway + next_hop_ilb: null + next_hop_instance: null + next_hop_vpn_tunnel: null + params: [] + priority: 1000 + project: testorg-prod-dp-dd-0 + tags: null + timeouts: null + +counts: + google_bigquery_dataset: 2 + google_bigquery_dataset_iam_binding: 1 + google_bigquery_default_service_account: 3 + google_compute_network: 1 + google_compute_route: 3 + google_data_catalog_policy_tag: 3 + google_data_catalog_taxonomy: 1 + google_dataplex_aspect_type: 1 + google_folder: 1 + google_logging_project_settings: 3 + google_project: 3 + google_project_iam_binding: 13 + google_project_iam_member: 8 + google_project_service: 23 + google_project_service_identity: 8 + google_service_account: 6 + google_service_account_iam_binding: 5 + google_storage_bucket: 4 + google_storage_bucket_iam_binding: 5 + google_storage_bucket_object: 1 + google_storage_project_service_account: 3 + google_tags_location_tag_binding: 2 + google_tags_tag_key: 1 + google_tags_tag_value: 1 + modules: 24 + resources: 104 + terraform_data: 2 + +outputs: + projects: __missing__ + vpcs: __missing__ diff --git a/tests/fast/stages/s2_project_factory/tftest.yaml b/tests/fast/stages/s2_project_factory/tftest.yaml index 2e7b27a19..4645a7006 100644 --- a/tests/fast/stages/s2_project_factory/tftest.yaml +++ b/tests/fast/stages/s2_project_factory/tftest.yaml @@ -16,3 +16,4 @@ module: fast/stages/2-project-factory tests: simple: + data-platform: diff --git a/tests/modules/project_factory/examples/example.yaml b/tests/modules/project_factory/examples/example.yaml index 6ce1ed424..22595f6d4 100644 --- a/tests/modules/project_factory/examples/example.yaml +++ b/tests/modules/project_factory/examples/example.yaml @@ -963,6 +963,14 @@ values: member: serviceAccount:app-0-be@test-pf-dev-tb-app0-1.iam.gserviceaccount.com project: test-pf-dev-tb-app0-1 timeouts: null + module.project-factory.module.taxonomies["dev-tb-app0-0"].google_data_catalog_taxonomy.default: + activated_policy_types: + - FINE_GRAINED_ACCESS_CONTROL + description: Taxonomy - Terraform managed + display_name: taxonomy + project: test-pf-dev-tb-app0-0 + region: europe-west1 + timeouts: null module.project-factory.terraform_data.defaults_preconditions: input: null output: null @@ -979,6 +987,7 @@ counts: google_cloud_asset_folder_feed: 1 google_compute_shared_vpc_host_project: 1 google_compute_shared_vpc_service_project: 1 + google_data_catalog_taxonomy: 1 google_essential_contacts_contact: 4 google_folder: 8 google_folder_iam_audit_config: 1 @@ -1012,8 +1021,8 @@ counts: google_tags_tag_key: 1 google_tags_tag_value: 2 google_tags_tag_value_iam_binding: 1 - modules: 37 - resources: 119 + modules: 38 + resources: 120 terraform_data: 2 outputs: {} diff --git a/tools/duplicate-diff.py b/tools/duplicate-diff.py index 01f4811e0..b9213aa6b 100755 --- a/tools/duplicate-diff.py +++ b/tools/duplicate-diff.py @@ -32,6 +32,15 @@ duplicates = [ "fast/stages/1-vpcsc/schemas/access-level.schema.json", "modules/vpc-sc/schemas/access-level.schema.json", ], + [ + "modules/dataplex-aspect-types/schemas/aspect-type.schema.json", + "modules/project-factory/schemas/aspect-type.schema.json", + "fast/stages/2-project-factory/schemas/aspect-type.schema.json", + ], + [ + "modules/data-catalog-policy-tag/schemas/policy-tag.schema.json", + "modules/project-factory/schemas/taxonomy.schema.json", + ], [ "fast/stages/2-project-factory/schemas/budget.schema.json", "fast/stages/0-org-setup/schemas/budget.schema.json", @@ -113,6 +122,7 @@ duplicates = [ "fast/stages/0-org-setup/schemas/tags.schema.json", "modules/project/schemas/tags.schema.json", "modules/organization/schemas/tags.schema.json", + "fast/stages/2-project-factory/schemas/tags.schema.json", ], [ "modules/cloud-function-v1/bundle.tf",