From c8a9cd3edb740575371b9c31d6a3591445bc1281 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 4 Apr 2025 09:06:26 +0200 Subject: [PATCH 01/10] Update VPC-SC README (#3006) Fixes #2983 --- modules/vpc-sc/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/vpc-sc/README.md b/modules/vpc-sc/README.md index d8f37e68b..f8251c851 100644 --- a/modules/vpc-sc/README.md +++ b/modules/vpc-sc/README.md @@ -120,9 +120,7 @@ The regular perimeters variable exposes all the complexity of the underlying res If you need to refer to access levels created by the same module in regular service perimeters, you can either use the module's outputs in the provided variables, or the key used to identify the relevant access level. The example below shows how to do this in practice. -/* -Resources for both perimeters have a `lifecycle` block that ignores changes to `spec` and `status` resources (projects), to allow using the additive resource `google_access_context_manager_service_perimeter_resource` at project creation. If this is not needed, the `lifecycle` blocks can be safely commented in the code. -*/ +If you are managing perimeter membership outside of this module via `google_access_context_manager_service_perimeter_resource`, for example at project creation in a project factory, you might want to uncomment the lifecycle blocks that are defined but currently unused in `service-perimeters-regular.tf` and `service-perimeters-bridge.tf`. #### Bridge type From e04079e3341fdcbdb12fd444b18c80231ea94b2a Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Fri, 4 Apr 2025 14:31:19 +0200 Subject: [PATCH 02/10] Add support to attach tags to service accounts (#3008) * Remove service account key upload. Add create_ignore_already_exists * Add tag bindings to service accounts * Add description to create_ignore_already_exists * Remove broken links --- blueprints/README.md | 2 +- blueprints/cloud-operations/README.md | 6 -- .../onprem-sa-key-management/OWNERS | 1 - .../onprem-sa-key-management/README.md | 91 ------------------- .../backend.tf.sample | 23 ----- .../cloud-shell-readme.txt | 46 ---------- .../onprem-sa-key-management/main.tf | 37 -------- .../onprem-sa-key-management/outputs.tf | 20 ---- .../public-keys/data-uploader/public_key.pem | 17 ---- .../prisma-security/public_key.pem | 17 ---- .../onprem-sa-key-management/variables.tf | 60 ------------ modules/iam-service-account/README.md | 71 ++++++++++----- modules/iam-service-account/main.tf | 44 +++------ modules/iam-service-account/outputs.tf | 7 +- modules/iam-service-account/variables.tf | 26 +++++- .../iam_service_account/examples/tags.yaml | 36 ++++++++ 16 files changed, 121 insertions(+), 383 deletions(-) delete mode 100644 blueprints/cloud-operations/onprem-sa-key-management/OWNERS delete mode 100644 blueprints/cloud-operations/onprem-sa-key-management/README.md delete mode 100644 blueprints/cloud-operations/onprem-sa-key-management/backend.tf.sample delete mode 100644 blueprints/cloud-operations/onprem-sa-key-management/cloud-shell-readme.txt delete mode 100644 blueprints/cloud-operations/onprem-sa-key-management/main.tf delete mode 100644 blueprints/cloud-operations/onprem-sa-key-management/outputs.tf delete mode 100644 blueprints/cloud-operations/onprem-sa-key-management/public-keys/data-uploader/public_key.pem delete mode 100644 blueprints/cloud-operations/onprem-sa-key-management/public-keys/prisma-security/public_key.pem delete mode 100644 blueprints/cloud-operations/onprem-sa-key-management/variables.tf create mode 100644 tests/modules/iam_service_account/examples/tags.yaml diff --git a/blueprints/README.md b/blueprints/README.md index b99441868..37a6ae979 100644 --- a/blueprints/README.md +++ b/blueprints/README.md @@ -5,7 +5,7 @@ This section provides **[networking blueprints](./networking/)** that implement Currently available blueprints: - **apigee** - [Apigee X foundations](./apigee/apigee-x-foundations/). [Apigee Hybrid on GKE](./apigee/hybrid-gke/), [Apigee X analytics in BigQuery](./apigee/bigquery-analytics), [Apigee network patterns](./apigee/network-patterns/) -- **cloud operations** - [Active Directory Federation Services](./cloud-operations/adfs), [Cloud Asset Inventory feeds for resource change tracking and remediation](./cloud-operations/asset-inventory-feed-remediation), [Fine-grained Cloud DNS IAM via Service Directory](./cloud-operations/dns-fine-grained-iam), [Cloud DNS & Shared VPC design](./cloud-operations/dns-shared-vpc), [Delegated Role Grants](./cloud-operations/iam-delegated-role-grants), [Network Quota Monitoring](./cloud-operations/network-quota-monitoring), [Managing on-prem service account keys by uploading public keys](./cloud-operations/onprem-sa-key-management), [Compute Image builder with Hashicorp Packer](./cloud-operations/packer-image-builder), [Packer example](./cloud-operations/packer-image-builder/packer), [Compute Engine quota monitoring](./cloud-operations/compute-quota-monitoring), [Scheduled Cloud Asset Inventory Export to Bigquery](./cloud-operations/scheduled-asset-inventory-export-bq), [Configuring workload identity federation with Terraform Cloud/Enterprise workflows](./cloud-operations/terraform-cloud-dynamic-credentials), [TCP healthcheck and restart for unmanaged GCE instances](./cloud-operations/unmanaged-instances-healthcheck), [Migrate for Compute Engine (v5) blueprints](./cloud-operations/vm-migration), [Configuring workload identity federation to access Google Cloud resources from apps running on Azure](./cloud-operations/workload-identity-federation) +- **cloud operations** - [Active Directory Federation Services](./cloud-operations/adfs), [Cloud Asset Inventory feeds for resource change tracking and remediation](./cloud-operations/asset-inventory-feed-remediation), [Fine-grained Cloud DNS IAM via Service Directory](./cloud-operations/dns-fine-grained-iam), [Cloud DNS & Shared VPC design](./cloud-operations/dns-shared-vpc), [Delegated Role Grants](./cloud-operations/iam-delegated-role-grants), [Network Quota Monitoring](./cloud-operations/network-quota-monitoring), [Compute Image builder with Hashicorp Packer](./cloud-operations/packer-image-builder), [Packer example](./cloud-operations/packer-image-builder/packer), [Compute Engine quota monitoring](./cloud-operations/compute-quota-monitoring), [Scheduled Cloud Asset Inventory Export to Bigquery](./cloud-operations/scheduled-asset-inventory-export-bq), [Configuring workload identity federation with Terraform Cloud/Enterprise workflows](./cloud-operations/terraform-cloud-dynamic-credentials), [TCP healthcheck and restart for unmanaged GCE instances](./cloud-operations/unmanaged-instances-healthcheck), [Migrate for Compute Engine (v5) blueprints](./cloud-operations/vm-migration), [Configuring workload identity federation to access Google Cloud resources from apps running on Azure](./cloud-operations/workload-identity-federation) - **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Minimal Data Platform](./data-solutions/data-platform-minimal), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder), [BigQuery ML and Vertex AI Pipeline](./data-solutions/bq-ml) - **factories** - [Fabric resource factories](./factories) - **GKE** - [Binary Authorization Pipeline Blueprint](./gke/binauthz), [Storage API](./gke/binauthz/image), [Multi-cluster mesh on GKE (fleet API)](./gke/multi-cluster-mesh-gke-fleet-api), [GKE Multitenant](../fast/stages/3-gke-dev), [Shared VPC with GKE support](./networking/shared-vpc-gke/), [GKE Autopilot](./gke/autopilot) diff --git a/blueprints/cloud-operations/README.md b/blueprints/cloud-operations/README.md index e4421f7be..7bba31e3c 100644 --- a/blueprints/cloud-operations/README.md +++ b/blueprints/cloud-operations/README.md @@ -46,12 +46,6 @@ The blueprint's feed tracks changes to Google Compute instances, and the Cloud F
-## On-prem Service Account key management - -This [blueprint](./onprem-sa-key-management) shows how to manage IAM Service Account Keys by manually generating a key pair and uploading the public part of the key to GCP. - -
- ## Packer image builder This [blueprint](./packer-image-builder) shows how to deploy infrastructure for a Compute Engine image builder based on [Hashicorp's Packer tool](https://www.packer.io). diff --git a/blueprints/cloud-operations/onprem-sa-key-management/OWNERS b/blueprints/cloud-operations/onprem-sa-key-management/OWNERS deleted file mode 100644 index c7db5ffd7..000000000 --- a/blueprints/cloud-operations/onprem-sa-key-management/OWNERS +++ /dev/null @@ -1 +0,0 @@ -averbuks diff --git a/blueprints/cloud-operations/onprem-sa-key-management/README.md b/blueprints/cloud-operations/onprem-sa-key-management/README.md deleted file mode 100644 index bcbf661e1..000000000 --- a/blueprints/cloud-operations/onprem-sa-key-management/README.md +++ /dev/null @@ -1,91 +0,0 @@ -# Managing on-prem service account keys by uploading public keys - -When managing GCP Service Accounts with terraform, it's often a question on **how to avoid Service Account Key in the terraform state?** - -This blueprint shows how to manage IAM Service Account Keys by manually generating a key pair and uploading the public part of the key to GCP. It has the following benefits: - - - no [passing keys between users](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys#pass-between-users) or systems - - no private keys stored in the terraform state (only public part of the key is in the state) - - let keys [expire automatically](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys#key-expiryhaving) - - -## Running the blueprint - -Clone this repository or [open it in cloud shell](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2Fterraform-google-modules%2Fcloud-foundation-fabric&cloudshell_print=cloud-shell-readme.txt&cloudshell_working_dir=blueprints%2Fcloud-operations%2Fonprem-sa-key-management&cloudshell_open_in_editor=cloudshell_open%2Fcloud-foundation-fabric%2Fblueprints%2Fcloud-operations%2Fonprem-sa-key-management%2Fvariables.tf), then go through the following steps to create resources: - -Cleaning up blueprint keys -```bash -rm -f /public-keys/data-uploader/ -rm -f /public-keys/prisma-security/ -``` - -Generate keys for service accounts -```bash -mkdir keys && cd keys -openssl req -x509 -nodes -newkey rsa:2048 -days 30 \ - -keyout data_uploader_private_key.pem \ - -out ../public-keys/data-uploader/public_key.pem \ - -subj "/CN=unused" -openssl req -x509 -nodes -newkey rsa:2048 -days 30 \ - -keyout prisma_security_private_key.pem \ - -out ../public-keys/prisma-security/public_key.pem \ - -subj "/CN=unused" -``` - -Deploy service accounts and keys -```bash -cd .. -terraform init -terraform apply -var project_id=$GOOGLE_CLOUD_PROJECT - -``` - -Extract JSON credentials templates from terraform output and put the private part of the keys into templates -```bash -terraform show -json | jq '.values.outputs."sa-credentials".value."data-uploader"."public_key.pem" | fromjson' > data-uploader.json -terraform show -json | jq '.values.outputs."sa-credentials".value."prisma-security"."public_key.pem" | fromjson' > prisma-security.json - -contents=$(jq --arg key "$(cat keys/data_uploader_private_key.pem)" '.private_key=$key' data-uploader.json) && echo "$contents" > data-uploader.json -contents=$(jq --arg key "$(cat keys/prisma_security_private_key.pem)" '.private_key=$key' prisma-security.json) && echo "$contents" > prisma-security.json -``` - -## Testing the blueprint -Validate that service accounts json credentials are valid -```bash -gcloud auth activate-service-account --key-file prisma-security.json -gcloud auth activate-service-account --key-file data-uploader.json -``` - -## Cleaning up -```bash -terraform destroy -var project_id=$GOOGLE_CLOUD_PROJECT -``` - - -## Variables - -| name | description | type | required | default | -|---|---|:---:|:---:|:---:| -| [project_id](variables.tf#L23) | Project id. | string | ✓ | | -| [project_create](variables.tf#L17) | Create project instead of using an existing one. | bool | | false | -| [service_accounts](variables.tf#L28) | List of service accounts. | list(object({…})) | | […] | -| [services](variables.tf#L56) | Service APIs to enable. | list(string) | | [] | - -## Outputs - -| name | description | sensitive | -|---|---|:---:| -| [sa-credentials](outputs.tf#L17) | SA json key templates. | | - - - -## Test - -```hcl -module "test" { - source = "./fabric/blueprints/cloud-operations/onprem-sa-key-management" - project_create = true - project_id = "test" -} -# tftest modules=4 resources=7 -``` diff --git a/blueprints/cloud-operations/onprem-sa-key-management/backend.tf.sample b/blueprints/cloud-operations/onprem-sa-key-management/backend.tf.sample deleted file mode 100644 index 4ef7e4435..000000000 --- a/blueprints/cloud-operations/onprem-sa-key-management/backend.tf.sample +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# set a valid bucket below and rename this file to backend.tf - -terraform { - backend "gcs" { - bucket = "" - prefix = "fabric/operations/onprem-sa-key-management" - } -} - diff --git a/blueprints/cloud-operations/onprem-sa-key-management/cloud-shell-readme.txt b/blueprints/cloud-operations/onprem-sa-key-management/cloud-shell-readme.txt deleted file mode 100644 index ff75626a6..000000000 --- a/blueprints/cloud-operations/onprem-sa-key-management/cloud-shell-readme.txt +++ /dev/null @@ -1,46 +0,0 @@ - - -################################# Quickstart ################################# - -# cleaning up example keys - -- rm -f /public-keys/data-uploader/ -- rm -f /public-keys/prisma-security/ - -# generate keys for service accounts - -- mkdir keys && cd keys -- openssl req -x509 -nodes -newkey rsa:2048 -days 30 \ - -keyout data_uploader_private_key.pem \ - -out ../public-keys/data-uploader/public_key.pem \ - -subj "/CN=unused" -- openssl req -x509 -nodes -newkey rsa:2048 -days 30 \ - -keyout prisma_security_private_key.pem \ - -out ../public-keys/prisma-security/public_key.pem \ - -subj "/CN=unused" - -# deploy service accounts and keys - -- cd .. -- terraform init -- terraform apply -var project_id=$GOOGLE_CLOUD_PROJECT - - -# extract JSON credentials templates from terraform output and put the private part of the keys into templates - -- terraform show -json | jq '.values.outputs."sa-credentials".value."data-uploader"."public_key.pem" | fromjson' > data-uploader.json -- terraform show -json | jq '.values.outputs."sa-credentials".value."prisma-security"."public_key.pem" | fromjson' > prisma-security.json - -- contents=$(jq --arg key "$(cat keys/data_uploader_private_key.pem)" '.private_key=$key' data-uploader.json) && echo "$contents" > data-uploader.json -- contents=$(jq --arg key "$(cat keys/prisma_security_private_key.pem)" '.private_key=$key' prisma-security.json) && echo "$contents" > prisma-security.json - - - -# validate that service accounts json credentials are valid - -- gcloud auth activate-service-account --key-file prisma-security.json -- gcloud auth activate-service-account --key-file data-uploader.json - - -# cleaning up -- terraform destroy -var project_id=$GOOGLE_CLOUD_PROJECT diff --git a/blueprints/cloud-operations/onprem-sa-key-management/main.tf b/blueprints/cloud-operations/onprem-sa-key-management/main.tf deleted file mode 100644 index 1eec60332..000000000 --- a/blueprints/cloud-operations/onprem-sa-key-management/main.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - service_accounts = { for sa in var.service_accounts : sa.name => sa } -} - -module "project" { - source = "../../../modules/project" - name = var.project_id - project_reuse = var.project_create != true ? {} : null - services = var.services -} - -module "integration-sa" { - source = "../../../modules/iam-service-account" - for_each = local.service_accounts - project_id = module.project.project_id - name = each.value.name - iam_project_roles = { - (module.project.project_id) = each.value.iam_project_roles - } - public_keys_directory = each.value.public_keys_path -} diff --git a/blueprints/cloud-operations/onprem-sa-key-management/outputs.tf b/blueprints/cloud-operations/onprem-sa-key-management/outputs.tf deleted file mode 100644 index 9174474cd..000000000 --- a/blueprints/cloud-operations/onprem-sa-key-management/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "sa-credentials" { - description = "SA json key templates." - value = { for key, value in module.integration-sa : key => value.service_account_credentials } -} diff --git a/blueprints/cloud-operations/onprem-sa-key-management/public-keys/data-uploader/public_key.pem b/blueprints/cloud-operations/onprem-sa-key-management/public-keys/data-uploader/public_key.pem deleted file mode 100644 index ad5bc9fe5..000000000 --- a/blueprints/cloud-operations/onprem-sa-key-management/public-keys/data-uploader/public_key.pem +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICnjCCAYYCCQDhgw8htVCGmTANBgkqhkiG9w0BAQsFADARMQ8wDQYDVQQDDAZ1 -bnVzZWQwHhcNMjExMjA2MDgwNTAyWhcNMzExMjA0MDgwNTAyWjARMQ8wDQYDVQQD -DAZ1bnVzZWQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC0xlwdjkBS -1ovANJ1RXKpFdbPQWYlqKUUo+/KLClNYC9KxRqrc+u5FtPIdCPv5WRH5sz+z8gcf -3zJMht0dO7fOwJ9wSDKzvHkMUdXTGBPbm2i9PNA6f+YEwJQjWJlAHFH4Lp3x6ddT -4KO4FRQEkN/5V1+sfmyGGFaSXaoi+PcDcQHvfUUlp5iyX4I+8tqwh1kdg1M5orkE -7iBG0wHWzfOSmZq5in6t9+lWzOZeYapi8bVBm7Vz+dmHZPKS6EGmAXS1wpLCSKHB -uv23KXY4gAXOPHiDI70JpeNiSJBE9WgXs+nL78vNjLTvDhpC10b9nOxLjRc6wA5b -3q2Am0dW1DPRAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAJqyTIibNZM/q30Fn+vR -V9q++19CIervZig1uCarH1M86cpPYRfKcYHOi6tnoCTL9VG8Ky8pbmkZNkET7vnN -OQirpsPmqu3d+FBoqXUt8w1mT1JVr0YiTo3i07zTH8rvQKHjEfPxR73IAyYNvJ3D -k3SdUvU3xXOa+otOQcBKIxX6mJPLhzXgZd144KCfD95qOvpoQOsNW4UWXZ3sPC0k -VcMlN5O8/+D65y63nNtyECXvLicLdn/cdpA2H7Tqhz2ZZR+6tLcDW1kSsA8b6+rQ -1IaKpF+TYo0jMD+WLatRrOHXOWije8871zooAXq9MLVJrT889TdsmEIYT7YPWIeJ -Jcg= ------END CERTIFICATE----- diff --git a/blueprints/cloud-operations/onprem-sa-key-management/public-keys/prisma-security/public_key.pem b/blueprints/cloud-operations/onprem-sa-key-management/public-keys/prisma-security/public_key.pem deleted file mode 100644 index 5da20f0b1..000000000 --- a/blueprints/cloud-operations/onprem-sa-key-management/public-keys/prisma-security/public_key.pem +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICnjCCAYYCCQDXMv59IiZqfTANBgkqhkiG9w0BAQsFADARMQ8wDQYDVQQDDAZ1 -bnVzZWQwHhcNMjExMjA2MDgwNjE3WhcNMzExMjA0MDgwNjE3WjARMQ8wDQYDVQQD -DAZ1bnVzZWQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDOK6XwgTzL -icSITBrQBmhnYNOuggDhQr40j8/pIuTOiFZbd+ne3MhcFxpE58T9cOXgR0i/S4ok -+kcGE74H2U7RsRpNi7fJhi62T9e2CXpibURQNJD6y0lXBQkfx6kCrhyvXqHbTxm5 -J0f5mpLlze+w7ATikmYI0mrU9XjtnRJOdxtGfiIaQ/suGTaZ0z4tZgAXy9RnwUAb -LPXn0BD1+GYpCs82+1q7HpMIf343VRH0AdsQJteQSj5LKfaZZTNUF9NIgKtMylck -z0Pt8TmBU0GtJX/XkSWCwMUdqdedXkvhY1XoAZPjaEBSSwq6P15PmdpDq9q2TYjD -8U3kuCX0AlWjAgMBAAEwDQYJKoZIhvcNAQELBQADggEBADAmjI1sg150MK97DCSl -d5OpEShCypaEZSLb/mFONW6mTX2OSdF9ipd9B07BQ2DrL8Xou2/V1aDtQZOWPIGu -Hlm1LKw8sZY2rWX0Rq/v/NxY5iGRlwPMh7Rn9fnpHgaC1PktoDJEcvNMpzBjtfKn -beKP9MNSChAFTTbJVWO5xT/ljE/yoPL3jyJKzKHH7y7AfbonrbQjAENbX/WCRYh3 -zOEWZG/fusRcKkZ/cO7wFFP1gzJFE9wFRu7LOA/FntCixtVSnclsOnunQfqQEVmp -Y0IjfceIerJysCTo0I5HfRw0DOFfZimallOa4Mv5BDmzMWWyX9TvppHCnmqvM2El -ISY= ------END CERTIFICATE----- diff --git a/blueprints/cloud-operations/onprem-sa-key-management/variables.tf b/blueprints/cloud-operations/onprem-sa-key-management/variables.tf deleted file mode 100644 index 329c5debd..000000000 --- a/blueprints/cloud-operations/onprem-sa-key-management/variables.tf +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_create" { - description = "Create project instead of using an existing one." - type = bool - default = false -} - -variable "project_id" { - description = "Project id." - type = string -} - -variable "service_accounts" { - description = "List of service accounts." - type = list(object({ - name = string - iam_project_roles = list(string) - public_keys_path = string - })) - default = [ - { - name = "data-uploader" - iam_project_roles = [ - "roles/bigquery.dataOwner", - "roles/bigquery.jobUser", - "roles/storage.objectAdmin" - ] - public_keys_path = "public-keys/data-uploader/" - }, - { - name = "prisma-security" - iam_project_roles = [ - "roles/iam.securityReviewer" - ] - public_keys_path = "public-keys/prisma-security/" - }, - ] - -} - -variable "services" { - description = "Service APIs to enable." - type = list(string) - default = [] -} diff --git a/modules/iam-service-account/README.md b/modules/iam-service-account/README.md index e039fc87f..60796a7d4 100644 --- a/modules/iam-service-account/README.md +++ b/modules/iam-service-account/README.md @@ -2,11 +2,20 @@ This module allows simplified creation and management of one a service account and its IAM bindings. -The Service Account `key` can be generated with `openssl` library and only the public part uploaded to the Service Account, for more refer to the [Onprem SA Key Management](../../blueprints/cloud-operations/onprem-sa-key-management/) example. - Note that outputs have no dependencies on IAM bindings to prevent resource cycles. -## Example +## TOC + + +- [TOC](#toc) +- [Simple Example](#simple-example) +- [Tag Bindings](#tag-bindings) +- [Files](#files) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Simple Example ```hcl module "myproject-default-service-accounts" { @@ -27,6 +36,25 @@ module "myproject-default-service-accounts" { } # tftest modules=1 resources=4 inventory=basic.yaml e2e ``` + +## Tag Bindings + +Use the `tag_bindings` variable to attach tags to the service account. Provide `project_number` to prevent potential permadiffs with the tag binding resource. + +```hcl +module "service-account-with-tags" { + source = "./fabric/modules/iam-service-account" + project_id = var.project_id + name = "test-service-account" + project_number = var.project_number + tag_bindings = { + foo = "tagValues/123456789" + } +} +# tftest modules=1 resources=2 inventory=tags.yaml +``` + + ## Files @@ -34,7 +62,7 @@ module "myproject-default-service-accounts" { | name | description | resources | |---|---|---| | [iam.tf](./iam.tf) | IAM bindings. | google_billing_account_iam_member · google_folder_iam_member · google_organization_iam_member · google_project_iam_member · google_service_account_iam_binding · google_service_account_iam_member · google_storage_bucket_iam_member | -| [main.tf](./main.tf) | Module-level locals and resources. | google_service_account · google_service_account_key | +| [main.tf](./main.tf) | Module-level locals and resources. | google_service_account · google_tags_tag_binding | | [outputs.tf](./outputs.tf) | Module outputs. | | | [variables.tf](./variables.tf) | Module variables. | | | [versions.tf](./versions.tf) | Version pins. | | @@ -43,22 +71,24 @@ module "myproject-default-service-accounts" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L108) | Name of the service account to create. | string | ✓ | | -| [project_id](variables.tf#L123) | Project id where service account will be created. | string | ✓ | | -| [description](variables.tf#L17) | Optional description. | string | | null | -| [display_name](variables.tf#L23) | Display name of the service account to create. | string | | "Terraform-managed." | -| [iam](variables.tf#L29) | IAM bindings on the service account in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_billing_roles](variables.tf#L36) | Billing account roles granted to this service account, by billing account id. Non-authoritative. | map(list(string)) | | {} | -| [iam_bindings](variables.tf#L43) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | -| [iam_bindings_additive](variables.tf#L58) | Individual additive IAM bindings on the service account. Keys are arbitrary. | map(object({…})) | | {} | -| [iam_folder_roles](variables.tf#L73) | Folder roles granted to this service account, by folder id. Non-authoritative. | map(list(string)) | | {} | -| [iam_organization_roles](variables.tf#L80) | Organization roles granted to this service account, by organization id. Non-authoritative. | map(list(string)) | | {} | -| [iam_project_roles](variables.tf#L87) | Project roles granted to this service account, by project id. | map(list(string)) | | {} | -| [iam_sa_roles](variables.tf#L94) | Service account roles granted to this service account, by service account name. | map(list(string)) | | {} | -| [iam_storage_roles](variables.tf#L101) | Storage roles granted to this service account, by bucket name. | map(list(string)) | | {} | -| [prefix](variables.tf#L113) | Prefix applied to service account names. | string | | null | -| [public_keys_directory](variables.tf#L128) | Path to public keys data files to upload to the service account (should have `.pem` extension). | string | | "" | -| [service_account_create](variables.tf#L134) | Create service account. When set to false, uses a data source to reference an existing service account. | bool | | true | +| [name](variables.tf#L118) | Name of the service account to create. | string | ✓ | | +| [project_id](variables.tf#L133) | Project id where service account will be created. | string | ✓ | | +| [create_ignore_already_exists](variables.tf#L17) | If set to true, skip service account creation if a service account with the same email already exists. | bool | | null | +| [description](variables.tf#L27) | Optional description. | string | | null | +| [display_name](variables.tf#L33) | Display name of the service account to create. | string | | "Terraform-managed." | +| [iam](variables.tf#L39) | IAM bindings on the service account in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_billing_roles](variables.tf#L46) | Billing account roles granted to this service account, by billing account id. Non-authoritative. | map(list(string)) | | {} | +| [iam_bindings](variables.tf#L53) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L68) | Individual additive IAM bindings on the service account. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_folder_roles](variables.tf#L83) | Folder roles granted to this service account, by folder id. Non-authoritative. | map(list(string)) | | {} | +| [iam_organization_roles](variables.tf#L90) | Organization roles granted to this service account, by organization id. Non-authoritative. | map(list(string)) | | {} | +| [iam_project_roles](variables.tf#L97) | Project roles granted to this service account, by project id. | map(list(string)) | | {} | +| [iam_sa_roles](variables.tf#L104) | Service account roles granted to this service account, by service account name. | map(list(string)) | | {} | +| [iam_storage_roles](variables.tf#L111) | Storage roles granted to this service account, by bucket name. | map(list(string)) | | {} | +| [prefix](variables.tf#L123) | Prefix applied to service account names. | string | | null | +| [project_number](variables.tf#L138) | Project number of var.project_id. Set this to avoid permadiffs when creating tag bindings. | string | | null | +| [service_account_create](variables.tf#L144) | Create service account. When set to false, uses a data source to reference an existing service account. | bool | | true | +| [tag_bindings](variables.tf#L151) | Tag bindings for this service accounts, in key => tag value id format. | map(string) | | {} | ## Outputs @@ -69,5 +99,4 @@ module "myproject-default-service-accounts" { | [id](outputs.tf#L33) | Fully qualified service account id. | | | [name](outputs.tf#L41) | Service account name. | | | [service_account](outputs.tf#L49) | Service account resource. | | -| [service_account_credentials](outputs.tf#L54) | Service account json credential templates for uploaded public keys data. | | diff --git a/modules/iam-service-account/main.tf b/modules/iam-service-account/main.tf index 9bf50f05c..b73d49126 100644 --- a/modules/iam-service-account/main.tf +++ b/modules/iam-service-account/main.tf @@ -30,36 +30,13 @@ locals { ? try(google_service_account.service_account[0], null) : try(data.google_service_account.service_account[0], null) ) - service_account_credential_templates = { - for file, _ in local.public_keys_data : file => jsonencode( - { - type : "service_account", - project_id : var.project_id, - private_key_id : split("/", google_service_account_key.upload_key[file].id)[5] - private_key : "REPLACE_ME_WITH_PRIVATE_KEY_DATA" - client_email : local.resource_email_static - client_id : local.service_account.unique_id, - auth_uri : "https://accounts.google.com/o/oauth2/auth", - token_uri : "https://oauth2.googleapis.com/token", - auth_provider_x509_cert_url : "https://www.googleapis.com/oauth2/v1/certs", - client_x509_cert_url : "https://www.googleapis.com/robot/v1/metadata/x509/${urlencode(local.resource_email_static)}" - } - ) - } - public_keys_data = ( - var.public_keys_directory != "" - ? { - for file in fileset("${path.root}/${var.public_keys_directory}", "*.pem") - : file => filebase64("${path.root}/${var.public_keys_directory}/${file}") } - : {} - ) + # universe-related locals universe = try(regex("^([^:]*):[a-z]", var.project_id)[0], "") project_id_no_universe = element(split(":", var.project_id), 1) sa_domain = join(".", compact([local.project_id_no_universe, local.universe])) } - data "google_service_account" "service_account" { count = var.service_account_create ? 0 : 1 project = var.project_id @@ -67,15 +44,16 @@ data "google_service_account" "service_account" { } resource "google_service_account" "service_account" { - count = var.service_account_create ? 1 : 0 - project = var.project_id - account_id = "${local.prefix}${local.name}" - display_name = var.display_name - description = var.description + count = var.service_account_create ? 1 : 0 + project = var.project_id + account_id = "${local.prefix}${local.name}" + display_name = var.display_name + description = var.description + create_ignore_already_exists = var.create_ignore_already_exists } -resource "google_service_account_key" "upload_key" { - for_each = local.public_keys_data - service_account_id = local.service_account.email - public_key_data = each.value +resource "google_tags_tag_binding" "binding" { + for_each = var.tag_bindings + parent = "//iam.googleapis.com/projects/${coalesce(var.project_number, var.project_id)}/serviceAccounts/${local.service_account.unique_id}" + tag_value = each.value } diff --git a/modules/iam-service-account/outputs.tf b/modules/iam-service-account/outputs.tf index 5f4771013..376262a7a 100644 --- a/modules/iam-service-account/outputs.tf +++ b/modules/iam-service-account/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -50,8 +50,3 @@ output "service_account" { description = "Service account resource." value = local.service_account } - -output "service_account_credentials" { - description = "Service account json credential templates for uploaded public keys data." - value = local.service_account_credential_templates -} diff --git a/modules/iam-service-account/variables.tf b/modules/iam-service-account/variables.tf index 852c87c45..e8ce36fbc 100644 --- a/modules/iam-service-account/variables.tf +++ b/modules/iam-service-account/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,16 @@ * limitations under the License. */ +variable "create_ignore_already_exists" { + description = "If set to true, skip service account creation if a service account with the same email already exists." + type = bool + default = null + validation { + condition = !(var.create_ignore_already_exists == true && var.service_account_create == false) + error_message = "Cannot set create_ignore_already_exists when service_account_create is false." + } +} + variable "description" { description = "Optional description." type = string @@ -125,14 +135,22 @@ variable "project_id" { type = string } -variable "public_keys_directory" { - description = "Path to public keys data files to upload to the service account (should have `.pem` extension)." +variable "project_number" { + description = "Project number of var.project_id. Set this to avoid permadiffs when creating tag bindings." type = string - default = "" + default = null } variable "service_account_create" { description = "Create service account. When set to false, uses a data source to reference an existing service account." type = bool default = true + nullable = false +} + +variable "tag_bindings" { + description = "Tag bindings for this service accounts, in key => tag value id format." + type = map(string) + nullable = false + default = {} } diff --git a/tests/modules/iam_service_account/examples/tags.yaml b/tests/modules/iam_service_account/examples/tags.yaml new file mode 100644 index 000000000..65bc1314e --- /dev/null +++ b/tests/modules/iam_service_account/examples/tags.yaml @@ -0,0 +1,36 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.service-account-with-tags.google_service_account.service_account[0]: + account_id: test-service-account + create_ignore_already_exists: null + description: null + disabled: false + display_name: Terraform-managed. + email: test-service-account@project-id.iam.gserviceaccount.com + member: serviceAccount:test-service-account@project-id.iam.gserviceaccount.com + project: project-id + timeouts: null + module.service-account-with-tags.google_tags_tag_binding.binding["foo"]: + tag_value: tagValues/123456789 + timeouts: null + +counts: + google_service_account: 1 + google_tags_tag_binding: 1 + modules: 1 + resources: 2 + +outputs: {} From 1c87246583b4e7e11bdce6f718e6f5cbac4878de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Tue, 1 Apr 2025 13:45:29 +0000 Subject: [PATCH 03/10] Use factory-projects-object for project object normalization --- modules/project-factory/factory-projects-object.tf | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/project-factory/factory-projects-object.tf b/modules/project-factory/factory-projects-object.tf index 4dca20ba2..0703beafb 100644 --- a/modules/project-factory/factory-projects-object.tf +++ b/modules/project-factory/factory-projects-object.tf @@ -21,7 +21,7 @@ # data_defaults = ... # }) # outputs: -# projects - map +# local._projects_output - map locals { __projects_config = { data_defaults = merge({ @@ -158,6 +158,11 @@ locals { ), null) } + iam = try(v.iam, {}) + iam_bindings = try(v.iam_bindings, {}) + iam_bindings_additive = try(v.iam_bindings_additive, {}) + iam_by_principals_additive = try(v.iam_by_principals_additive, {}) + iam_by_principals = try(v.iam_by_principals, {}) labels = coalesce( try(v.labels, null), local.__projects_config.data_defaults.labels From 6d54ff9a45bce0b389b4229de430b4e6436a8420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Tue, 1 Apr 2025 14:49:41 +0000 Subject: [PATCH 04/10] Use the same keys in output as in input for projects --- .../project-factory/factory-projects-object.tf | 3 ++- modules/project-factory/factory-projects.tf | 11 ++++++----- modules/project-factory/main.tf | 16 ++++++++-------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/modules/project-factory/factory-projects-object.tf b/modules/project-factory/factory-projects-object.tf index 0703beafb..e23b818c8 100644 --- a/modules/project-factory/factory-projects-object.tf +++ b/modules/project-factory/factory-projects-object.tf @@ -116,7 +116,7 @@ locals { ) } _projects_output = { - for k, v in local._projects_input : lookup(v, "name", k) => merge(v, { + for k, v in local._projects_input : k => merge(v, { billing_account = try(coalesce( local.__projects_config.data_overrides.billing_account, try(v.billing_account, null), @@ -171,6 +171,7 @@ locals { try(v.metric_scopes, null), local.__projects_config.data_defaults.metric_scopes ) + name = lookup(v, "name", k) org_policies = try(v.org_policies, {}) parent = coalesce( local.__projects_config.data_overrides.parent, diff --git a/modules/project-factory/factory-projects.tf b/modules/project-factory/factory-projects.tf index deb619822..6efcc9c32 100644 --- a/modules/project-factory/factory-projects.tf +++ b/modules/project-factory/factory-projects.tf @@ -60,7 +60,8 @@ locals { buckets = flatten([ for k, v in local.projects : [ for name, opts in v.buckets : { - project = k + project_key = k + project_name = v.name name = name description = lookup(opts, "description", "Terraform-managed.") encryption_key = lookup(opts, "encryption_key", null) @@ -88,10 +89,10 @@ locals { ] ]) service_accounts = flatten([ - for k, v in local.projects : [ - for name, opts in v.service_accounts : { - project = k - name = name + for k, project in local.projects : [ + for name, opts in project.service_accounts : { + project_key = k + name = name display_name = coalesce( try(var.data_overrides.service_accounts.display_name, null), try(opts.display_name, null), diff --git a/modules/project-factory/main.tf b/modules/project-factory/main.tf index 6787c0cf0..d6ee531c6 100644 --- a/modules/project-factory/main.tf +++ b/modules/project-factory/main.tf @@ -195,19 +195,19 @@ module "projects-iam" { module "buckets" { source = "../gcs" for_each = { - for k in local.buckets : "${k.project}/${k.name}" => k + for k in local.buckets : "${k.project_key}/${k.name}" => k } - project_id = module.projects[each.value.project].project_id + project_id = module.projects[each.value.project_key].project_id prefix = each.value.prefix - name = "${each.value.project}-${each.value.name}" + name = "${each.value.project_name}-${each.value.name}" encryption_key = each.value.encryption_key iam = { for k, v in each.value.iam : k => [ for vv in v : try( # project service accounts - module.service-accounts["${each.value.project}/${vv}"].iam_email, + module.service-accounts["${each.value.project_key}/${vv}"].iam_email, # automation service account - local.context.iam_principals["${each.value.project}/${vv}"], + local.context.iam_principals["${each.value.project_key}/${vv}"], # other projects service accounts module.service-accounts[vv].iam_email, # other automation service account @@ -265,9 +265,9 @@ module "buckets" { module "service-accounts" { source = "../iam-service-account" for_each = { - for k in local.service_accounts : "${k.project}/${k.name}" => k + for k in local.service_accounts : "${k.project_key}/${k.name}" => k } - project_id = module.projects[each.value.project].project_id + project_id = module.projects[each.value.project_key].project_id name = each.value.name display_name = each.value.display_name iam_project_roles = merge( @@ -276,7 +276,7 @@ module "service-accounts" { lookup(var.factories_config.context.vpc_host_projects, k, k) => v }, each.value.iam_self_roles == null ? {} : { - (module.projects[each.value.project].project_id) = each.value.iam_self_roles + (module.projects[each.value.project_key].project_id) = each.value.iam_self_roles } ) } From ce05505a2c94eae5bf80b7771cbf3ef99ef34aa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Wed, 2 Apr 2025 16:55:23 +0000 Subject: [PATCH 05/10] Add type information to project_config attributes --- .../factory-projects-object.tf | 53 +++++++++---------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/modules/project-factory/factory-projects-object.tf b/modules/project-factory/factory-projects-object.tf index e23b818c8..582e8256c 100644 --- a/modules/project-factory/factory-projects-object.tf +++ b/modules/project-factory/factory-projects-object.tf @@ -117,18 +117,18 @@ locals { } _projects_output = { for k, v in local._projects_input : k => merge(v, { - billing_account = try(coalesce( + billing_account = try(coalesce( # type: string local.__projects_config.data_overrides.billing_account, try(v.billing_account, null), local.__projects_config.data_defaults.billing_account ), null) - contacts = coalesce( + contacts = coalesce( # type: map local.__projects_config.data_overrides.contacts, try(v.contacts, null), local.__projects_config.data_defaults.contacts ) - factories_config = { - custom_roles = try( + factories_config = { # type: object + custom_roles = try( # type: string coalesce( local.__projects_config.data_overrides.factories_config.custom_roles, try(v.factories_config.custom_roles, null), @@ -136,21 +136,21 @@ locals { ), null ) - observability = try( + observability = try( # type: string coalesce( local.__projects_config.data_overrides.factories_config.observability, try(v.factories_config.observability, null), local.__projects_config.data_defaults.factories_config.observability ), null) - org_policies = try( + org_policies = try( # type: string coalesce( local.__projects_config.data_overrides.factories_config.org_policies, try(v.factories_config.org_policies, null), local.__projects_config.data_defaults.factories_config.org_policies ), null) - quotas = try( + quotas = try( # type: string coalesce( local.__projects_config.data_overrides.factories_config.quotas, try(v.factories_config.quotas, null), @@ -158,42 +158,42 @@ locals { ), null) } - iam = try(v.iam, {}) - iam_bindings = try(v.iam_bindings, {}) - iam_bindings_additive = try(v.iam_bindings_additive, {}) - iam_by_principals_additive = try(v.iam_by_principals_additive, {}) - iam_by_principals = try(v.iam_by_principals, {}) - labels = coalesce( + iam = try(v.iam, {}) # type: map(list(string)) + iam_bindings = try(v.iam_bindings, {}) # type: map(object({...})) + iam_bindings_additive = try(v.iam_bindings_additive, {}) # type: map(object({...})) + iam_by_principals_additive = try(v.iam_by_principals_additive, {}) # type: map(list(string)) + iam_by_principals = try(v.iam_by_principals, {}) # map(list(string)) + labels = coalesce( # type: map(string) try(v.labels, null), local.__projects_config.data_defaults.labels ) - metric_scopes = coalesce( + metric_scopes = coalesce( # type: list(string) try(v.metric_scopes, null), local.__projects_config.data_defaults.metric_scopes ) - name = lookup(v, "name", k) - org_policies = try(v.org_policies, {}) - parent = coalesce( + name = lookup(v, "name", k) # type: string + org_policies = try(v.org_policies, {}) # type: map(object({...})) + parent = coalesce( # type: string local.__projects_config.data_overrides.parent, try(v.parent, null), local.__projects_config.data_defaults.parent ) - prefix = coalesce( + prefix = coalesce( # type: string local.__projects_config.data_overrides.prefix, try(v.prefix, null), local.__projects_config.data_defaults.prefix ) - service_encryption_key_ids = coalesce( + service_encryption_key_ids = coalesce( # type: map(list(string)) local.__projects_config.data_overrides.service_encryption_key_ids, try(v.service_encryption_key_ids, null), local.__projects_config.data_defaults.service_encryption_key_ids ) - services = coalesce( + services = coalesce( # type: list(string) local.__projects_config.data_overrides.services, try(v.services, null), local.__projects_config.data_defaults.services ) - shared_vpc_host_config = ( + shared_vpc_host_config = ( # type: object({...}) try(v.shared_vpc_host_config, null) != null ? merge( { service_projects = [] }, @@ -201,7 +201,7 @@ locals { ) : null ) - shared_vpc_service_config = ( + shared_vpc_service_config = ( # type: object({...}) try(v.shared_vpc_service_config, null) != null ? merge( { @@ -216,12 +216,12 @@ locals { ) : local.__projects_config.data_defaults.shared_vpc_service_config ) - tag_bindings = coalesce( + tag_bindings = coalesce( # type: map(string) local.__projects_config.data_overrides.tag_bindings, try(v.tag_bindings, null), local.__projects_config.data_defaults.tag_bindings ) - vpc_sc = ( + vpc_sc = ( # type: object local.__projects_config.data_overrides.vpc_sc != null ? local.__projects_config.data_overrides.vpc_sc : ( @@ -234,14 +234,11 @@ locals { : local.__projects_config.data_defaults.vpc_sc ) ) - logging_data_access = coalesce( + logging_data_access = coalesce( # type: map(object({...})) local.__projects_config.data_overrides.logging_data_access, try(v.logging_data_access, null), local.__projects_config.data_defaults.logging_data_access ) - # non-project resources - # buckets = try(v.buckets, {}) - # service_accounts = try(v.service_accounts, {}) }) } } From 46f731fee076643fe3aec84b8905411f32fdcc11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Wed, 2 Apr 2025 17:07:11 +0000 Subject: [PATCH 06/10] Use null values for data_overrides --- modules/project-factory/README.md | 4 +-- .../factory-projects-object.tf | 31 +++++++++---------- modules/project-factory/variables.tf | 3 +- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/modules/project-factory/README.md b/modules/project-factory/README.md index d3f6de13b..38fc12848 100644 --- a/modules/project-factory/README.md +++ b/modules/project-factory/README.md @@ -484,10 +484,10 @@ service_accounts: | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [factories_config](variables.tf#L120) | Path to folder with YAML resource description data files. | object({…}) | ✓ | | +| [factories_config](variables.tf#L121) | Path to folder with YAML resource description data files. | object({…}) | ✓ | | | [data_defaults](variables.tf#L17) | Optional default values used when corresponding project data from files are missing. | object({…}) | | {} | | [data_merges](variables.tf#L64) | Optional values that will be merged with corresponding data from files. Combines with `data_defaults`, file data, and `data_overrides`. | object({…}) | | {} | -| [data_overrides](variables.tf#L83) | Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`. | object({…}) | | {} | +| [data_overrides](variables.tf#L83) | Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`. | object({…}) | | {} | ## Outputs diff --git a/modules/project-factory/factory-projects-object.tf b/modules/project-factory/factory-projects-object.tf index 582e8256c..b9cc8c60a 100644 --- a/modules/project-factory/factory-projects-object.tf +++ b/modules/project-factory/factory-projects-object.tf @@ -78,9 +78,10 @@ locals { }, try(local._projects_config.data_defaults, {}) ) + # data_overrides default to null's, to mark that they should not override data_overrides = merge({ billing_account = null - contacts = {} + contacts = null factories_config = merge({ custom_roles = null observability = null @@ -95,27 +96,25 @@ locals { ) parent = null prefix = null - service_encryption_key_ids = {} + service_encryption_key_ids = null storage_location = null - tag_bindings = {} - services = [] - service_accounts = {} - vpc_sc = merge({ - perimeter_name = null - perimeter_bridges = [] - is_dry_run = false - }, try(local._projects_config.data_overrides.vpc_sc, { - perimeter_name = null - perimeter_bridges = [] - is_dry_run = false - }) - ) - logging_data_access = {} + tag_bindings = null + services = null + service_accounts = null + vpc_sc = try(local._projects_config.data_overrides.vpc_sc, null) + logging_data_access = null }, try(local._projects_config.data_overrides, {}) ) } _projects_output = { + # Semantics of the merges are: + # * if data_overrides. is not null, use this value + # * if _projects_inputs. is not null, use this value + # * use data_default value, which if not set, will provide "empty" type + # This logic is easily implemented using coalesce, even on maps and list and allows to + # set data_overrides. to "", [] or {} to ensure, that empty value is always passed, or do + # the same in _projects_input to prevent falling back to default value for k, v in local._projects_input : k => merge(v, { billing_account = try(coalesce( # type: string local.__projects_config.data_overrides.billing_account, diff --git a/modules/project-factory/variables.tf b/modules/project-factory/variables.tf index b5bb14812..4f9bb18db 100644 --- a/modules/project-factory/variables.tf +++ b/modules/project-factory/variables.tf @@ -83,6 +83,7 @@ variable "data_merges" { variable "data_overrides" { description = "Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`." type = object({ + # data overrides default to null to mark that they should not override billing_account = optional(string) contacts = optional(map(list(string))) factories_config = optional(object({ @@ -111,7 +112,7 @@ variable "data_overrides" { ADMIN_READ = optional(object({ exempted_members = optional(list(string)) })), DATA_READ = optional(object({ exempted_members = optional(list(string)) })), DATA_WRITE = optional(object({ exempted_members = optional(list(string)) })) - })), {}) + }))) }) nullable = false default = {} From c90005553aad327fdcf3885fb44b207ed1d5bd03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Wed, 2 Apr 2025 19:54:59 +0000 Subject: [PATCH 07/10] Add tests for data_defaults / data_overrides --- tests/fixtures.py | 15 +- .../projects/service1.yaml | 25 +++ .../projects/service2.yaml | 17 ++ .../data_overrides_defaults.tfvars | 51 ++++++ .../data_overrides_defaults.yaml | 64 +++++++ .../project_factory/empty_vpc_defaults.tfvars | 42 +++++ .../project_factory/empty_vpc_defaults.yaml | 173 ++++++++++++++++++ tests/modules/project_factory/tftest.yaml | 3 + 8 files changed, 385 insertions(+), 5 deletions(-) create mode 100644 tests/modules/project_factory/data/data_overrides_defaults/projects/service1.yaml create mode 100644 tests/modules/project_factory/data/data_overrides_defaults/projects/service2.yaml create mode 100644 tests/modules/project_factory/data_overrides_defaults.tfvars create mode 100644 tests/modules/project_factory/data_overrides_defaults.yaml create mode 100644 tests/modules/project_factory/empty_vpc_defaults.tfvars create mode 100644 tests/modules/project_factory/empty_vpc_defaults.yaml diff --git a/tests/fixtures.py b/tests/fixtures.py index 241484c17..c45d19697 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -207,11 +207,16 @@ def plan_validator(module_path, inventory_paths, basedir, tf_var_files=None, try: expected_counts = inventory['counts'] for type_, expected_count in expected_counts.items(): - assert type_ in summary.counts, \ - f'{relative_path}: module does not create any resources of type `{type_}`' - plan_count = summary.counts[type_] - assert plan_count == expected_count, \ - f'{relative_path}: count of {type_} resources failed. Got {plan_count}, expected {expected_count}' + # modules and resources always exists in summary + if expected_count == 0 and type_ not in ('modules', 'resources'): + assert type_ not in summary.counts, \ + f'{relative_path}: module creates resources of type `{type_}` when expected not to create any' + else: + assert type_ in summary.counts, \ + f'{relative_path}: module does not create any resources of type `{type_}`' + plan_count = summary.counts[type_] + assert plan_count == expected_count, \ + f'{relative_path}: count of {type_} resources failed. Got {plan_count}, expected {expected_count}' except AssertionError: print(f'\n{path}') print(yaml.dump({'counts': summary.counts})) diff --git a/tests/modules/project_factory/data/data_overrides_defaults/projects/service1.yaml b/tests/modules/project_factory/data/data_overrides_defaults/projects/service1.yaml new file mode 100644 index 000000000..e34bb85c9 --- /dev/null +++ b/tests/modules/project_factory/data/data_overrides_defaults/projects/service1.yaml @@ -0,0 +1,25 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +billing_account: 012345-67890A-BCDEF0 + +contacts: # this should be overridden by value + admin-default@example.org: + - "ALL" + +tag_bindings: # this should be overridden with empty value + name1: project_id1 + +services: + - run.googleapis.com diff --git a/tests/modules/project_factory/data/data_overrides_defaults/projects/service2.yaml b/tests/modules/project_factory/data/data_overrides_defaults/projects/service2.yaml new file mode 100644 index 000000000..fb254a9d2 --- /dev/null +++ b/tests/modules/project_factory/data/data_overrides_defaults/projects/service2.yaml @@ -0,0 +1,17 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +billing_account: 012345-67890A-BCDEF0 + +# take defaults + overrides only diff --git a/tests/modules/project_factory/data_overrides_defaults.tfvars b/tests/modules/project_factory/data_overrides_defaults.tfvars new file mode 100644 index 000000000..168bd8f0b --- /dev/null +++ b/tests/modules/project_factory/data_overrides_defaults.tfvars @@ -0,0 +1,51 @@ +data_defaults = { + billing_account = "1245-5678-9012" + parent = "folders/1234" + storage_location = "EU" + contacts = { + "admin-default@example.org" = ["ALL"] # should not surface, as overrides provide value + } + tag_bindings = { # should not surface, as overrides provide empty value + name1 = "default-id1" + name2 = "default-id2" + } + services = [ + "default-service.googleapis.com" + ] +} +# make sure the environment label and stackdriver service are always added +data_merges = { + labels = { + environment = "test" + } + services = [ + "stackdriver.googleapis.com" + ] +} +# always use this contacts and prefix, regardless of what is in the yaml file +data_overrides = { + contacts = { + "admin@example.org" = ["ALL"] + } + tag_bindings = {} # prevent setting any encryption keys + prefix = "test-pf" +} +# location where the yaml files are read from +factories_config = { + projects_data_path = "projects" + context = { + folder_ids = { + default = "folders/5678901234" + teams = "folders/5678901234" + } + iam_principals = { + gcp-devops = "group:gcp-devops@example.org" + } + tag_values = { + "org-policies/drs-allow-all" = "tagValues/123456" + } + vpc_host_projects = { + dev-spoke-0 = "test-pf-dev-net-spoke-0" + } + } +} diff --git a/tests/modules/project_factory/data_overrides_defaults.yaml b/tests/modules/project_factory/data_overrides_defaults.yaml new file mode 100644 index 000000000..8afed2dc6 --- /dev/null +++ b/tests/modules/project_factory/data_overrides_defaults.yaml @@ -0,0 +1,64 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.projects["service1"].google_essential_contacts_contact.contact["admin@example.org"]: + email: admin@example.org + parent: projects/test-pf-service1 + module.projects["service1"].google_project.project[0]: + billing_account: 012345-67890A-BCDEF0 + folder_id: '1234' + labels: + environment: test + name: test-pf-service1 + project_id: test-pf-service1 + module.projects["service1"].google_project_service.project_services["run.googleapis.com"]: + project: test-pf-service1 + service: run.googleapis.com + module.projects["service1"].google_project_service.project_services["stackdriver.googleapis.com"]: + project: test-pf-service1 + service: stackdriver.googleapis.com + module.projects["service2"].google_essential_contacts_contact.contact["admin@example.org"]: + email: admin@example.org + parent: projects/test-pf-service2 + module.projects["service2"].google_project.project[0]: + billing_account: 012345-67890A-BCDEF0 + folder_id: '1234' + labels: + environment: test + name: test-pf-service2 + project_id: test-pf-service2 + module.projects["service2"].google_project_service.project_services["default-service.googleapis.com"]: + project: test-pf-service2 + service: default-service.googleapis.com + module.projects["service2"].google_project_service.project_services["stackdriver.googleapis.com"]: + project: test-pf-service2 + service: stackdriver.googleapis.com + +counts: + google_essential_contacts_contact: 2 + google_project: 2 + google_project_iam_member: 1 + google_project_service: 4 + google_project_service_identity: 1 + google_tags_tag_binding: 0 # keep this, to ensure that tag_bindings are not created + modules: 2 + resources: 10 + +outputs: + buckets: {} + folders: {} + foo: {} + projects: __missing__ + service_accounts: {} diff --git a/tests/modules/project_factory/empty_vpc_defaults.tfvars b/tests/modules/project_factory/empty_vpc_defaults.tfvars new file mode 100644 index 000000000..acf42163e --- /dev/null +++ b/tests/modules/project_factory/empty_vpc_defaults.tfvars @@ -0,0 +1,42 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +data_defaults = { + billing_account = "1245-5678-9012" + storage_location = "EU" + prefix = "my-prefix" + parent = "folders/1234" + shared_vpc_service_config = null +} +data_merges = { + services = [ + "stackdriver.googleapis.com" + ] +} +data_overrides = { + prefix = "myprefix" +} +# location where the yaml files are read from +factories_config = { + projects_data_path = "projects" + context = { + folder_ids = { + default = "folders/5678901234" + teams = "folders/4321056789" + } + vpc_host_projects = { + dev-spoke-0 = "test-pf-dev-net-spoke-0" + } + } +} diff --git a/tests/modules/project_factory/empty_vpc_defaults.yaml b/tests/modules/project_factory/empty_vpc_defaults.yaml new file mode 100644 index 000000000..c45202289 --- /dev/null +++ b/tests/modules/project_factory/empty_vpc_defaults.yaml @@ -0,0 +1,173 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.automation-service-accounts["service1/ro"].google_service_account.service_account[0]: + account_id: myprefix-service1-ro + create_ignore_already_exists: null + description: Service read-only automation sa. + disabled: false + display_name: Service account ro for service1. + email: myprefix-service1-ro@service-iac.iam.gserviceaccount.com + member: serviceAccount:myprefix-service1-ro@service-iac.iam.gserviceaccount.com + project: service-iac + timeouts: null + module.automation-service-accounts["service1/rw"].google_service_account.service_account[0]: + account_id: myprefix-service1-rw + create_ignore_already_exists: null + description: Service read/write automation sa. + disabled: false + display_name: Service account rw for service1. + email: myprefix-service1-rw@service-iac.iam.gserviceaccount.com + member: serviceAccount:myprefix-service1-rw@service-iac.iam.gserviceaccount.com + project: service-iac + timeouts: null + module.projects-iam["service1"].google_compute_shared_vpc_service_project.shared_vpc_service[0]: + deletion_policy: null + host_project: test-pf-dev-net-spoke-0 + service_project: myprefix-service1 + timeouts: null + ? module.projects-iam["service1"].google_project_iam_member.shared_vpc_host_iam["serviceAccount:myprefix-service1-ro@service-iac.iam.gserviceaccount.com"] + : condition: [] + member: serviceAccount:myprefix-service1-ro@service-iac.iam.gserviceaccount.com + project: test-pf-dev-net-spoke-0 + role: roles/compute.networkUser + ? module.projects-iam["service1"].google_project_iam_member.shared_vpc_host_iam["serviceAccount:myprefix-service1-rw@service-iac.iam.gserviceaccount.com"] + : condition: [] + member: serviceAccount:myprefix-service1-rw@service-iac.iam.gserviceaccount.com + project: test-pf-dev-net-spoke-0 + role: roles/compute.networkUser + ? module.projects-iam["service1"].google_project_iam_member.shared_vpc_host_iam["serviceAccount:terraform-rw@myprefix-service1.iam.gserviceaccount.com"] + : condition: [] + member: serviceAccount:terraform-rw@myprefix-service1.iam.gserviceaccount.com + project: test-pf-dev-net-spoke-0 + role: roles/compute.networkUser + module.projects["service1"].google_project.project[0]: + auto_create_network: false + billing_account: 012345-67890A-BCDEF0 + deletion_policy: DELETE + effective_labels: + goog-terraform-provisioned: 'true' + folder_id: '1234' + labels: null + name: myprefix-service1 + org_id: null + project_id: myprefix-service1 + tags: null + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null + module.projects["service1"].google_project_service.project_services["stackdriver.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: myprefix-service1 + service: stackdriver.googleapis.com + timeouts: null + module.projects["service2"].data.google_storage_project_service_account.gcs_sa[0]: + project: myprefix-service2 + user_project: null + module.projects["service2"].google_project.project[0]: + auto_create_network: false + billing_account: 012345-67890A-BCDEF0 + deletion_policy: DELETE + effective_labels: + goog-terraform-provisioned: 'true' + folder_id: '1234' + labels: null + name: myprefix-service2 + org_id: null + project_id: myprefix-service2 + tags: null + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null + module.projects["service2"].google_project_iam_member.service_agents["compute-system"]: + condition: [] + project: myprefix-service2 + role: roles/compute.serviceAgent + module.projects["service2"].google_project_service.project_services["compute.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: myprefix-service2 + service: compute.googleapis.com + timeouts: null + module.projects["service2"].google_project_service.project_services["stackdriver.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: myprefix-service2 + service: stackdriver.googleapis.com + timeouts: null + module.projects["service2"].google_project_service.project_services["storage.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: myprefix-service2 + service: storage.googleapis.com + timeouts: null + module.service-accounts["service1/app-be-0"].google_service_account.service_account[0]: + account_id: app-be-0 + create_ignore_already_exists: null + description: null + disabled: false + display_name: Terraform-managed. + email: app-be-0@myprefix-service1.iam.gserviceaccount.com + member: serviceAccount:app-be-0@myprefix-service1.iam.gserviceaccount.com + project: myprefix-service1 + timeouts: null + module.service-accounts["service1/terraform-rw"].google_service_account.service_account[0]: + account_id: terraform-rw + create_ignore_already_exists: null + description: null + disabled: false + display_name: Terraform-managed. + email: terraform-rw@myprefix-service1.iam.gserviceaccount.com + member: serviceAccount:terraform-rw@myprefix-service1.iam.gserviceaccount.com + project: myprefix-service1 + timeouts: null + module.service-accounts["service2/app-be-0"].google_service_account.service_account[0]: + account_id: app-be-0 + create_ignore_already_exists: null + description: null + disabled: false + display_name: Terraform-managed. + email: app-be-0@myprefix-service2.iam.gserviceaccount.com + member: serviceAccount:app-be-0@myprefix-service2.iam.gserviceaccount.com + project: myprefix-service2 + timeouts: null + module.service-accounts["service2/terraform-rw"].google_service_account.service_account[0]: + account_id: terraform-rw + create_ignore_already_exists: null + description: null + disabled: false + display_name: Terraform-managed. + email: terraform-rw@myprefix-service2.iam.gserviceaccount.com + member: serviceAccount:terraform-rw@myprefix-service2.iam.gserviceaccount.com + project: myprefix-service2 + timeouts: null + +counts: + google_compute_shared_vpc_service_project: 1 + google_project: 2 + google_project_iam_member: 4 + google_project_service: 4 + google_service_account: 6 + google_storage_project_service_account: 1 + modules: 9 + resources: 18 + +outputs: + buckets: {} + folders: {} + foo: {} + projects: __missing__ + service_accounts: __missing__ diff --git a/tests/modules/project_factory/tftest.yaml b/tests/modules/project_factory/tftest.yaml index a33c93964..9960eda2b 100644 --- a/tests/modules/project_factory/tftest.yaml +++ b/tests/modules/project_factory/tftest.yaml @@ -21,3 +21,6 @@ tests: shared_vpc_network_user: extra_dirs: - ../../tests/modules/project_factory/data/shared_vpc_network_user/projects + data_overrides_defaults: + extra_dirs: + - ../../tests/modules/project_factory/data/data_overrides_defaults/projects From 4769dc3dd760b77304934797df2ac9a1795ac378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Wed, 2 Apr 2025 20:18:58 +0000 Subject: [PATCH 08/10] Allow null parent and prefix --- .../factory-projects-object.tf | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/modules/project-factory/factory-projects-object.tf b/modules/project-factory/factory-projects-object.tf index b9cc8c60a..01cd6e229 100644 --- a/modules/project-factory/factory-projects-object.tf +++ b/modules/project-factory/factory-projects-object.tf @@ -172,15 +172,19 @@ locals { ) name = lookup(v, "name", k) # type: string org_policies = try(v.org_policies, {}) # type: map(object({...})) - parent = coalesce( # type: string - local.__projects_config.data_overrides.parent, - try(v.parent, null), - local.__projects_config.data_defaults.parent + parent = try( # type: string, nullable + coalesce( + local.__projects_config.data_overrides.parent, + try(v.parent, null), + local.__projects_config.data_defaults.parent + ), null ) - prefix = coalesce( # type: string - local.__projects_config.data_overrides.prefix, - try(v.prefix, null), - local.__projects_config.data_defaults.prefix + prefix = try( # type: string, nullable + coalesce( + local.__projects_config.data_overrides.prefix, + try(v.prefix, null), + local.__projects_config.data_defaults.prefix + ), null ) service_encryption_key_ids = coalesce( # type: map(list(string)) local.__projects_config.data_overrides.service_encryption_key_ids, From d63a425b623de0cb7b031a0175d514b85d596ce5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Sat, 5 Apr 2025 14:30:34 +0000 Subject: [PATCH 09/10] Allow same filename in different directories As long, as they do override default project name using `name`. --- modules/project-factory/automation.tf | 2 +- modules/project-factory/factory-projects.tf | 21 +- tests/modules/project_factory/bucket_iam.yaml | 248 +++++++++++++++++- .../hierarchy/team-a/automation.yaml | 20 ++ .../bucket_iam/hierarchy/team-b/_config.yaml | 15 ++ .../hierarchy/team-b/automation.yaml | 20 ++ .../bucket_iam/hierarchy/team-b/project3.yaml | 20 ++ .../data/bucket_iam/projects/project3.yaml | 21 ++ .../data_overrides_defaults.yaml | 1 - 9 files changed, 348 insertions(+), 20 deletions(-) create mode 100644 tests/modules/project_factory/data/bucket_iam/hierarchy/team-a/automation.yaml create mode 100644 tests/modules/project_factory/data/bucket_iam/hierarchy/team-b/_config.yaml create mode 100644 tests/modules/project_factory/data/bucket_iam/hierarchy/team-b/automation.yaml create mode 100644 tests/modules/project_factory/data/bucket_iam/hierarchy/team-b/project3.yaml create mode 100644 tests/modules/project_factory/data/bucket_iam/projects/project3.yaml diff --git a/modules/project-factory/automation.tf b/modules/project-factory/automation.tf index 8ddbe2a07..18c9f45f1 100644 --- a/modules/project-factory/automation.tf +++ b/modules/project-factory/automation.tf @@ -35,7 +35,7 @@ locals { ] ]) } -output "foo" { value = local.automation_buckets } + module "automation-bucket" { source = "../gcs" for_each = local.automation_buckets diff --git a/modules/project-factory/factory-projects.tf b/modules/project-factory/factory-projects.tf index 6efcc9c32..aa7b891ae 100644 --- a/modules/project-factory/factory-projects.tf +++ b/modules/project-factory/factory-projects.tf @@ -17,10 +17,10 @@ # tfdoc:file:description Projects factory locals. locals { - _hierarchy_projects = ( + _hierarchy_projects_full_path = ( { for f in try(fileset(local._folders_path, "**/*.yaml"), []) : - basename(trimsuffix(f, ".yaml")) => merge( + trimsuffix(f, ".yaml") => merge( { parent = dirname(f) == "." ? "default" : dirname(f) }, yamldecode(file("${local._folders_path}/${f}")) ) @@ -28,13 +28,16 @@ locals { } ) _project_path = try(pathexpand(var.factories_config.projects_data_path), null) - _projects_input = merge( - { - for f in try(fileset(local._project_path, "**/*.yaml"), []) : - basename(trimsuffix(f, ".yaml")) => yamldecode(file("${local._project_path}/${f}")) - }, - local._hierarchy_projects - ) + _projects_full_path = { + for f in try(fileset(local._project_path, "**/*.yaml"), []) : + trimsuffix(f, ".yaml") => yamldecode(file("${local._project_path}/${f}")) + } + _projects_input = { + # will raise error, if the same filename is raised multiple times + # and project name is not set via name in YAML + for k, v in merge(local._hierarchy_projects_full_path, local._projects_full_path) : + lookup(v, "name", basename(k)) => v + } _project_budgets = flatten([ for k, v in local._projects_input : [ for b in try(v.billing_budgets, []) : { diff --git a/tests/modules/project_factory/bucket_iam.yaml b/tests/modules/project_factory/bucket_iam.yaml index fc7bf9f95..c321b3d62 100644 --- a/tests/modules/project_factory/bucket_iam.yaml +++ b/tests/modules/project_factory/bucket_iam.yaml @@ -83,6 +83,124 @@ values: - group:gcp-devops@example.org - group:team-a-admins@example.org role: roles/viewer + module.hierarchy-folder-lvl-1["team-b"].google_folder.folder[0]: + deletion_protection: false + display_name: Team B + parent: folders/5678901234 + tags: null + timeouts: null + module.projects["auto-team-a"].data.google_storage_project_service_account.gcs_sa[0]: + project: test-pf-auto-team-a + user_project: null + module.projects["auto-team-a"].google_essential_contacts_contact.contact["admin@example.org"]: + email: admin@example.org + language_tag: en + notification_category_subscriptions: + - ALL + parent: projects/test-pf-auto-team-a + timeouts: null + module.projects["auto-team-a"].google_project.project[0]: + auto_create_network: false + billing_account: 012345-67890A-BCDEF0 + deletion_policy: DELETE + effective_labels: + environment: test + goog-terraform-provisioned: 'true' + labels: + environment: test + name: test-pf-auto-team-a + project_id: test-pf-auto-team-a + tags: null + terraform_labels: + environment: test + goog-terraform-provisioned: 'true' + timeouts: null + module.projects["auto-team-a"].google_project_iam_member.service_agents["container-engine-robot"]: + condition: [] + project: test-pf-auto-team-a + role: roles/container.serviceAgent + module.projects["auto-team-a"].google_project_iam_member.service_agents["gkenode"]: + condition: [] + project: test-pf-auto-team-a + role: roles/container.defaultNodeServiceAgent + module.projects["auto-team-a"].google_project_service.project_services["container.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: test-pf-auto-team-a + service: container.googleapis.com + timeouts: null + module.projects["auto-team-a"].google_project_service.project_services["stackdriver.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: test-pf-auto-team-a + service: stackdriver.googleapis.com + timeouts: null + module.projects["auto-team-a"].google_project_service.project_services["storage.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: test-pf-auto-team-a + service: storage.googleapis.com + timeouts: null + module.projects["auto-team-a"].google_project_service_identity.default["container.googleapis.com"]: + project: test-pf-auto-team-a + service: container.googleapis.com + timeouts: null + module.projects["auto-team-b"].data.google_storage_project_service_account.gcs_sa[0]: + project: test-pf-auto-team-b + user_project: null + module.projects["auto-team-b"].google_essential_contacts_contact.contact["admin@example.org"]: + email: admin@example.org + language_tag: en + notification_category_subscriptions: + - ALL + parent: projects/test-pf-auto-team-b + timeouts: null + module.projects["auto-team-b"].google_project.project[0]: + auto_create_network: false + billing_account: 012345-67890A-BCDEF0 + deletion_policy: DELETE + effective_labels: + environment: test + goog-terraform-provisioned: 'true' + labels: + environment: test + name: test-pf-auto-team-b + project_id: test-pf-auto-team-b + tags: null + terraform_labels: + environment: test + goog-terraform-provisioned: 'true' + timeouts: null + module.projects["auto-team-b"].google_project_iam_member.service_agents["container-engine-robot"]: + condition: [] + project: test-pf-auto-team-b + role: roles/container.serviceAgent + module.projects["auto-team-b"].google_project_iam_member.service_agents["gkenode"]: + condition: [] + project: test-pf-auto-team-b + role: roles/container.defaultNodeServiceAgent + module.projects["auto-team-b"].google_project_service.project_services["container.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: test-pf-auto-team-b + service: container.googleapis.com + timeouts: null + module.projects["auto-team-b"].google_project_service.project_services["stackdriver.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: test-pf-auto-team-b + service: stackdriver.googleapis.com + timeouts: null + module.projects["auto-team-b"].google_project_service.project_services["storage.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: test-pf-auto-team-b + service: storage.googleapis.com + timeouts: null + module.projects["auto-team-b"].google_project_service_identity.default["container.googleapis.com"]: + project: test-pf-auto-team-b + service: container.googleapis.com + timeouts: null module.projects["project1"].data.google_storage_project_service_account.gcs_sa[0]: project: test-pf-project1 user_project: null @@ -174,6 +292,118 @@ values: project: test-pf-project2 service: stackdriver.googleapis.com timeouts: null + module.projects["project3"].data.google_storage_project_service_account.gcs_sa[0]: + project: test-pf-project3 + user_project: null + module.projects["project3"].google_essential_contacts_contact.contact["admin@example.org"]: + email: admin@example.org + language_tag: en + notification_category_subscriptions: + - ALL + parent: projects/test-pf-project3 + timeouts: null + module.projects["project3"].google_project.project[0]: + auto_create_network: false + billing_account: 012345-67890A-BCDEF0 + deletion_policy: DELETE + effective_labels: + environment: test + goog-terraform-provisioned: 'true' + labels: + environment: test + name: test-pf-project3 + project_id: test-pf-project3 + tags: null + terraform_labels: + environment: test + goog-terraform-provisioned: 'true' + timeouts: null + module.projects["project3"].google_project_iam_member.service_agents["container-engine-robot"]: + condition: [] + project: test-pf-project3 + role: roles/container.serviceAgent + module.projects["project3"].google_project_iam_member.service_agents["gkenode"]: + condition: [] + project: test-pf-project3 + role: roles/container.defaultNodeServiceAgent + module.projects["project3"].google_project_service.project_services["container.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: test-pf-project3 + service: container.googleapis.com + timeouts: null + module.projects["project3"].google_project_service.project_services["stackdriver.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: test-pf-project3 + service: stackdriver.googleapis.com + timeouts: null + module.projects["project3"].google_project_service.project_services["storage.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: test-pf-project3 + service: storage.googleapis.com + timeouts: null + module.projects["project3"].google_project_service_identity.default["container.googleapis.com"]: + project: test-pf-project3 + service: container.googleapis.com + timeouts: null + module.projects["top-project3"].data.google_storage_project_service_account.gcs_sa[0]: + project: test-pf-top-project3 + user_project: null + module.projects["top-project3"].google_essential_contacts_contact.contact["admin@example.org"]: + email: admin@example.org + language_tag: en + notification_category_subscriptions: + - ALL + parent: projects/test-pf-top-project3 + timeouts: null + module.projects["top-project3"].google_project.project[0]: + auto_create_network: false + billing_account: 012345-67890A-BCDEF0 + deletion_policy: DELETE + effective_labels: + environment: test + goog-terraform-provisioned: 'true' + labels: + environment: test + name: test-pf-top-project3 + project_id: test-pf-top-project3 + tags: null + terraform_labels: + environment: test + goog-terraform-provisioned: 'true' + timeouts: null + module.projects["top-project3"].google_project_iam_member.service_agents["container-engine-robot"]: + condition: [] + project: test-pf-top-project3 + role: roles/container.serviceAgent + module.projects["top-project3"].google_project_iam_member.service_agents["gkenode"]: + condition: [] + project: test-pf-top-project3 + role: roles/container.defaultNodeServiceAgent + module.projects["top-project3"].google_project_service.project_services["container.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: test-pf-top-project3 + service: container.googleapis.com + timeouts: null + module.projects["top-project3"].google_project_service.project_services["stackdriver.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: test-pf-top-project3 + service: stackdriver.googleapis.com + timeouts: null + module.projects["top-project3"].google_project_service.project_services["storage.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: test-pf-top-project3 + service: storage.googleapis.com + timeouts: null + module.projects["top-project3"].google_project_service_identity.default["container.googleapis.com"]: + project: test-pf-top-project3 + service: container.googleapis.com + timeouts: null module.service-accounts["project1/app-be-0"].google_service_account.service_account[0]: account_id: app-be-0 create_ignore_already_exists: null @@ -252,19 +482,19 @@ values: timeouts: null counts: - google_essential_contacts_contact: 2 - google_folder: 1 + google_essential_contacts_contact: 6 + google_folder: 2 google_folder_iam_binding: 1 - google_project: 2 - google_project_iam_member: 6 - google_project_service: 4 - google_project_service_identity: 1 + google_project: 6 + google_project_iam_member: 14 + google_project_service: 16 + google_project_service_identity: 5 google_service_account: 6 google_storage_bucket: 2 google_storage_bucket_iam_binding: 1 - google_storage_project_service_account: 1 - modules: 11 - resources: 27 + google_storage_project_service_account: 5 + modules: 16 + resources: 64 outputs: buckets: diff --git a/tests/modules/project_factory/data/bucket_iam/hierarchy/team-a/automation.yaml b/tests/modules/project_factory/data/bucket_iam/hierarchy/team-a/automation.yaml new file mode 100644 index 000000000..0a744e97f --- /dev/null +++ b/tests/modules/project_factory/data/bucket_iam/hierarchy/team-a/automation.yaml @@ -0,0 +1,20 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +billing_account: 012345-67890A-BCDEF0 +services: + - container.googleapis.com + - storage.googleapis.com + +name: auto-team-a diff --git a/tests/modules/project_factory/data/bucket_iam/hierarchy/team-b/_config.yaml b/tests/modules/project_factory/data/bucket_iam/hierarchy/team-b/_config.yaml new file mode 100644 index 000000000..fbdc4437e --- /dev/null +++ b/tests/modules/project_factory/data/bucket_iam/hierarchy/team-b/_config.yaml @@ -0,0 +1,15 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Team B diff --git a/tests/modules/project_factory/data/bucket_iam/hierarchy/team-b/automation.yaml b/tests/modules/project_factory/data/bucket_iam/hierarchy/team-b/automation.yaml new file mode 100644 index 000000000..58a698cd7 --- /dev/null +++ b/tests/modules/project_factory/data/bucket_iam/hierarchy/team-b/automation.yaml @@ -0,0 +1,20 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +billing_account: 012345-67890A-BCDEF0 +services: + - container.googleapis.com + - storage.googleapis.com + +name: auto-team-b diff --git a/tests/modules/project_factory/data/bucket_iam/hierarchy/team-b/project3.yaml b/tests/modules/project_factory/data/bucket_iam/hierarchy/team-b/project3.yaml new file mode 100644 index 000000000..c953163bd --- /dev/null +++ b/tests/modules/project_factory/data/bucket_iam/hierarchy/team-b/project3.yaml @@ -0,0 +1,20 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +billing_account: 012345-67890A-BCDEF0 +services: + - container.googleapis.com + - storage.googleapis.com + +prefix: team-b diff --git a/tests/modules/project_factory/data/bucket_iam/projects/project3.yaml b/tests/modules/project_factory/data/bucket_iam/projects/project3.yaml new file mode 100644 index 000000000..e6b2fdd4c --- /dev/null +++ b/tests/modules/project_factory/data/bucket_iam/projects/project3.yaml @@ -0,0 +1,21 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +billing_account: 012345-67890A-BCDEF0 +services: + - container.googleapis.com + - storage.googleapis.com + +name: top-project3 +parent: team-b diff --git a/tests/modules/project_factory/data_overrides_defaults.yaml b/tests/modules/project_factory/data_overrides_defaults.yaml index 8afed2dc6..9250d9189 100644 --- a/tests/modules/project_factory/data_overrides_defaults.yaml +++ b/tests/modules/project_factory/data_overrides_defaults.yaml @@ -59,6 +59,5 @@ counts: outputs: buckets: {} folders: {} - foo: {} projects: __missing__ service_accounts: {} From 6d2173aada77b79419a52cfb862090552a5c58a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Sun, 6 Apr 2025 13:48:16 +0000 Subject: [PATCH 10/10] Ensure vpc_sc has correct type for overrides --- modules/project-factory/factory-projects-object.tf | 14 ++++++++++++-- modules/project-factory/factory-projects.tf | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/modules/project-factory/factory-projects-object.tf b/modules/project-factory/factory-projects-object.tf index 01cd6e229..3e9c2d714 100644 --- a/modules/project-factory/factory-projects-object.tf +++ b/modules/project-factory/factory-projects-object.tf @@ -101,8 +101,18 @@ locals { tag_bindings = null services = null service_accounts = null - vpc_sc = try(local._projects_config.data_overrides.vpc_sc, null) - logging_data_access = null + vpc_sc = try( + merge( + { + perimeter_name = null + perimeter_bridges = [] + is_dry_run = false + }, + local._projects_config.data_overrides.vpc_sc + ), + null + ) + logging_data_access = null }, try(local._projects_config.data_overrides, {}) ) diff --git a/modules/project-factory/factory-projects.tf b/modules/project-factory/factory-projects.tf index aa7b891ae..b104acb41 100644 --- a/modules/project-factory/factory-projects.tf +++ b/modules/project-factory/factory-projects.tf @@ -33,7 +33,7 @@ locals { trimsuffix(f, ".yaml") => yamldecode(file("${local._project_path}/${f}")) } _projects_input = { - # will raise error, if the same filename is raised multiple times + # will raise error, if the same filename is used multiple times # and project name is not set via name in YAML for k, v in merge(local._hierarchy_projects_full_path, local._projects_full_path) : lookup(v, "name", basename(k)) => v