From 625a6b7c278c7ad8d1d06e633c24c5388e53194d Mon Sep 17 00:00:00 2001 From: Luca Prete Date: Wed, 16 Jul 2025 18:18:20 +0200 Subject: [PATCH 1/4] Add force destroy option to buckets in project factory module (#3238) --- modules/project-factory/README.md | 10 +++++----- modules/project-factory/automation.tf | 6 +++++- modules/project-factory/factory-projects.tf | 16 ++++++++++------ modules/project-factory/variables.tf | 8 ++++++++ 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/modules/project-factory/README.md b/modules/project-factory/README.md index 6faa885aa..1729b9a71 100644 --- a/modules/project-factory/README.md +++ b/modules/project-factory/README.md @@ -526,11 +526,11 @@ service_accounts: | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [factories_config](variables.tf#L140) | Path to folder with YAML resource description data files. | object({…}) | ✓ | | -| [data_defaults](variables.tf#L17) | Optional default values used when corresponding project data from files are missing. | object({…}) | | {} | -| [data_merges](variables.tf#L82) | Optional values that will be merged with corresponding data from files. Combines with `data_defaults`, file data, and `data_overrides`. | object({…}) | | {} | -| [data_overrides](variables.tf#L101) | Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`. | object({…}) | | {} | -| [factories_data](variables.tf#L168) | Alternate factory data input allowing to use this module as a library. Merged with local YAML data. | object({…}) | | {} | +| [factories_config](variables.tf#L146) | Path to folder with YAML resource description data files. | object({…}) | ✓ | | +| [data_defaults](variables.tf#L17) | Optional default values used when corresponding project data from files are missing. | object({…}) | | {} | +| [data_merges](variables.tf#L85) | Optional values that will be merged with corresponding data from files. Combines with `data_defaults`, file data, and `data_overrides`. | object({…}) | | {} | +| [data_overrides](variables.tf#L104) | Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`. | object({…}) | | {} | +| [factories_data](variables.tf#L174) | Alternate factory data input allowing to use this module as a library. Merged with local YAML data. | object({…}) | | {} | ## Outputs diff --git a/modules/project-factory/automation.tf b/modules/project-factory/automation.tf index e03d850ac..856587cfe 100644 --- a/modules/project-factory/automation.tf +++ b/modules/project-factory/automation.tf @@ -53,7 +53,11 @@ module "automation-bucket" { prefix = each.value.prefix name = "tf-state" encryption_key = lookup(each.value, "encryption_key", null) - force_destroy = lookup(each.value, "force_destroy", null) + force_destroy = try(coalesce( + var.data_overrides.bucket.force_destroy, + each.value.force_destroy, + var.data_defaults.bucket.force_destroy, + ), null) iam = { for k, v in lookup(each.value, "iam", {}) : k => [ for vv in v : try( diff --git a/modules/project-factory/factory-projects.tf b/modules/project-factory/factory-projects.tf index 0df5b260a..0d0d595ce 100644 --- a/modules/project-factory/factory-projects.tf +++ b/modules/project-factory/factory-projects.tf @@ -68,12 +68,16 @@ locals { buckets = flatten([ for k, v in local.projects : [ for name, opts in v.buckets : { - project_key = k - project_name = v.name - name = name - description = lookup(opts, "description", "Terraform-managed.") - encryption_key = lookup(opts, "encryption_key", null) - force_destroy = lookup(opts, "force_destroy", null) + project_key = k + project_name = v.name + name = name + description = lookup(opts, "description", "Terraform-managed.") + encryption_key = lookup(opts, "encryption_key", null) + force_destroy = try(coalesce( + var.data_overrides.bucket.force_destroy, + try(opts.force_destroy, null), + var.data_defaults.bucket.force_destroy, + ), null) iam = lookup(opts, "iam", {}) iam_bindings = lookup(opts, "iam_bindings", {}) iam_bindings_additive = lookup(opts, "iam_bindings_additive", {}) diff --git a/modules/project-factory/variables.tf b/modules/project-factory/variables.tf index cceabcfb9..6e9ce7e30 100644 --- a/modules/project-factory/variables.tf +++ b/modules/project-factory/variables.tf @@ -18,6 +18,9 @@ variable "data_defaults" { description = "Optional default values used when corresponding project data from files are missing." type = object({ billing_account = optional(string) + bucket = optional(object({ + force_destroy = optional(bool) + }), {}) contacts = optional(map(list(string)), {}) deletion_policy = optional(string) factories_config = optional(object({ @@ -103,6 +106,9 @@ variable "data_overrides" { type = object({ # data overrides default to null to mark that they should not override billing_account = optional(string) + bucket = optional(object({ + force_destroy = optional(bool) + }), {}) contacts = optional(map(list(string))) deletion_policy = optional(string) factories_config = optional(object({ @@ -246,6 +252,7 @@ variable "factories_data" { bucket = optional(object({ location = string description = optional(string) + force_destroy = optional(bool) prefix = optional(string) storage_class = optional(string, "STANDARD") uniform_bucket_level_access = optional(bool, true) @@ -305,6 +312,7 @@ variable "factories_data" { buckets = optional(map(object({ location = string description = optional(string) + force_destroy = optional(bool) prefix = optional(string) storage_class = optional(string, "STANDARD") uniform_bucket_level_access = optional(bool, true) From 45b30a15810f3817bcbebfb5cf0e1c224bd67e7e Mon Sep 17 00:00:00 2001 From: la-luce Date: Thu, 17 Jul 2025 04:32:24 -0400 Subject: [PATCH 2/4] La luce/net firewall policy doc update (#3232) * update README.md: explain dynamic fieldname mapping for firewall rules * Update README.md: add firewall rule factory schema * Update README.md to flag undocumented implicit defaults * shorting line length I think I failed a linting check because a line was too long / would cause readability issues * fix tfdoc mark * update TOC --------- Co-authored-by: Ludovico Magnocavallo --- modules/net-firewall-policy/README.md | 70 +++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/modules/net-firewall-policy/README.md b/modules/net-firewall-policy/README.md index ac02f48b7..9f76a1013 100644 --- a/modules/net-firewall-policy/README.md +++ b/modules/net-firewall-policy/README.md @@ -15,6 +15,11 @@ The module also makes fewer assumptions about implicit defaults, only using one - [Global Network policy](#global-network-policy) - [Regional Network policy](#regional-network-policy) - [Factory](#factory) + - [Firewall Rule Factory Schema](#firewall-rule-factory-schema) + - [Dynamic Rule Matching](#dynamic-rule-matching) + - [Ingress Rules (](#ingress-rules-) + - [Egress Rules (](#egress-rules-) + - [Rule-Level Mappings](#rule-level-mappings) - [Variables](#variables) - [Outputs](#outputs) @@ -174,6 +179,7 @@ Factory configuration is via three optional attributes in the `rules_factory_con - `ingress_rules_file_path` specifying the path to the ingress rules file Factory rules are merged with rules declared in code, with the latter taking precedence where both use the same key. +Also, the factory applies implicit defaults: `action` defaults to `deny` for egress and `allow` for ingress, while omitting `layer4_configs` makes the rule match all protocols. This is an example of a simple factory: @@ -309,7 +315,71 @@ http: ports: - 80 ``` + +#### Firewall Rule Factory Schema + +The following schema outlines all available fields for defining a rule within a factory YAML file. Use this as a reference, and note the inline comments for fields that apply only to specific policy types. + +```yaml +rule-name: + priority: + action: + description: + disabled: + enable_logging: + security_profile_group: # Not for Regional policies + target_service_accounts: [] + target_tags: [] # Not for Hierarchical policies + target_resources: [] # For Hierarchical policies only + tls_inspect: # Not for Regional policies + match: + source_ranges: [] + destination_ranges: [] + source_tags: [] # Not for Hierarchical policies + threat_intelligences: [] + fqdns: [] + address_groups: [] + region_codes: [] + layer4_configs: + - protocol: + ports: [] +``` + +### Dynamic Rule Matching + +This module simplifies firewall rule creation by using generic, context-aware variables within the `match` block. Based on the rule's specified `direction` (`INGRESS` or `EGRESS`), the module maps these generic variables to the correct source- (`src_*`) or destination-specific (`dest_*`) arguments in the underlying resource. + +The tables below provide a complete reference for these dynamic mappings. + +#### Ingress Rules (`direction = "INGRESS"`) + +| Module Variable (`match.*`) | Mapped Resource Attribute | +| :--- | :--- | +| `address_groups` | `src_address_groups` | +| `fqdns` | `src_fqdns` | +| `region_codes` | `src_region_codes` | +| `source_tags` | `src_secure_tags` | +| `threat_intelligences` | `src_threat_intelligences` | + +#### Egress Rules (`direction = "EGRESS"`) + +| Module Variable (`match.*`) | Mapped Resource Attribute | +| :--- | :--- | +| `address_groups` | `dest_address_groups` | +| `fqdns` | `dest_fqdns` | +| `region_codes` | `dest_region_codes` | +| `threat_intelligences` | `dest_threat_intelligences` | + +#### Rule-Level Mappings + +The following variable is defined at the top level of the rule (not within the `match` block) and is mapped directly, regardless of the rule's direction. + +| Module Variable | Mapped Resource Attribute | +| :--- | :--- | +| `target_tags` | `target_secure_tags` | + + ## Variables | name | description | type | required | default | From 3b2c95b80afb164c51c1258e3ad53d3d06bc7e90 Mon Sep 17 00:00:00 2001 From: eeila <87441835+eeila@users.noreply.github.com> Date: Thu, 17 Jul 2025 08:27:07 -0400 Subject: [PATCH 3/4] Add support for cloudsql regional replicas (#3239) * [feat] - adding support for cloudsql regional replicas * Remove validation already done by the provider. --------- Co-authored-by: Julio Castillo --- modules/cloudsql-instance/README.md | 14 +++++++------- modules/cloudsql-instance/main.tf | 12 ++++++------ modules/cloudsql-instance/variables.tf | 5 +++-- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/modules/cloudsql-instance/README.md b/modules/cloudsql-instance/README.md index ab66fc2c1..8b2e9ffc7 100644 --- a/modules/cloudsql-instance/README.md +++ b/modules/cloudsql-instance/README.md @@ -413,7 +413,7 @@ module "db" { | [network_config](variables.tf#L184) | Network configuration for the instance. Only one between private_network and psc_config can be used. | object({…}) | ✓ | | | [project_id](variables.tf#L231) | The ID of the project where this instances will be created. | string | ✓ | | | [region](variables.tf#L236) | Region of the primary instance. | string | ✓ | | -| [tier](variables.tf#L287) | The machine type to use for the instances. | string | ✓ | | +| [tier](variables.tf#L288) | The machine type to use for the instances. | string | ✓ | | | [activation_policy](variables.tf#L16) | This variable specifies when the instance should be active. Can be either ALWAYS, NEVER or ON_DEMAND. Default is ALWAYS. | string | | "ALWAYS" | | [availability_type](variables.tf#L27) | Availability type for the primary replica. Either `ZONAL` or `REGIONAL`. | string | | "ZONAL" | | [backup_configuration](variables.tf#L33) | Backup settings for primary instance. Will be automatically enabled if using MySQL with one or more replicas. | object({…}) | | {…} | @@ -433,12 +433,12 @@ module "db" { | [maintenance_config](variables.tf#L146) | Set maintenance window configuration and maintenance deny period (up to 90 days). Date format: 'yyyy-mm-dd'. | object({…}) | | {} | | [password_validation_policy](variables.tf#L207) | Password validation policy configuration for instances. | object({…}) | | null | | [prefix](variables.tf#L221) | Optional prefix used to generate instance names. | string | | null | -| [replicas](variables.tf#L241) | Map of NAME=> {REGION, KMS_KEY} for additional read replicas. Set to null to disable replica creation. | map(object({…})) | | {} | -| [root_password](variables.tf#L251) | Root password of the Cloud SQL instance, or flag to create a random password. Required for MS SQL Server. | object({…}) | | {} | -| [ssl](variables.tf#L265) | Setting to enable SSL, set config and certificates. | object({…}) | | {} | -| [terraform_deletion_protection](variables.tf#L280) | Prevent terraform from deleting instances. | bool | | true | -| [time_zone](variables.tf#L292) | The time_zone to be used by the database engine (supported only for SQL Server), in SQL Server timezone format. | string | | null | -| [users](variables.tf#L298) | Map of users to create in the primary instance (and replicated to other replicas). For MySQL, anything after the first `@` (if present) will be used as the user's host. Set PASSWORD to null if you want to get an autogenerated password. The user types available are: 'BUILT_IN', 'CLOUD_IAM_USER' or 'CLOUD_IAM_SERVICE_ACCOUNT'. | map(object({…})) | | {} | +| [replicas](variables.tf#L241) | Map of NAME=> {REGION, KMS_KEY, AVAILABILITY_TYPE} for additional read replicas. Set to null to disable replica creation. | map(object({…})) | | {} | +| [root_password](variables.tf#L252) | Root password of the Cloud SQL instance, or flag to create a random password. Required for MS SQL Server. | object({…}) | | {} | +| [ssl](variables.tf#L266) | Setting to enable SSL, set config and certificates. | object({…}) | | {} | +| [terraform_deletion_protection](variables.tf#L281) | Prevent terraform from deleting instances. | bool | | true | +| [time_zone](variables.tf#L293) | The time_zone to be used by the database engine (supported only for SQL Server), in SQL Server timezone format. | string | | null | +| [users](variables.tf#L299) | Map of users to create in the primary instance (and replicated to other replicas). For MySQL, anything after the first `@` (if present) will be used as the user's host. Set PASSWORD to null if you want to get an autogenerated password. The user types available are: 'BUILT_IN', 'CLOUD_IAM_USER' or 'CLOUD_IAM_SERVICE_ACCOUNT'. | map(object({…})) | | {} | ## Outputs diff --git a/modules/cloudsql-instance/main.tf b/modules/cloudsql-instance/main.tf index cafe9b56e..663d1172e 100644 --- a/modules/cloudsql-instance/main.tf +++ b/modules/cloudsql-instance/main.tf @@ -215,12 +215,12 @@ resource "google_sql_database_instance" "replicas" { disk_autoresize_limit = var.disk_autoresize_limit disk_size = var.disk_size disk_type = var.disk_type - # availability_type = var.availability_type - user_labels = var.labels - activation_policy = var.activation_policy - collation = var.collation - connector_enforcement = var.connector_enforcement - time_zone = var.time_zone + availability_type = each.value.availability_type + user_labels = var.labels + activation_policy = var.activation_policy + collation = var.collation + connector_enforcement = var.connector_enforcement + time_zone = var.time_zone ip_configuration { diff --git a/modules/cloudsql-instance/variables.tf b/modules/cloudsql-instance/variables.tf index 72a4fa5da..b566426d9 100644 --- a/modules/cloudsql-instance/variables.tf +++ b/modules/cloudsql-instance/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -239,10 +239,11 @@ variable "region" { } variable "replicas" { - description = "Map of NAME=> {REGION, KMS_KEY} for additional read replicas. Set to null to disable replica creation." + description = "Map of NAME=> {REGION, KMS_KEY, AVAILABILITY_TYPE} for additional read replicas. Set to null to disable replica creation." type = map(object({ region = string encryption_key_name = optional(string) + availability_type = optional(string) })) default = {} nullable = false From 8ce41711626f11eae94eaed6008ec3c2c6d3a652 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 17 Jul 2025 14:54:17 +0200 Subject: [PATCH 4/4] Fix #3240 (#3241) --- modules/project/cmek.tf | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/modules/project/cmek.tf b/modules/project/cmek.tf index 0d369d2c4..50b954aec 100644 --- a/modules/project/cmek.tf +++ b/modules/project/cmek.tf @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ locals { "artifactregistry.googleapis.com" : ["artifactregistry"] "bigtableadmin.googleapis.com" : ["bigtable"] "bigquery.googleapis.com" : ["bigquery-encryption"] - # the list for composer now track composer 3 + # the list for composer now tracks composer 3 # https://cloud.google.com/composer/docs/composer-3/configure-cmek-encryption#grant-roles-permissions "composer.googleapis.com" : ["composer", "storage"] "compute.googleapis.com" : ["compute"] @@ -49,20 +49,28 @@ locals { "storage.googleapis.com" : ["storage"] "run.googleapis.com" : ["cloudrun"] } - _cmek_members = merge(flatten([ + _all_cmek_bindings = flatten([ for service, keys in var.service_encryption_key_ids : [ - # use the deps listed above, if the service does not appear - # there, use all the service agents belonging to the service - for dep in try(local._cmek_agents_by_service[service], [for x in local._service_agents_by_api[service] : x.name], [service]) : { - # use index in map key, to allow specifying keys, that will be created in the same apply - for index, key in keys : - "key-${index}.${local._aliased_service_agents[dep].name}" => { - key = key - agent = local._aliased_service_agents[dep].iam_email + for dep in try(local._cmek_agents_by_service[service], [for x in local._service_agents_by_api[service] : x.name], [service]) : [ + for key in keys : { + key_id = key + agent_name = local._aliased_service_agents[dep].name + agent_email = local._aliased_service_agents[dep].iam_email } - } + ] ] - ])...) + ]) + _cmek_bindings_grouped_by_agent = { + for binding in local._all_cmek_bindings : binding.agent_name => binding... + } + _cmek_members = merge([ + for agent_name, bindings in local._cmek_bindings_grouped_by_agent : { + for i, binding in bindings : "key-${i}.${agent_name}" => { + key = binding.key_id + agent = binding.agent_email + } + } + ]...) } resource "google_kms_crypto_key_iam_member" "service_agent_cmek" {