New project factory improvements (#3303)

* Add separate prefix for automation resources in pf * fix example * add automation to pf outputs
2025-09-06 10:01:36 +02:00
parent d6f0a168f6
commit 86d50ffb62
6 changed files with 127 additions and 39 deletions
--- a/modules/project-factory/README.md
+++ b/modules/project-factory/README.md
@@ -224,6 +224,7 @@ Assuming keys of the form `my_folder`, `my_project`, `my_sa`, etc. this is an ex
 - `$notification_channels:my_channel`
 - `$project_ids:my_project`
 - `$service_account_ids:my_project/my_sa`
+- `$service_account_ids:my_project/automation/my_sa`
 - `$service_agents:compute`
 - `$tag_values:my_value`
 - `$vpc_host_projects:my_project`
@@ -256,6 +257,21 @@ iam_by_principals:
    - roles/viewer
 ```

+Service accounts defined in the `automation` block will have an `automation` prefix prepended to their context id.
+
+```yaml
+automation:
+  project: $project_ids:prod-iac-core-0
+  bucket:
+    name: tf-state
+  service_accounts:
+    ro: {}
+    rw:
+      iam_sa_roles:
+        $service_account_ids:dev-app0-be-0/automation/ro:
+          - roles.iam.serviceAccountTokenCreator
+```
+
 The only exception is when setting IAM binding for a service account on a different service account via the `iam_sa_roles` attribute, which interpolates using the `$service_account_ids` namespace. As an example, granting a role to the `rw` service account above on the `ro` service account in the same project will use `$service_account_ids:app-0-0/ro`.

 ```yaml
@@ -497,9 +513,9 @@ services:
 - storage.googleapis.com
 iam:
  "roles/owner":
-    - $iam_principals:service_accounts/dev-tb-app0-0/rw
+    - $iam_principals:service_accounts/dev-tb-app0-0/automation/rw
  "roles/viewer":
-    - $iam_principals:service_accounts/dev-tb-app0-0/ro
+    - $iam_principals:service_accounts/dev-tb-app0-0/automation/ro
 shared_vpc_host_config:
  enabled: true
 service_accounts:
@@ -510,7 +526,7 @@ service_accounts:
      - roles/monitoring.metricWriter
    iam:
      roles/iam.serviceAccountTokenCreator:
-        - $iam_principals:service_accounts/dev-tb-app0-0/rw
+        - $iam_principals:service_accounts/dev-tb-app0-0/automation/rw
 automation:
  project: test-pf-teams-iac-0
  # prefix used for automation resources can be explicitly set if needed
@@ -524,12 +540,12 @@ automation:
    description: Team B app 0 Terraform state bucket.
    iam:
      roles/storage.objectCreator:
-        - $iam_principals:service_accounts/dev-tb-app0-0/rw
+        - $iam_principals:service_accounts/dev-tb-app0-0/automation/rw
      roles/storage.objectViewer:
        - $iam_principals:gcp-devops
        - group:team-b-admins@example.org
-        - $iam_principals:service_accounts/dev-tb-app0-0/rw
-        - $iam_principals:service_accounts/dev-tb-app0-0/ro
+        - $iam_principals:service_accounts/dev-tb-app0-0/automation/rw
+        - $iam_principals:service_accounts/dev-tb-app0-0/automation/ro

 # tftest-file id=7 path=data/projects/dev-tb-app0-0.yaml schema=project.schema.json
 ```
@@ -620,17 +636,17 @@ service_accounts:

 | name | description | sensitive |
 |---|---|:---:|
-| [folder_ids](outputs.tf#L49) | Folder ids. |  |
-| [iam_principals](outputs.tf#L54) | IAM principals mappings. |  |
-| [log_buckets](outputs.tf#L59) | Log bucket ids. |  |
-| [project_ids](outputs.tf#L66) | Project ids. |  |
-| [project_numbers](outputs.tf#L71) | Project numbers. |  |
-| [projects](outputs.tf#L78) | Project attributes. |  |
-| [service_account_emails](outputs.tf#L83) | Service account emails. |  |
-| [service_account_iam_emails](outputs.tf#L90) | Service account IAM-format emails. |  |
-| [service_account_ids](outputs.tf#L97) | Service account IDs. |  |
-| [service_accounts](outputs.tf#L104) | Service account emails. |  |
-| [storage_buckets](outputs.tf#L109) | Bucket names. |  |
+| [folder_ids](outputs.tf#L78) | Folder ids. |  |
+| [iam_principals](outputs.tf#L83) | IAM principals mappings. |  |
+| [log_buckets](outputs.tf#L88) | Log bucket ids. |  |
+| [project_ids](outputs.tf#L95) | Project ids. |  |
+| [project_numbers](outputs.tf#L100) | Project numbers. |  |
+| [projects](outputs.tf#L107) | Project attributes. |  |
+| [service_account_emails](outputs.tf#L112) | Service account emails. |  |
+| [service_account_iam_emails](outputs.tf#L119) | Service account IAM-format emails. |  |
+| [service_account_ids](outputs.tf#L126) | Service account IDs. |  |
+| [service_accounts](outputs.tf#L133) | Service account emails. |  |
+| [storage_buckets](outputs.tf#L138) | Bucket names. |  |
 <!-- END TFDOC -->
 ## Tests

--- a/modules/project-factory/automation.tf
+++ b/modules/project-factory/automation.tf
@@ -43,6 +43,7 @@ locals {
  _automation_buckets = {
    for k, v in local._automation : k => merge(v.bucket, {
      automation_project = v.project
+      source_project     = k
      name               = lookup(v, "name", "tf-state")
      # project automation always has a prefix
      prefix = try(coalesce(
@@ -56,6 +57,7 @@ locals {
    for k, v in local._automation : [
      for sk, sv in v.service_accounts : merge(sv, {
        automation_project = v.project
+        source_project     = k
        name               = sk
        parent             = k
        prefix             = v.prefix
@@ -64,15 +66,15 @@ locals {
  ]))
  automation_buckets = {
    for k, v in local._automation_buckets :
-    "${k}/${v.name}" => v
+    "${k}/automation/${v.name}" => v
  }
  automation_sas = {
    for k in local._automation_sas :
-    "${k.parent}/${k.name}" => k
+    "${k.parent}/automation/${k.name}" => k
  }
  automation_sas_iam_emails = {
    for k, v in local.automation_sas :
-    "service_accounts/${v.parent}/${v.name}" => module.automation-service-accounts[k].iam_email
+    "service_accounts/${v.parent}/automation/${v.name}" => module.automation-service-accounts[k].iam_email
  }
 }

@@ -139,7 +141,24 @@ module "automation-service-accounts" {
  iam_folder_roles       = lookup(each.value, "iam_folder_roles", {})
  iam_organization_roles = lookup(each.value, "iam_organization_roles", {})
  iam_project_roles      = lookup(each.value, "iam_project_roles", {})
-  iam_sa_roles           = lookup(each.value, "iam_sa_roles", {})
+  # iam_sa_roles           = lookup(each.value, "iam_sa_roles", {})
  # we don't interpolate buckets here as we can't use a dynamic key
  iam_storage_roles = lookup(each.value, "iam_storage_roles", {})
 }
+
+module "automation-service-accounts-iam" {
+  source = "../iam-service-account"
+  for_each = {
+    for k, v in local.automation_sas :
+    k => v if lookup(v, "iam_sa_roles", null) != null
+  }
+  project_id = (
+    module.automation-service-accounts[each.key].service_account.project
+  )
+  name                   = module.automation-service-accounts[each.key].name
+  service_account_create = false
+  context = merge(local.ctx, {
+    service_account_ids = local.project_sas_ids
+  })
+  iam_sa_roles = lookup(each.value, "iam_sa_roles", {})
+}
--- a/modules/project-factory/outputs.tf
+++ b/modules/project-factory/outputs.tf
@@ -15,8 +15,28 @@
 */

 locals {
+  _outputs_automation_buckets = {
+    for k, v in local.automation_buckets : v.source_project => k
+  }
+  _outputs_automation_sas = {
+    for k, v in local.automation_sas : v.source_project => k...
+  }
  outputs_projects = {
    for k, v in local.projects_input : k => {
+      automation = {
+        bucket = try(
+          module.automation-bucket[local._outputs_automation_buckets[k]].name,
+          null
+        )
+        service_accounts = {
+          for sa in lookup(local._outputs_automation_sas, k, []) :
+          sa => {
+            email     = module.automation-service-accounts[sa].email
+            iam_email = module.automation-service-accounts[sa].iam_email
+            id        = module.automation-service-accounts[sa].id
+          }
+        }
+      }
      number     = module.projects[k].number
      project_id = module.projects[k].project_id
      log_buckets = {
@@ -41,9 +61,18 @@ locals {
      }
    }
  }
-  outputs_service_accounts = merge([
-    for k, v in local.outputs_projects : v.service_accounts
-  ]...)
+  outputs_service_accounts = merge(
+    merge([
+      for k, v in local.outputs_projects : v.service_accounts
+    ]...),
+    {
+      for k, v in module.automation-service-accounts : k => {
+        email     = v.email
+        iam_email = v.iam_email
+        id        = v.id
+      }
+    }
+  )
 }

 output "folder_ids" {
@@ -108,7 +137,12 @@ output "service_accounts" {

 output "storage_buckets" {
  description = "Bucket names."
-  value = merge([
-    for k, v in local.outputs_projects : v.storage_buckets
-  ]...)
+  value = merge(
+    merge([
+      for k, v in local.outputs_projects : v.storage_buckets
+    ]...),
+    {
+      for k, v in module.automation-bucket : k => v.name
+    }
+  )
 }
--- a/modules/project-factory/projects-service-accounts.tf
+++ b/modules/project-factory/projects-service-accounts.tf
@@ -44,9 +44,14 @@ locals {
  projects_sas_iam_emails = {
    for k, v in module.service-accounts : "service_accounts/${k}" => v.iam_email
  }
-  project_sas_ids = {
-    for k, v in module.service-accounts : k => v.id
-  }
+  project_sas_ids = merge(
+    {
+      for k, v in module.service-accounts : k => v.id
+    },
+    {
+      for k, v in module.automation-service-accounts : k => v.id
+    }
+  )
 }

 module "service-accounts" {
@@ -76,7 +81,9 @@ module "service_accounts-iam" {
    "${k.project_key}/${k.name}" => k
    if k.iam_sa_roles != {} || k.iam != {}
  }
-  project_id             = module.service-accounts[each.key].service_account.project
+  project_id = (
+    module.service-accounts[each.key].service_account.project
+  )
  name                   = each.value.name
  service_account_create = false
  context = merge(local.ctx, {
--- a/modules/project-factory/sample-data-1/projects/app-0/dev-app0-be-0.yaml
+++ b/modules/project-factory/sample-data-1/projects/app-0/dev-app0-be-0.yaml
@@ -36,4 +36,14 @@ shared_vpc_service_config:
  host_project: $project_ids:dev-net-spoke-0
  service_agent_iam:
    roles/compute.networkUser:
-      - $service_agents:compute
+      - $service_agents:compute
+automation:
+  project: $project_ids:prod-iac-core-0
+  bucket:
+    name: tf-state
+  service_accounts:
+    ro: {}
+    rw:
+      iam_sa_roles:
+        $service_account_ids:dev-app0-be-0/automation/ro:
+          - roles.iam.serviceAccountTokenCreator
--- a/tests/modules/project_factory/examples/example.yaml
+++ b/tests/modules/project_factory/examples/example.yaml
@@ -13,7 +13,7 @@
 # limitations under the License.

 values:
-  module.project-factory.module.automation-bucket["dev-tb-app0-0/tf-state"].google_storage_bucket.bucket[0]:
+  module.project-factory.module.automation-bucket["dev-tb-app0-0/automation/tf-state"].google_storage_bucket.bucket[0]:
    autoclass: []
    cors: []
    custom_placement_config: []
@@ -40,13 +40,13 @@ values:
    uniform_bucket_level_access: true
    versioning:
    - enabled: false
-  ? module.project-factory.module.automation-bucket["dev-tb-app0-0/tf-state"].google_storage_bucket_iam_binding.authoritative["roles/storage.objectCreator"]
+  ? module.project-factory.module.automation-bucket["dev-tb-app0-0/automation/tf-state"].google_storage_bucket_iam_binding.authoritative["roles/storage.objectCreator"]
  : bucket: test-pf-dev-tb-app0-0-tf-state
    condition: []
    members:
    - serviceAccount:test-pf-dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com
    role: roles/storage.objectCreator
-  ? module.project-factory.module.automation-bucket["dev-tb-app0-0/tf-state"].google_storage_bucket_iam_binding.authoritative["roles/storage.objectViewer"]
+  ? module.project-factory.module.automation-bucket["dev-tb-app0-0/automation/tf-state"].google_storage_bucket_iam_binding.authoritative["roles/storage.objectViewer"]
  : bucket: test-pf-dev-tb-app0-0-tf-state
    condition: []
    members:
@@ -55,8 +55,8 @@ values:
    - serviceAccount:test-pf-dev-tb-app0-0-ro@test-pf-teams-iac-0.iam.gserviceaccount.com
    - serviceAccount:test-pf-dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com
    role: roles/storage.objectViewer
-  module.project-factory.module.automation-service-accounts["dev-tb-app0-0/ro"].google_service_account.service_account[0]:
-    account_id: test-pf-dev-tb-app0-0-ro
+  ? module.project-factory.module.automation-service-accounts["dev-tb-app0-0/automation/ro"].google_service_account.service_account[0]
+  : account_id: test-pf-dev-tb-app0-0-ro
    create_ignore_already_exists: null
    description: Team B app 0 read-only automation sa.
    disabled: false
@@ -65,8 +65,8 @@ values:
    member: serviceAccount:test-pf-dev-tb-app0-0-ro@test-pf-teams-iac-0.iam.gserviceaccount.com
    project: test-pf-teams-iac-0
    timeouts: null
-  module.project-factory.module.automation-service-accounts["dev-tb-app0-0/rw"].google_service_account.service_account[0]:
-    account_id: test-pf-dev-tb-app0-0-rw
+  ? module.project-factory.module.automation-service-accounts["dev-tb-app0-0/automation/rw"].google_service_account.service_account[0]
+  : account_id: test-pf-dev-tb-app0-0-rw
    create_ignore_already_exists: null
    description: Team B app 0 read/write automation sa.
    disabled: false
@@ -89,11 +89,13 @@ values:
    billing_account: 123456-123456-123456
    budget_filter:
    - calendar_period: null
+      credit_types: null
      credit_types_treatment: INCLUDE_ALL_CREDITS
      custom_period: []
      projects: null
      resource_ancestors:
      - folders/1234567890
+      subaccounts: null
    display_name: 100 dollars in current spend
    ownership_scope: null
    threshold_rules: