Fix issues with FAST CI/CD support (#3454)

* wip, broken

* wip

* streamline locals

* tfdoc

* update yaml files

* refactor

fast/stages/0-org-setup/README.md

@@ -646,7 +646,7 @@ Define values for the `var.environments` variable in a tfvars file.
 | name | description | modules | resources |
 |---|---|---|---|
 | [billing.tf](./billing.tf) | None | <code>billing-account</code> |  |
-| [cicd.tf](./cicd.tf) | None |  | <code>google_iam_workload_identity_pool</code> · <code>google_iam_workload_identity_pool_provider</code> · <code>google_storage_bucket_object</code> · <code>local_file</code> |
+| [cicd.tf](./cicd.tf) | None | <code>iam-service-account</code> | <code>google_storage_bucket_object</code> · <code>local_file</code> |
 | [factory.tf](./factory.tf) | None | <code>project-factory</code> |  |
 | [imports.tf](./imports.tf) | None |  |  |
 | [main.tf](./main.tf) | Module-level locals and resources. |  | <code>terraform_data</code> |
@@ -655,6 +655,7 @@ Define values for the `var.environments` variable in a tfvars file.
 | [outputs.tf](./outputs.tf) | Module outputs. |  |  |
 | [variables.tf](./variables.tf) | Module variables. |  |  |
 | [wif-definitions.tf](./wif-definitions.tf) | Workload Identity provider definitions. |  |  |
+| [wif-providers.tf](./wif-providers.tf) | None |  | <code>google_iam_workload_identity_pool</code> · <code>google_iam_workload_identity_pool_provider</code> |
 
 ## Variables
 

fast/stages/0-org-setup/assets/workflow-github.yaml

@@ -26,20 +26,20 @@ on:
 env:
   FAST_SERVICE_ACCOUNT: ${service_accounts.apply}
   FAST_SERVICE_ACCOUNT_PLAN: ${service_accounts.plan}
-  FAST_WIF_PROVIDER: ${identity_provider}
+  FAST_WIF_PROVIDER: ${workload_identity.pool_id}
   SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-  TF_PROVIDERS_FILE: ${tf_providers_files.apply}
-  TF_PROVIDERS_FILE_PLAN: ${tf_providers_files.plan}
+  TF_PROVIDERS_FILE: ${provider_files.apply}
+  TF_PROVIDERS_FILE_PLAN: ${provider_files.plan}
   TF_VERSION: 1.12.2
 
 jobs:
   fast-pr:
     # Skip PRs which are closed without being merged.
     if: >-
-      github.event.action == 'closed' && 
+      github.event.action == 'closed' &&
       github.event.pull_request.merged == true ||
       github.event.action == 'opened' ||
-      github.event.action == 'synchronize'  
+      github.event.action == 'synchronize'
     permissions:
       contents: read
       id-token: write
@@ -99,7 +99,7 @@ jobs:
         run: |
           gcloud storage cp -r \
             "gs://${outputs_bucket}/providers/$${{env.provider_file}}" ./
-          %{~ for f in tf_var_files ~}
+          %{~ for f in tfvars_files ~}
           gcloud storage cp -r \
             "gs://${outputs_bucket}/tfvars/${f}" ./
           %{~ endfor ~}

fast/stages/0-org-setup/assets/workflow-gitlab.yaml

@@ -15,10 +15,10 @@
 variables:
   GOOGLE_CREDENTIALS: cicd-sa-credentials.json
   FAST_OUTPUTS_BUCKET: ${outputs_bucket}
-  FAST_WIF_PROVIDER: ${identity_provider}
+  FAST_WIF_PROVIDER: ${workload_identity.pool_id}
   SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-  %{~ if tf_var_files != [] ~}
-  TF_VAR_FILES: ${join("\n    ", tf_var_files)}
+  %{~ if tfvars_files != [] ~}
+  TF_VAR_FILES: ${join("\n    ", tfvars_files)}
   %{~ endif ~}
 
 workflow:
@@ -28,13 +28,13 @@ workflow:
       variables:
         COMMAND: apply
         FAST_SERVICE_ACCOUNT: ${service_accounts.apply}
-        TF_PROVIDERS_FILE: ${tf_providers_files.apply}
+        TF_PROVIDERS_FILE: ${provider_files.apply}
     # pr / plan
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       variables:
         COMMAND: plan
         FAST_SERVICE_ACCOUNT: ${service_accounts.plan}
-        TF_PROVIDERS_FILE: ${tf_providers_files.plan}
+        TF_PROVIDERS_FILE: ${provider_files.plan}
 
 stages:
   - gcp-setup
@@ -50,13 +50,13 @@ gcp-setup:
     paths:
       - cicd-sa-credentials.json
       - providers.tf
-      %{~ for f in tf_var_files ~}
+      %{~ for f in tfvars_files ~}
       - ${f}
       %{~ endfor ~}
   id_tokens:
     GITLAB_TOKEN:
       aud:
-      %{~ for aud in audiences ~}
+      %{~ for aud in workload_identity.audiences ~}
         - ${aud}
       %{~ endfor ~}
   before_script:
@@ -71,7 +71,7 @@ gcp-setup:
         --credential-source-file=token.txt
     - gcloud config set auth/credential_file_override $GOOGLE_CREDENTIALS
     - gcloud storage cp -r "gs://$FAST_OUTPUTS_BUCKET/providers/$TF_PROVIDERS_FILE" ./providers.tf
-    %{~ for f in tf_var_files ~}
+    %{~ for f in tfvars_files ~}
     - gcloud storage cp gs://$FAST_OUTPUTS_BUCKET/tfvars/${f} ./
     %{~ endfor ~}
 
@@ -83,7 +83,7 @@ tf-plan-apply:
   id_tokens:
     GITLAB_TOKEN:
       aud:
-      %{~ for aud in audiences ~}
+      %{~ for aud in workload_identity.audiences ~}
         - ${aud}
       %{~ endfor ~}
   image:

fast/stages/0-org-setup/cicd.tf

@@ -15,126 +15,120 @@
  */
 
 locals {
-  _cicd = try(yamldecode(file(local.paths.cicd)), {})
-  _cicd_identity_providers = {
-    for k, v in google_iam_workload_identity_pool_provider.default :
-    "$wif_providers:${k}" => v.name
-  }
-  _cicd_output_files = {
-    for k, v in google_storage_bucket_object.providers :
-    "$output_files:providers/${k}" => split("/", v.name)[1]
-  }
-  cicd_project_ids = {
-    for k, v in merge(
-      var.context.project_ids,
-      module.factory.project_ids
-    ) : "$project_ids:${k}" => v
+  # raw configuration (the wif files are also users of this local)
+  cicd = try(yamldecode(file(local.paths.cicd)), {})
+  # dereferencing maps
+  cicd_ctx_sa = {
+    for k, v in merge(local.ctx.iam_principals, module.factory.iam_principals) :
+    "$iam_principals:${k}" => v
   }
+  cicd_ctx_wif = try({
+    "$wif_pools:${local.wif_pool_name}" = google_iam_workload_identity_pool.default.0.name
+  }, {})
+  # normalize workflow configurations
   cicd_workflows = {
-    for k, v in lookup(local._cicd, "workflows", {}) : k => {
-      outputs_bucket = lookup(
-        local.of_buckets,
-        v.output_files.storage_bucket,
-        v.output_files.storage_bucket
-      )
-      workflow = templatefile("assets/workflow-${v.template}.yaml", {
-        identity_provider = lookup(
-          local._cicd_identity_providers,
-          v.workload_identity_provider.id,
-          v.workload_identity_provider.id
-        )
-        audiences = try(v.workload_identity_provider.audiences, [])
-        service_accounts = {
-          apply = lookup(
-            local.of_service_accounts,
-            v.service_accounts.apply,
-            v.service_accounts.apply
-          )
-          plan = lookup(
-            local.of_service_accounts,
-            v.service_accounts.plan,
-            v.service_accounts.plan
-          )
-        }
-        outputs_bucket = lookup(
-          local.of_buckets,
-          v.output_files.storage_bucket,
-          v.output_files.storage_bucket
-        )
-        stage_name = k
-        tf_providers_files = {
-          apply = lookup(
-            local._cicd_output_files,
-            v.output_files.providers.apply,
-            v.output_files.providers.apply
-          )
-          plan = lookup(
-            local._cicd_output_files,
-            v.output_files.providers.plan,
-            v.output_files.providers.plan
-          )
-        }
-        tf_var_files = try(v.output_files.files, [])
+    for k, v in lookup(local.cicd, "workflows", {}) : k => merge(v, {
+      iam_principal_templates = {
+        branch = local.wif_defs[v.repository.type].principal_branch
+        repo   = local.wif_defs[v.repository.type].principal_repo
+      }
+      repository = merge(v.repository, {
+        apply_branches = try(v.repository.apply_branches, [])
       })
-    }
-  }
-  wif_project = try(local._cicd.workload_identity_federation.project, null)
-  wif_providers = {
-    for k, v in try(local._cicd.workload_identity_federation.providers, {}) :
-    k => merge(v, lookup(local.wif_defs, v.issuer, {}))
-  }
-}
-
-resource "google_iam_workload_identity_pool" "default" {
-  count = local.wif_project == null ? 0 : 1
-  project = lookup(
-    local.cicd_project_ids, local.wif_project, local.wif_project
-  )
-  workload_identity_pool_id = try(
-    local._cicd.workload_identity_federation.pool_name, "iac-0"
-  )
-}
-
-resource "google_iam_workload_identity_pool_provider" "default" {
-  for_each = local.wif_providers
-  project = (
-    google_iam_workload_identity_pool.default[0].project
-  )
-  workload_identity_pool_id = (
-    google_iam_workload_identity_pool.default[0].workload_identity_pool_id
-  )
-  workload_identity_pool_provider_id = lookup(each.value, "provider_id", each.key)
-  attribute_condition = lookup(
-    each.value, "attribute_condition", null
-  )
-  attribute_mapping = lookup(
-    each.value, "attribute_mapping", {}
-  )
-  oidc {
-    # Setting an empty list configures allowed_audiences to the url of the provider
-    allowed_audiences = try(each.value.custom_settings.audiences, [])
-    # If users don't provide an issuer_uri, we set the public one for the platform chosen.
-    issuer_uri = (
-      try(each.value.custom_settings.issuer_uri, null) != null
-      ? each.value.custom_settings.issuer_uri
-      : try(each.value.issuer_uri, null)
+      service_accounts = {
+        apply = trimprefix(try(
+          local.cicd_ctx_sa[v.service_accounts.apply], v.service_accounts.apply
+        ), "serviceAccount:")
+        plan = trimprefix(try(
+          local.cicd_ctx_sa[v.service_accounts.plan], v.service_accounts.plan
+        ), "serviceAccount:")
+      }
+      workload_identity = {
+        audiences = try(v.workload_identity.audiences, [])
+        pool_id = try(
+          local.cicd_ctx_wif[v.workload_identity.pool_id],
+          v.workload_identity.pool_id
+        )
+      }
+    })
+    if(
+      try(local.wif_defs[v.repository.type], null) != null &&
+      try(v.provider_files.apply, null) != null &&
+      try(v.provider_files.plan, null) != null &&
+      try(v.repository.name, null) != null &&
+      try(v.service_accounts.apply, null) != null &&
+      try(v.service_accounts.plan, null) != null &&
+      try(v.workload_identity.pool_id, null) != null
     )
-    # OIDC JWKs in JSON String format. If no value is provided, they key is
-    # fetched from the `.well-known` path for the issuer_uri
-    jwks_json = try(each.value.custom_settings.jwks_json, null)
+  }
+  # generate workflow files contents
+  cicd_workflows_contents = {
+    for k, v in local.cicd_workflows : k => templatefile(
+      "assets/workflow-${v.repository.type}.yaml", merge(v, {
+        outputs_bucket = local.of_outputs_bucket
+        stage_name     = k
+      })
+    )
+  }
+}
+
+module "cicd-sa-apply" {
+  source   = "../../../modules/iam-service-account"
+  for_each = local.cicd_workflows
+  name     = each.value.service_accounts.apply
+  service_account_reuse = {
+    use_data_source = false
+  }
+  iam = {
+    "roles/iam.workloadIdentityUser" = (
+      length(each.value.repository.apply_branches) == 0
+      ? [
+        format(
+          each.value.iam_principal_templates.repo,
+          each.value.workload_identity.pool_id,
+          each.value.repository.name
+        )
+      ]
+      : [
+        for v in each.value.repository.apply_branches : format(
+          each.value.iam_principal_templates.branch,
+          each.value.workload_identity.pool_id,
+          each.value.repository.name,
+          v
+        )
+      ]
+    )
+  }
+}
+
+module "cicd-sa-plan" {
+  source   = "../../../modules/iam-service-account"
+  for_each = local.cicd_workflows
+  name     = each.value.service_accounts.plan
+  service_account_reuse = {
+    use_data_source = false
+  }
+  iam = {
+    "roles/iam.workloadIdentityUser" = [
+      format(
+        each.value.iam_principal_templates.repo,
+        each.value.workload_identity.pool_id,
+        each.value.repository.name
+      )
+    ]
   }
 }
 
 resource "local_file" "workflows" {
-  for_each        = local.of_path == null ? {} : local.cicd_workflows
+  for_each        = local.of_path == null ? {} : local.cicd_workflows_contents
   file_permission = "0644"
   filename        = "${local.of_path}/workflows/${each.key}.yaml"
-  content         = each.value.workflow
+  content         = each.value
 }
 
 resource "google_storage_bucket_object" "workflows" {
-  for_each = local.cicd_workflows
-  bucket   = each.value.outputs_bucket
+  for_each = local.output_files.storage_bucket == null ? {} : local.cicd_workflows_contents
+  bucket   = local.of_outputs_bucket
   name     = "workflows/${each.key}.yaml"
-  content  = each.value.workflow
+  content  = each.value
 }

fast/stages/0-org-setup/datasets/classic/cicd.yaml

@@ -28,20 +28,20 @@ workload_identity_federation:
       #   jwks_json_path:
 workflows:
   org-setup:
-    template: github
-    workload_identity_provider:
-      id: $wif_providers:github
-      audiences: []
+    provider_files:
+      apply: 0-org-setup-providers.tf
+      plan: 0-org-setup-providers-ro.tf
     repository:
-      name: org-setup
-      branch: main
-    output_files:
-      storage_bucket: $storage_buckets:iac-0/iac-outputs
-      providers:
-        apply: $output_files:providers/0-org-setup
-        plan: $output_files:providers/0-org-setup-ro
-      files:
-        - 0-org-setup.auto.tfvars.json
+      name: gh-org/gh-repo
+      type: github
+      apply_branches:
+        - master
+        - fast-dev
     service_accounts:
       apply: $iam_principals:service_accounts/iac-0/iac-org-cicd-rw
       plan: $iam_principals:service_accounts/iac-0/iac-org-cicd-ro
+    tfvars_files:
+      - 0-org-setup.auto.tfvars
+    workload_identity:
+      pool_id: $wif_pools:iac-0
+      audiences: []

fast/stages/0-org-setup/schemas/cicd.schema.json

@@ -4,6 +4,106 @@
   "type": "object",
   "additionalProperties": false,
   "properties": {
+    "workflows": {
+      "type": "object",
+      "additionalProperties": false,
+      "patternProperties": {
+        "^[a-z-][a-z0-9-]+$": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "provider_files",
+            "repository",
+            "service_accounts",
+            "workload_identity"
+          ],
+          "properties": {
+            "provider_files": {
+              "type": "object",
+              "additionalProperties": false,
+              "required": [
+                "apply",
+                "plan"
+              ],
+              "properties": {
+                "apply": {
+                  "type": "string"
+                },
+                "plan": {
+                  "type": "string"
+                }
+              }
+            },
+            "repository": {
+              "type": "object",
+              "additionalProperties": false,
+              "required": [
+                "name",
+                "type"
+              ],
+              "properties": {
+                "name": {
+                  "type": "string"
+                },
+                "type": {
+                  "type": "string",
+                  "enum": [
+                    "github",
+                    "gitlab"
+                  ]
+                },
+                "apply_branches": {
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                }
+              }
+            },
+            "service_accounts": {
+              "type": "object",
+              "additionalProperties": false,
+              "required": [
+                "apply",
+                "plan"
+              ],
+              "properties": {
+                "apply": {
+                  "type": "string"
+                },
+                "plan": {
+                  "type": "string"
+                }
+              }
+            },
+            "tfvars_files": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            },
+            "workload_identity": {
+              "type": "object",
+              "additionalProperties": false,
+              "required": [
+                "pool_id"
+              ],
+              "properties": {
+                "pool_id": {
+                  "type": "string"
+                },
+                "audiences": {
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
     "workload_identity_federation": {
       "type": "object",
       "additionalProperties": false,
@@ -66,106 +166,6 @@
           }
         }
       }
-    },
-    "workflows": {
-      "type": "object",
-      "additionalProperties": false,
-      "patternProperties": {
-        "^[a-z-][a-z0-9-]+$": {
-          "type": "object",
-          "additionalProperties": false,
-          "minProperties": 5,
-          "properties": {
-            "output_files": {
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "providers",
-                "storage_bucket"
-              ],
-              "properties": {
-                "files": {
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
-                },
-                "providers": {
-                  "type": "object",
-                  "additionalProperties": false,
-                  "required": [
-                    "apply",
-                    "plan"
-                  ],
-                  "properties": {
-                    "apply": {
-                      "type": "string"
-                    },
-                    "plan": {
-                      "type": "string"
-                    }
-                  }
-                },
-                "storage_bucket": {
-                  "type": "string"
-                }
-              }
-            },
-            "repository": {
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "name"
-              ],
-              "properties": {
-                "name": {
-                  "type": "string"
-                },
-                "branch": {
-                  "type": "string"
-                }
-              }
-            },
-            "service_accounts": {
-              "type": "object",
-              "additionalProperties": false,
-              "properties": {
-                "apply": {
-                  "type": "string"
-                },
-                "plan": {
-                  "type": "string"
-                }
-              }
-            },
-            "template": {
-              "type": "string",
-              "enum": [
-                "github",
-                "gitlab"
-              ]
-            },
-            "workload_identity_provider": {
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "id"
-              ],
-              "properties": {
-                "audiences": {
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
-                },
-                "id": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
     }
   }
 }

fast/stages/0-org-setup/wif-providers.tf

@@ -0,0 +1,62 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  wif_ctx_project_ids = {
+    for k, v in merge(var.context.project_ids, module.factory.project_ids) :
+    "$project_ids:${k}" => v
+  }
+  wif_pool_name = try(
+    local.cicd.workload_identity_federation.pool_name, "iac-0"
+  )
+  wif_project = try(local.cicd.workload_identity_federation.project, null)
+  wif_providers = local.wif_project == null ? {} : {
+    for k, v in try(local.cicd.workload_identity_federation.providers, {}) :
+    k => merge(v, lookup(local.wif_defs, v.issuer, {}))
+  }
+}
+
+resource "google_iam_workload_identity_pool" "default" {
+  count = local.wif_project == null ? 0 : 1
+  project = lookup(
+    local.wif_ctx_project_ids, local.wif_project, local.wif_project
+  )
+  workload_identity_pool_id = local.wif_pool_name
+}
+
+resource "google_iam_workload_identity_pool_provider" "default" {
+  for_each = local.wif_providers
+  project  = google_iam_workload_identity_pool.default[0].project
+  workload_identity_pool_id = (
+    google_iam_workload_identity_pool.default[0].workload_identity_pool_id
+  )
+  workload_identity_pool_provider_id = lookup(each.value, "provider_id", each.key)
+  attribute_condition                = lookup(each.value, "attribute_condition", null)
+  attribute_mapping                  = lookup(each.value, "attribute_mapping", {})
+  oidc {
+    # Setting an empty list configures allowed_audiences to the url of the provider
+    allowed_audiences = try(each.value.custom_settings.audiences, [])
+    # If users don't provide an issuer_uri, we set the public one for the platform chosen.
+    issuer_uri = (
+      try(each.value.custom_settings.issuer_uri, null) != null
+      ? each.value.custom_settings.issuer_uri
+      : try(each.value.issuer_uri, null)
+    )
+    # OIDC JWKs in JSON String format. If no value is provided, they key is
+    # fetched from the `.well-known` path for the issuer_uri
+    jwks_json = try(each.value.custom_settings.jwks_json, null)
+  }
+}