diff --git a/fast/stages/0-org-setup/README.md b/fast/stages/0-org-setup/README.md index 13607dc66..e33d63055 100644 --- a/fast/stages/0-org-setup/README.md +++ b/fast/stages/0-org-setup/README.md @@ -646,7 +646,7 @@ Define values for the `var.environments` variable in a tfvars file. | name | description | modules | resources | |---|---|---|---| | [billing.tf](./billing.tf) | None | billing-account | | -| [cicd.tf](./cicd.tf) | None | | google_iam_workload_identity_pool · google_iam_workload_identity_pool_provider · google_storage_bucket_object · local_file | +| [cicd.tf](./cicd.tf) | None | iam-service-account | google_storage_bucket_object · local_file | | [factory.tf](./factory.tf) | None | project-factory | | | [imports.tf](./imports.tf) | None | | | | [main.tf](./main.tf) | Module-level locals and resources. | | terraform_data | @@ -655,6 +655,7 @@ Define values for the `var.environments` variable in a tfvars file. | [outputs.tf](./outputs.tf) | Module outputs. | | | | [variables.tf](./variables.tf) | Module variables. | | | | [wif-definitions.tf](./wif-definitions.tf) | Workload Identity provider definitions. | | | +| [wif-providers.tf](./wif-providers.tf) | None | | google_iam_workload_identity_pool · google_iam_workload_identity_pool_provider | ## Variables diff --git a/fast/stages/0-org-setup/assets/workflow-github.yaml b/fast/stages/0-org-setup/assets/workflow-github.yaml index 465fd29b8..ef14c6833 100644 --- a/fast/stages/0-org-setup/assets/workflow-github.yaml +++ b/fast/stages/0-org-setup/assets/workflow-github.yaml @@ -26,20 +26,20 @@ on: env: FAST_SERVICE_ACCOUNT: ${service_accounts.apply} FAST_SERVICE_ACCOUNT_PLAN: ${service_accounts.plan} - FAST_WIF_PROVIDER: ${identity_provider} + FAST_WIF_PROVIDER: ${workload_identity.pool_id} SSH_AUTH_SOCK: /tmp/ssh_agent.sock - TF_PROVIDERS_FILE: ${tf_providers_files.apply} - TF_PROVIDERS_FILE_PLAN: ${tf_providers_files.plan} + TF_PROVIDERS_FILE: ${provider_files.apply} + TF_PROVIDERS_FILE_PLAN: ${provider_files.plan} TF_VERSION: 1.12.2 jobs: fast-pr: # Skip PRs which are closed without being merged. if: >- - github.event.action == 'closed' && + github.event.action == 'closed' && github.event.pull_request.merged == true || github.event.action == 'opened' || - github.event.action == 'synchronize' + github.event.action == 'synchronize' permissions: contents: read id-token: write @@ -99,7 +99,7 @@ jobs: run: | gcloud storage cp -r \ "gs://${outputs_bucket}/providers/$${{env.provider_file}}" ./ - %{~ for f in tf_var_files ~} + %{~ for f in tfvars_files ~} gcloud storage cp -r \ "gs://${outputs_bucket}/tfvars/${f}" ./ %{~ endfor ~} diff --git a/fast/stages/0-org-setup/assets/workflow-gitlab.yaml b/fast/stages/0-org-setup/assets/workflow-gitlab.yaml index 150340835..3ee417097 100644 --- a/fast/stages/0-org-setup/assets/workflow-gitlab.yaml +++ b/fast/stages/0-org-setup/assets/workflow-gitlab.yaml @@ -15,10 +15,10 @@ variables: GOOGLE_CREDENTIALS: cicd-sa-credentials.json FAST_OUTPUTS_BUCKET: ${outputs_bucket} - FAST_WIF_PROVIDER: ${identity_provider} + FAST_WIF_PROVIDER: ${workload_identity.pool_id} SSH_AUTH_SOCK: /tmp/ssh_agent.sock - %{~ if tf_var_files != [] ~} - TF_VAR_FILES: ${join("\n ", tf_var_files)} + %{~ if tfvars_files != [] ~} + TF_VAR_FILES: ${join("\n ", tfvars_files)} %{~ endif ~} workflow: @@ -28,13 +28,13 @@ workflow: variables: COMMAND: apply FAST_SERVICE_ACCOUNT: ${service_accounts.apply} - TF_PROVIDERS_FILE: ${tf_providers_files.apply} + TF_PROVIDERS_FILE: ${provider_files.apply} # pr / plan - if: $CI_PIPELINE_SOURCE == 'merge_request_event' variables: COMMAND: plan FAST_SERVICE_ACCOUNT: ${service_accounts.plan} - TF_PROVIDERS_FILE: ${tf_providers_files.plan} + TF_PROVIDERS_FILE: ${provider_files.plan} stages: - gcp-setup @@ -50,13 +50,13 @@ gcp-setup: paths: - cicd-sa-credentials.json - providers.tf - %{~ for f in tf_var_files ~} + %{~ for f in tfvars_files ~} - ${f} %{~ endfor ~} id_tokens: GITLAB_TOKEN: aud: - %{~ for aud in audiences ~} + %{~ for aud in workload_identity.audiences ~} - ${aud} %{~ endfor ~} before_script: @@ -71,7 +71,7 @@ gcp-setup: --credential-source-file=token.txt - gcloud config set auth/credential_file_override $GOOGLE_CREDENTIALS - gcloud storage cp -r "gs://$FAST_OUTPUTS_BUCKET/providers/$TF_PROVIDERS_FILE" ./providers.tf - %{~ for f in tf_var_files ~} + %{~ for f in tfvars_files ~} - gcloud storage cp gs://$FAST_OUTPUTS_BUCKET/tfvars/${f} ./ %{~ endfor ~} @@ -83,7 +83,7 @@ tf-plan-apply: id_tokens: GITLAB_TOKEN: aud: - %{~ for aud in audiences ~} + %{~ for aud in workload_identity.audiences ~} - ${aud} %{~ endfor ~} image: diff --git a/fast/stages/0-org-setup/cicd.tf b/fast/stages/0-org-setup/cicd.tf index 20a59593b..6d09e56e7 100644 --- a/fast/stages/0-org-setup/cicd.tf +++ b/fast/stages/0-org-setup/cicd.tf @@ -15,126 +15,120 @@ */ locals { - _cicd = try(yamldecode(file(local.paths.cicd)), {}) - _cicd_identity_providers = { - for k, v in google_iam_workload_identity_pool_provider.default : - "$wif_providers:${k}" => v.name - } - _cicd_output_files = { - for k, v in google_storage_bucket_object.providers : - "$output_files:providers/${k}" => split("/", v.name)[1] - } - cicd_project_ids = { - for k, v in merge( - var.context.project_ids, - module.factory.project_ids - ) : "$project_ids:${k}" => v + # raw configuration (the wif files are also users of this local) + cicd = try(yamldecode(file(local.paths.cicd)), {}) + # dereferencing maps + cicd_ctx_sa = { + for k, v in merge(local.ctx.iam_principals, module.factory.iam_principals) : + "$iam_principals:${k}" => v } + cicd_ctx_wif = try({ + "$wif_pools:${local.wif_pool_name}" = google_iam_workload_identity_pool.default.0.name + }, {}) + # normalize workflow configurations cicd_workflows = { - for k, v in lookup(local._cicd, "workflows", {}) : k => { - outputs_bucket = lookup( - local.of_buckets, - v.output_files.storage_bucket, - v.output_files.storage_bucket - ) - workflow = templatefile("assets/workflow-${v.template}.yaml", { - identity_provider = lookup( - local._cicd_identity_providers, - v.workload_identity_provider.id, - v.workload_identity_provider.id - ) - audiences = try(v.workload_identity_provider.audiences, []) - service_accounts = { - apply = lookup( - local.of_service_accounts, - v.service_accounts.apply, - v.service_accounts.apply - ) - plan = lookup( - local.of_service_accounts, - v.service_accounts.plan, - v.service_accounts.plan - ) - } - outputs_bucket = lookup( - local.of_buckets, - v.output_files.storage_bucket, - v.output_files.storage_bucket - ) - stage_name = k - tf_providers_files = { - apply = lookup( - local._cicd_output_files, - v.output_files.providers.apply, - v.output_files.providers.apply - ) - plan = lookup( - local._cicd_output_files, - v.output_files.providers.plan, - v.output_files.providers.plan - ) - } - tf_var_files = try(v.output_files.files, []) + for k, v in lookup(local.cicd, "workflows", {}) : k => merge(v, { + iam_principal_templates = { + branch = local.wif_defs[v.repository.type].principal_branch + repo = local.wif_defs[v.repository.type].principal_repo + } + repository = merge(v.repository, { + apply_branches = try(v.repository.apply_branches, []) }) - } - } - wif_project = try(local._cicd.workload_identity_federation.project, null) - wif_providers = { - for k, v in try(local._cicd.workload_identity_federation.providers, {}) : - k => merge(v, lookup(local.wif_defs, v.issuer, {})) - } -} - -resource "google_iam_workload_identity_pool" "default" { - count = local.wif_project == null ? 0 : 1 - project = lookup( - local.cicd_project_ids, local.wif_project, local.wif_project - ) - workload_identity_pool_id = try( - local._cicd.workload_identity_federation.pool_name, "iac-0" - ) -} - -resource "google_iam_workload_identity_pool_provider" "default" { - for_each = local.wif_providers - project = ( - google_iam_workload_identity_pool.default[0].project - ) - workload_identity_pool_id = ( - google_iam_workload_identity_pool.default[0].workload_identity_pool_id - ) - workload_identity_pool_provider_id = lookup(each.value, "provider_id", each.key) - attribute_condition = lookup( - each.value, "attribute_condition", null - ) - attribute_mapping = lookup( - each.value, "attribute_mapping", {} - ) - oidc { - # Setting an empty list configures allowed_audiences to the url of the provider - allowed_audiences = try(each.value.custom_settings.audiences, []) - # If users don't provide an issuer_uri, we set the public one for the platform chosen. - issuer_uri = ( - try(each.value.custom_settings.issuer_uri, null) != null - ? each.value.custom_settings.issuer_uri - : try(each.value.issuer_uri, null) + service_accounts = { + apply = trimprefix(try( + local.cicd_ctx_sa[v.service_accounts.apply], v.service_accounts.apply + ), "serviceAccount:") + plan = trimprefix(try( + local.cicd_ctx_sa[v.service_accounts.plan], v.service_accounts.plan + ), "serviceAccount:") + } + workload_identity = { + audiences = try(v.workload_identity.audiences, []) + pool_id = try( + local.cicd_ctx_wif[v.workload_identity.pool_id], + v.workload_identity.pool_id + ) + } + }) + if( + try(local.wif_defs[v.repository.type], null) != null && + try(v.provider_files.apply, null) != null && + try(v.provider_files.plan, null) != null && + try(v.repository.name, null) != null && + try(v.service_accounts.apply, null) != null && + try(v.service_accounts.plan, null) != null && + try(v.workload_identity.pool_id, null) != null ) - # OIDC JWKs in JSON String format. If no value is provided, they key is - # fetched from the `.well-known` path for the issuer_uri - jwks_json = try(each.value.custom_settings.jwks_json, null) + } + # generate workflow files contents + cicd_workflows_contents = { + for k, v in local.cicd_workflows : k => templatefile( + "assets/workflow-${v.repository.type}.yaml", merge(v, { + outputs_bucket = local.of_outputs_bucket + stage_name = k + }) + ) + } +} + +module "cicd-sa-apply" { + source = "../../../modules/iam-service-account" + for_each = local.cicd_workflows + name = each.value.service_accounts.apply + service_account_reuse = { + use_data_source = false + } + iam = { + "roles/iam.workloadIdentityUser" = ( + length(each.value.repository.apply_branches) == 0 + ? [ + format( + each.value.iam_principal_templates.repo, + each.value.workload_identity.pool_id, + each.value.repository.name + ) + ] + : [ + for v in each.value.repository.apply_branches : format( + each.value.iam_principal_templates.branch, + each.value.workload_identity.pool_id, + each.value.repository.name, + v + ) + ] + ) + } +} + +module "cicd-sa-plan" { + source = "../../../modules/iam-service-account" + for_each = local.cicd_workflows + name = each.value.service_accounts.plan + service_account_reuse = { + use_data_source = false + } + iam = { + "roles/iam.workloadIdentityUser" = [ + format( + each.value.iam_principal_templates.repo, + each.value.workload_identity.pool_id, + each.value.repository.name + ) + ] } } resource "local_file" "workflows" { - for_each = local.of_path == null ? {} : local.cicd_workflows + for_each = local.of_path == null ? {} : local.cicd_workflows_contents file_permission = "0644" filename = "${local.of_path}/workflows/${each.key}.yaml" - content = each.value.workflow + content = each.value } resource "google_storage_bucket_object" "workflows" { - for_each = local.cicd_workflows - bucket = each.value.outputs_bucket + for_each = local.output_files.storage_bucket == null ? {} : local.cicd_workflows_contents + bucket = local.of_outputs_bucket name = "workflows/${each.key}.yaml" - content = each.value.workflow + content = each.value } diff --git a/fast/stages/0-org-setup/datasets/classic/cicd.yaml b/fast/stages/0-org-setup/datasets/classic/cicd.yaml index de300b14d..64cfbd5a0 100644 --- a/fast/stages/0-org-setup/datasets/classic/cicd.yaml +++ b/fast/stages/0-org-setup/datasets/classic/cicd.yaml @@ -28,20 +28,20 @@ workload_identity_federation: # jwks_json_path: workflows: org-setup: - template: github - workload_identity_provider: - id: $wif_providers:github - audiences: [] + provider_files: + apply: 0-org-setup-providers.tf + plan: 0-org-setup-providers-ro.tf repository: - name: org-setup - branch: main - output_files: - storage_bucket: $storage_buckets:iac-0/iac-outputs - providers: - apply: $output_files:providers/0-org-setup - plan: $output_files:providers/0-org-setup-ro - files: - - 0-org-setup.auto.tfvars.json + name: gh-org/gh-repo + type: github + apply_branches: + - master + - fast-dev service_accounts: apply: $iam_principals:service_accounts/iac-0/iac-org-cicd-rw plan: $iam_principals:service_accounts/iac-0/iac-org-cicd-ro + tfvars_files: + - 0-org-setup.auto.tfvars + workload_identity: + pool_id: $wif_pools:iac-0 + audiences: [] diff --git a/fast/stages/0-org-setup/schemas/cicd.schema.json b/fast/stages/0-org-setup/schemas/cicd.schema.json index 82e3a7f5a..dce84bdd1 100644 --- a/fast/stages/0-org-setup/schemas/cicd.schema.json +++ b/fast/stages/0-org-setup/schemas/cicd.schema.json @@ -4,6 +4,106 @@ "type": "object", "additionalProperties": false, "properties": { + "workflows": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z-][a-z0-9-]+$": { + "type": "object", + "additionalProperties": false, + "required": [ + "provider_files", + "repository", + "service_accounts", + "workload_identity" + ], + "properties": { + "provider_files": { + "type": "object", + "additionalProperties": false, + "required": [ + "apply", + "plan" + ], + "properties": { + "apply": { + "type": "string" + }, + "plan": { + "type": "string" + } + } + }, + "repository": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type" + ], + "properties": { + "name": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "github", + "gitlab" + ] + }, + "apply_branches": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "service_accounts": { + "type": "object", + "additionalProperties": false, + "required": [ + "apply", + "plan" + ], + "properties": { + "apply": { + "type": "string" + }, + "plan": { + "type": "string" + } + } + }, + "tfvars_files": { + "type": "array", + "items": { + "type": "string" + } + }, + "workload_identity": { + "type": "object", + "additionalProperties": false, + "required": [ + "pool_id" + ], + "properties": { + "pool_id": { + "type": "string" + }, + "audiences": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, "workload_identity_federation": { "type": "object", "additionalProperties": false, @@ -66,106 +166,6 @@ } } } - }, - "workflows": { - "type": "object", - "additionalProperties": false, - "patternProperties": { - "^[a-z-][a-z0-9-]+$": { - "type": "object", - "additionalProperties": false, - "minProperties": 5, - "properties": { - "output_files": { - "type": "object", - "additionalProperties": false, - "required": [ - "providers", - "storage_bucket" - ], - "properties": { - "files": { - "type": "array", - "items": { - "type": "string" - } - }, - "providers": { - "type": "object", - "additionalProperties": false, - "required": [ - "apply", - "plan" - ], - "properties": { - "apply": { - "type": "string" - }, - "plan": { - "type": "string" - } - } - }, - "storage_bucket": { - "type": "string" - } - } - }, - "repository": { - "type": "object", - "additionalProperties": false, - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string" - }, - "branch": { - "type": "string" - } - } - }, - "service_accounts": { - "type": "object", - "additionalProperties": false, - "properties": { - "apply": { - "type": "string" - }, - "plan": { - "type": "string" - } - } - }, - "template": { - "type": "string", - "enum": [ - "github", - "gitlab" - ] - }, - "workload_identity_provider": { - "type": "object", - "additionalProperties": false, - "required": [ - "id" - ], - "properties": { - "audiences": { - "type": "array", - "items": { - "type": "string" - } - }, - "id": { - "type": "string" - } - } - } - } - } - } } } } \ No newline at end of file diff --git a/fast/stages/0-org-setup/wif-providers.tf b/fast/stages/0-org-setup/wif-providers.tf new file mode 100644 index 000000000..466093bb7 --- /dev/null +++ b/fast/stages/0-org-setup/wif-providers.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + wif_ctx_project_ids = { + for k, v in merge(var.context.project_ids, module.factory.project_ids) : + "$project_ids:${k}" => v + } + wif_pool_name = try( + local.cicd.workload_identity_federation.pool_name, "iac-0" + ) + wif_project = try(local.cicd.workload_identity_federation.project, null) + wif_providers = local.wif_project == null ? {} : { + for k, v in try(local.cicd.workload_identity_federation.providers, {}) : + k => merge(v, lookup(local.wif_defs, v.issuer, {})) + } +} + +resource "google_iam_workload_identity_pool" "default" { + count = local.wif_project == null ? 0 : 1 + project = lookup( + local.wif_ctx_project_ids, local.wif_project, local.wif_project + ) + workload_identity_pool_id = local.wif_pool_name +} + +resource "google_iam_workload_identity_pool_provider" "default" { + for_each = local.wif_providers + project = google_iam_workload_identity_pool.default[0].project + workload_identity_pool_id = ( + google_iam_workload_identity_pool.default[0].workload_identity_pool_id + ) + workload_identity_pool_provider_id = lookup(each.value, "provider_id", each.key) + attribute_condition = lookup(each.value, "attribute_condition", null) + attribute_mapping = lookup(each.value, "attribute_mapping", {}) + oidc { + # Setting an empty list configures allowed_audiences to the url of the provider + allowed_audiences = try(each.value.custom_settings.audiences, []) + # If users don't provide an issuer_uri, we set the public one for the platform chosen. + issuer_uri = ( + try(each.value.custom_settings.issuer_uri, null) != null + ? each.value.custom_settings.issuer_uri + : try(each.value.issuer_uri, null) + ) + # OIDC JWKs in JSON String format. If no value is provided, they key is + # fetched from the `.well-known` path for the issuer_uri + jwks_json = try(each.value.custom_settings.jwks_json, null) + } +} diff --git a/tests/fast/stages/s0_org_setup/data-simple/cicd.yaml b/tests/fast/stages/s0_org_setup/data-simple/cicd.yaml index 726c0ff83..8578fb96e 100644 --- a/tests/fast/stages/s0_org_setup/data-simple/cicd.yaml +++ b/tests/fast/stages/s0_org_setup/data-simple/cicd.yaml @@ -14,6 +14,22 @@ # yaml-language-server: $schema=../../../../../fast/stages/0-org-setup/schemas/cicd.schema.json +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../schemas/cicd.schema.json + workload_identity_federation: pool_name: iac-0 project: $project_ids:iac-0 @@ -28,20 +44,20 @@ workload_identity_federation: # jwks_json_path: workflows: org-setup: - template: github - workload_identity_provider: - id: $wif_providers:github - audiences: [] + provider_files: + apply: 0-org-setup-providers.tf + plan: 0-org-setup-providers-ro.tf repository: - name: org-setup - branch: main - output_files: - storage_bucket: $storage_buckets:iac-0/iac-outputs - providers: - apply: $output_files:providers/0-org-setup - plan: $output_files:providers/0-org-setup-ro - files: - - tfvars/0-boostrap.auto.tfvars.json + name: gh-org/gh-repo + type: github + apply_branches: + - master + - fast-dev service_accounts: apply: $iam_principals:service_accounts/iac-0/iac-org-cicd-rw plan: $iam_principals:service_accounts/iac-0/iac-org-cicd-ro + tfvars_files: + - 0-org-setup.auto.tfvars + workload_identity: + pool_id: $wif_pools:iac-0 + audiences: [] diff --git a/tests/fast/stages/s0_org_setup/not-simple.yaml b/tests/fast/stages/s0_org_setup/not-simple.yaml index d7b8f47ec..7f2fe7b2f 100644 --- a/tests/fast/stages/s0_org_setup/not-simple.yaml +++ b/tests/fast/stages/s0_org_setup/not-simple.yaml @@ -277,24 +277,6 @@ values: source_md5hash: null temporary_hold: null timeouts: null - google_storage_bucket_object.workflows["org-setup"]: - bucket: ft0-prod-iac-core-0-iac-outputs - cache_control: null - content_disposition: null - content_encoding: null - content_language: null - customer_encryption: [] - deletion_policy: null - detect_md5hash: different hash - event_based_hold: null - force_empty_content_type: null - metadata: null - name: workflows/org-setup.yaml - retention: [] - source: null - source_md5hash: null - temporary_hold: null - timeouts: null local_file.providers["0-org-setup"]: content: "/**\n * Copyright 2022 Google LLC\n *\n * Licensed under the Apache\ \ License, Version 2.0 (the \"License\");\n * you may not use this file except\ @@ -440,13 +422,6 @@ values: filename: /tmp/fast-config/tfvars/0-org-setup.auto.tfvars.json sensitive_content: null source: null - local_file.workflows["org-setup"]: - content_base64: null - directory_permission: '0777' - file_permission: '0644' - filename: /tmp/fast-config/workflows/org-setup.yaml - sensitive_content: null - source: null module.billing-accounts["default"].google_billing_account_iam_member.bindings["billing_admin_org_admins"]: billing_account_id: 012345-012345-012345 condition: [] @@ -2846,6 +2821,7 @@ counts: google_project_service: 33 google_project_service_identity: 9 google_service_account: 14 + google_service_account_iam_binding: 2 google_service_account_iam_member: 4 google_storage_bucket: 3 google_storage_bucket_iam_binding: 4 @@ -2858,6 +2834,6 @@ counts: google_tags_tag_value: 5 google_tags_tag_value_iam_binding: 4 local_file: 9 - modules: 46 - resources: 309 + modules: 48 + resources: 311 terraform_data: 2