From 725f7effce7bdb69522b8ad004b78cda31dcb7ce Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 12 Apr 2022 08:17:27 +0200 Subject: [PATCH] Initial MVP for CI/CD (#608) * preliminary support for wif in stage 0 * IAM wif role * IAM wif role TODO * add support for external SA IAM to SA module * add name output to SA module * separate cicd SA * tfdoc * GITLAB principal (untested) * make GCS name output static * outputs bucket * fix stage 1 test * tweak outputs * tfdoc * move wif_pool to automation variable * add support for top-level and repository providers * add missing boilerplate * fix branchless principal * initial workflow * symlink provider template in stages * remove service accounts from stage 0 cicd tfvars * add cicd interface variable to resman stage * fix cicd variable in resman stage * better condition on outputs_location * fix last change * change outputs_location type * revert outputs_location change * split outputs in stage 0 * update ci/cd temporary notes * rename additive IAM resource in SA module * split outputs in stage 1 * remove unused locals * fix stage 1 tests * tfdoc * Upload action files to outputs_bucket * Fix tests and README * rename template, streamline outputs * local templates and gcs output for all stage 2 * add workflows to local output files * Use lowercase WIF providers everywhere * Bring back suffix for workflow files * Remove unused files * Update READMEs * preliminary CI/CD implementation for stage 1 * fix stage 1 * stage 1 cicd * tfdoc * fix tests * readme and links for cicd and wif * refactor wif providers * refactor cicd for stage 1 * fix stage 1 * wif org policies * split identity provider configuration from cicd * add type attribute to cicd repositories * valid cicd repositories have a workflow template * refactor stage 01 * fix stage 01 tests * minimal CI/CD documentation * better check_links error reporting * fix links * Added Gitlab specific configurations Set the default issuer_uri for Gitlab. Added allowed audiences to OIDC configuration. * Fixed TF formatting in identity providers. * Changing identity provider audience to null Changing identity provider audience to default to null. * add instructions for renaming workflows * address Julio's comments Co-authored-by: Julio Castillo Co-authored-by: alexmeissner --- .gitignore | 2 +- fast/README.md | 9 +- .../{providers.tpl => providers.tf.tpl} | 0 fast/assets/templates/workflow-github.yaml | 149 ++++++++++++++++++ fast/stages/00-bootstrap/IAM.md | 5 +- fast/stages/00-bootstrap/README.md | 141 ++++++++++++++--- fast/stages/00-bootstrap/automation.tf | 35 +++- fast/stages/00-bootstrap/cicd.tf | 63 ++++++++ .../stages/00-bootstrap/identity-providers.tf | 75 +++++++++ fast/stages/00-bootstrap/outputs-files.tf | 45 ++++++ fast/stages/00-bootstrap/outputs-gcs.tf | 44 ++++++ fast/stages/00-bootstrap/outputs.tf | 111 ++++++++++--- fast/stages/00-bootstrap/templates | 1 + fast/stages/00-bootstrap/variables.tf | 52 +++++- fast/stages/01-resman/README.md | 48 +++--- fast/stages/01-resman/branch-data-platform.tf | 132 ++++++++++++---- fast/stages/01-resman/branch-networking.tf | 82 +++++++--- fast/stages/01-resman/branch-sandbox.tf | 4 +- fast/stages/01-resman/branch-security.tf | 48 +++++- fast/stages/01-resman/branch-teams.tf | 142 +++++++++++++---- .../stages/01-resman/globals.auto.tfvars.json | 1 + fast/stages/01-resman/main.tf | 15 +- fast/stages/01-resman/organization.tf | 12 ++ fast/stages/01-resman/outputs-files.tf | 38 +++++ fast/stages/01-resman/outputs-gcs.tf | 37 +++++ fast/stages/01-resman/outputs.tf | 106 ++++++++++--- fast/stages/01-resman/templates | 1 + fast/stages/01-resman/variables.tf | 82 +++++++++- fast/stages/02-networking-nva/README.md | 43 ++--- fast/stages/02-networking-nva/outputs.tf | 8 +- fast/stages/02-networking-nva/variables.tf | 8 + fast/stages/02-networking-peering/README.md | 43 ++--- fast/stages/02-networking-peering/outputs.tf | 8 +- .../stages/02-networking-peering/variables.tf | 8 + fast/stages/02-networking-vpn/README.md | 43 ++--- fast/stages/02-networking-vpn/outputs.tf | 8 +- fast/stages/02-networking-vpn/variables.tf | 8 + fast/stages/02-security/README.md | 41 ++--- fast/stages/02-security/outputs.tf | 8 +- fast/stages/02-security/variables.tf | 8 + fast/stages/README.md | 2 +- modules/gcs/README.md | 6 +- modules/gcs/outputs.tf | 6 +- modules/iam-service-account/README.md | 28 ++-- modules/iam-service-account/iam.tf | 17 ++ modules/iam-service-account/main.tf | 10 +- modules/iam-service-account/outputs.tf | 5 + modules/iam-service-account/variables.tf | 17 +- tests/fast/stages/s01_resman/fixture/main.tf | 9 +- .../stages/s02_networking_nva/fixture/main.tf | 3 + .../s02_networking_peering/fixture/main.tf | 3 + .../stages/s02_networking_vpn/fixture/main.tf | 3 + .../fast/stages/s02_security/fixture/main.tf | 3 + tools/check_links.py | 8 +- tools/state_iam.py | 8 +- 55 files changed, 1539 insertions(+), 303 deletions(-) rename fast/assets/templates/{providers.tpl => providers.tf.tpl} (100%) create mode 100644 fast/assets/templates/workflow-github.yaml create mode 100644 fast/stages/00-bootstrap/cicd.tf create mode 100644 fast/stages/00-bootstrap/identity-providers.tf create mode 100644 fast/stages/00-bootstrap/outputs-files.tf create mode 100644 fast/stages/00-bootstrap/outputs-gcs.tf create mode 120000 fast/stages/00-bootstrap/templates create mode 120000 fast/stages/01-resman/globals.auto.tfvars.json create mode 100644 fast/stages/01-resman/outputs-files.tf create mode 100644 fast/stages/01-resman/outputs-gcs.tf create mode 120000 fast/stages/01-resman/templates diff --git a/.gitignore b/.gitignore index 816a11090..af6a2d21e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,7 @@ bundle.zip **/*.pkrvars.hcl fixture_* fast/configs -fast/stages/**/*providers.tf +fast/stages/**/[0-9]*providers.tf fast/stages/**/terraform.tfvars fast/stages/**/terraform.tfvars.json fast/stages/**/terraform-*.auto.tfvars.json diff --git a/fast/README.md b/fast/README.md index de860a451..8f30dc6d5 100644 --- a/fast/README.md +++ b/fast/README.md @@ -20,6 +20,8 @@ From the perspective of FAST's overall design, stages also work as contacts or i Stages diagram

+Please refer to the [stages](./stages/) section for further details on each stage. + ### Security-first design Security was, from the beginning, one of the most critical elements in the design of Fabric FAST. Many of FAST's design decisions aim to build the foundations of a secure organization. In fact, the first two stages deal mainly with the organization-wide security setup. @@ -32,10 +34,9 @@ A resource factory consumes a simple representation of a resource (e.g., in YAML FAST uses YAML-based factories to deploy subnets and firewall rules and, as its name suggests, in the [project factory](./stages/03-project-factory/) stage. -## Stages and high level design +### CI/CD -As mentioned before, fast relies on multiple stages to progressively bring up your GCP organization(s). -Please refer to the [stages](./stages/) section for further details. +One of our objectives with FAST is to provide a lightweight reference design for the IaC repositories, and a built-in implementation for running our code in automated pipelines. Our CI/CD approach leverages [Workload Identity Federation](https://cloud.google.com/iam/docs/workload-identity-federation), and provides sample workflow configurations for several major providers. Refer to the [CI/CD section in the bootstrap stage](stages/00-bootstrap/README.md#cicd) for more details. ## Implementation @@ -60,5 +61,5 @@ Besides the features already described, FAST roadmap includes: - Stage to deploy environment-specific multitenant GKE clusters following Google's best practices - Stage to deploy a fully featured data platform -- Reference implementation to use FAST in CI/CD pipelines +- Reference implementation to use FAST in CI/CD pipelines (in progress) - Static policy enforcement diff --git a/fast/assets/templates/providers.tpl b/fast/assets/templates/providers.tf.tpl similarity index 100% rename from fast/assets/templates/providers.tpl rename to fast/assets/templates/providers.tf.tpl diff --git a/fast/assets/templates/workflow-github.yaml b/fast/assets/templates/workflow-github.yaml new file mode 100644 index 000000000..913ecf511 --- /dev/null +++ b/fast/assets/templates/workflow-github.yaml @@ -0,0 +1,149 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "FAST ${stage_name} stage" + +on: + pull_request: + branches: + - main + types: + - closed + - opened + +env: + FAST_OUTPUTS_BUCKET: ${outputs_bucket} + FAST_SERVICE_ACCOUNT: ${service_account} + FAST_WIF_PROVIDER: ${identity_provider} + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + TF_PROVIDERS_FILE: ${tf_providers_file} + TF_VAR_FILES: ${tf_var_files == [] ? "''" : join("\n ", tf_var_files)} + TF_VERSION: 1.1.7 + +jobs: + fast-pr: + permissions: + contents: read + id-token: write + issues: write + pull-requests: write + runs-on: ubuntu-latest + steps: + - id: checkout + name: Checkout repository + uses: actions/checkout@v3 + + # set up SSH key authentication to the modules repository + - id: ssh-config + name: Configure SSH authentication + run: | + ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null + ssh-add - <<< "$${{ secrets.CICD_MODULES_KEY }}" + + # set up authentication via Workload identity Federation + - id: gcp-auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v0 + with: + workload_identity_provider: $${{ env.FAST_WIF_PROVIDER }} + service_account: $${{ env.FAST_SERVICE_ACCOUNT }} + access_token_lifetime: 3600s + + - id: gcp-sdk + name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v0 + with: + install_components: alpha + + # copy provider and tfvars files + - id: tf-config + name: Copy Terraform output files + run: | + gcloud alpha storage cp -r \ + "gs://$${{env.FAST_OUTPUTS_BUCKET}}/providers/$${{env.TF_PROVIDERS_FILE}}" ./ + gcloud alpha storage cp -r \ + "gs://$${{env.FAST_OUTPUTS_BUCKET}}/tfvars" ./ + for f in $${{env.TF_VAR_FILES}}; do + ln -s "tfvars/$f" ./ + done + + - id: tf-setup + name: Set up Terraform + uses: hashicorp/setup-terraform@v1 + with: + terraform_version: $${{ env.TF_VERSION }} + + # run Terraform init/validate/plan + - id: tf-init + name: Terraform init + run: | + terraform init -no-color + + - id: tf-validate + name: Terraform validate + run: terraform validate -no-color + + - id: tf-plan + name: Terraform plan + continue-on-error: true + run: | + terraform plan -out ../plan.out -no-color + + - id: tf-apply + if: github.event.pull_request.merged == true + name: Terraform apply + continue-on-error: true + run: | + terraform apply -input=false -auto-approve -no-color ../plan.out + + - id: pr-comment + name: Post comment to Pull Request + uses: actions/github-script@v6 + if: github.event_name == 'pull_request' + env: + PLAN: terraform\n$${{ steps.tf-plan.outputs.stdout }} + with: + script: | + const output = `#### Terraform Initialization โš™๏ธ\`$${{ steps.tf-init.outcome }}\` + #### Terraform Validation ๐Ÿค–\`$${{ steps.tf-validate.outcome }}\` +
Validation Output + + \`\`\`\n + $${{ steps.tf-validate.outputs.stdout }} + \`\`\` + +
+ + #### Terraform Plan ๐Ÿ“–\`$${{ steps.tf-plan.outcome }}\` + +
Show Plan + + \`\`\`\n + $${process.env.PLAN} + \`\`\` + +
+ + #### Terraform Apply ๐Ÿ“–\`$${{ steps.tf-apply.outcome }}\` + + *Pusher: @$${{ github.actor }}, Action: \`$${{ github.event_name }}\`, Working Directory: \`$${{ env.tf_actions_working_dir }}\`, Workflow: \`$${{ github.workflow }}\`*`; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output + }) + + # jq -j -r '.resource_changes[] | (.change.actions | join(",")), " ", .address, "\n" ' foo.json diff --git a/fast/stages/00-bootstrap/IAM.md b/fast/stages/00-bootstrap/IAM.md index 2d7190463..0cc7d7fe7 100644 --- a/fast/stages/00-bootstrap/IAM.md +++ b/fast/stages/00-bootstrap/IAM.md @@ -6,7 +6,7 @@ Legend: + additive, โ€ข conditional. | members | roles | |---|---| -|
domain|[roles/browser](https://cloud.google.com/iam/docs/understanding-roles#browser)
[roles/resourcemanager.organizationViewer](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.organizationViewer) | +|GCP organization domain
domain|[roles/browser](https://cloud.google.com/iam/docs/understanding-roles#browser)
[roles/resourcemanager.organizationViewer](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.organizationViewer) | |gcp-network-admins
group|[roles/cloudasset.owner](https://cloud.google.com/iam/docs/understanding-roles#cloudasset.owner)
[roles/cloudsupport.techSupportEditor](https://cloud.google.com/iam/docs/understanding-roles#cloudsupport.techSupportEditor)
[roles/compute.orgFirewallPolicyAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.orgFirewallPolicyAdmin) +
[roles/compute.xpnAdmin](https://cloud.google.com/iam/docs/understanding-roles#compute.xpnAdmin) +| |gcp-organization-admins
group|[roles/cloudasset.owner](https://cloud.google.com/iam/docs/understanding-roles#cloudasset.owner)
[roles/cloudsupport.admin](https://cloud.google.com/iam/docs/understanding-roles#cloudsupport.admin)
[roles/compute.osAdminLogin](https://cloud.google.com/iam/docs/understanding-roles#compute.osAdminLogin)
[roles/compute.osLoginExternalUser](https://cloud.google.com/iam/docs/understanding-roles#compute.osLoginExternalUser)
[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner)
[roles/resourcemanager.folderAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.folderAdmin)
[roles/resourcemanager.organizationAdmin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.organizationAdmin)
[roles/resourcemanager.projectCreator](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectCreator)
[roles/billing.admin](https://cloud.google.com/iam/docs/understanding-roles#billing.admin) +
[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) +| |gcp-security-admins
group|[roles/cloudasset.owner](https://cloud.google.com/iam/docs/understanding-roles#cloudasset.owner)
[roles/cloudsupport.techSupportEditor](https://cloud.google.com/iam/docs/understanding-roles#cloudsupport.techSupportEditor)
[roles/iam.securityReviewer](https://cloud.google.com/iam/docs/understanding-roles#iam.securityReviewer)
[roles/logging.admin](https://cloud.google.com/iam/docs/understanding-roles#logging.admin)
[roles/securitycenter.admin](https://cloud.google.com/iam/docs/understanding-roles#securitycenter.admin)
[roles/accesscontextmanager.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#accesscontextmanager.policyAdmin) +
[roles/iam.organizationRoleAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.organizationRoleAdmin) +
[roles/orgpolicy.policyAdmin](https://cloud.google.com/iam/docs/understanding-roles#orgpolicy.policyAdmin) +| @@ -32,5 +32,6 @@ Legend: + additive, โ€ข conditional. |---|---| |gcp-devops
group|[roles/iam.serviceAccountAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountAdmin)
[roles/iam.serviceAccountTokenCreator](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountTokenCreator) | |gcp-organization-admins
group|[roles/iam.serviceAccountTokenCreator](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountTokenCreator) | +|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +| |prod-bootstrap-0
serviceAccount|[roles/owner](https://cloud.google.com/iam/docs/understanding-roles#owner) | -|prod-resman-0
serviceAccount|[roles/iam.serviceAccountAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountAdmin)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | +|prod-resman-0
serviceAccount|[roles/iam.serviceAccountAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountAdmin)
[roles/iam.workloadIdentityPoolAdmin](https://cloud.google.com/iam/docs/understanding-roles#iam.workloadIdentityPoolAdmin)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | diff --git a/fast/stages/00-bootstrap/README.md b/fast/stages/00-bootstrap/README.md index 8edf9f199..0eabb9ac7 100644 --- a/fast/stages/00-bootstrap/README.md +++ b/fast/stages/00-bootstrap/README.md @@ -78,6 +78,23 @@ The convention is used in its full form only for specific resources with globall The [Customizations](#names-and-naming-convention) section on names below explains how to configure tokens, or implement a different naming convention. +## Workload Identity Federation and CI/CD + +This stage also implements initial support for two interrelated features + +- configuration of [Workload Identity Federation](https://cloud.google.com/iam/docs/workload-identity-federation) pools and providers +- configuration of CI/CD repositories to allow impersonation via Workload identity Federation, and stage running via provided workflow templates + +Workload Identity Federation support allows configuring external providers independently from CI/CD, and offers predefined attributes for a few well known ones (more can be easily added by editing the `identity-providers.tf` file). Once providers have been configured their names are passed to the following stages via interface outputs, and can be leveraged to set up access or impersonation in IAM bindings. + +CI/CD support is fully implemented for GitHub, Gitlab support is almost complete and will be published soon, and Cloud Source Repositories / Cloud Build will follow. + + + +For details on how to configure both features, refer to the Customizations sections below on [Workload Identity Federation](#workload-identity-federation) and [CI/CD repositories](#cicd-repositories). + +These features are optional and only enabled if the relevant variables have been populated. + ## How to run this stage This stage has straightforward initial requirements, as it is designed to work on newly created GCP organizations. Four steps are needed to bring up this stage: @@ -196,19 +213,19 @@ prefix="fast" ### Output files and cross-stage variables -At any time during the life of this stage, you can configure it to automatically generate provider configurations and variable files consumed by the following stages, to simplify passing outputs to input variables by not having to edit files manually. +Each foundational FAST stage generates provider configurations and variable files can be consumed by the following stages, and saves them in a dedicated GCS bucket in the automation project. Thise files are a handy way to simplify stage configuration, and are also used by our CI/CD workflows to configure the repository files in the pipelines that validate and apply the code. -Automatic generation of files is disabled by default. To enable the mechanism, set the `outputs_location` variable to a valid path on a local filesystem, e.g. +Alongisde the GCS stored files, you can also configure a second copy to be saves on the local filesystem, as a convenience when developing or bringing up the infrastructure before a proper CI/CD setup is in place. + +This second set of files is disabled by default, you can enable it by setting the `outputs_location` variable to a valid path on a local filesystem, e.g. ```hcl outputs_location = "~/fast-config" ``` -This is especially suited for initial bootstrapping and development. You might want to adapt it to your practices for production deployments. - Once the variable is set, `apply` will generate and manage providers and variables files, including the initial one used for this stage after the first run. You can then link these files in the relevant stages, instead of manually transfering outputs from one stage, to Terraform variables in another. -Below is the outline of the output files generated by all stages: +Below is the outline of the output files generated by all stages, which is identical for both the GCS and local filesystem copies: ```bash [path specified in outputs_location] @@ -221,10 +238,12 @@ Below is the outline of the output files generated by all stages: โ”‚ย ย  โ”œโ”€โ”€ 03-project-factory-prod-providers.tf โ”‚ย ย  โ””โ”€โ”€ 99-sandbox-providers.tf โ””โ”€โ”€ tfvars - โ”œโ”€โ”€ 00-bootstrap.auto.tfvars.json - โ”œโ”€โ”€ 01-resman.auto.tfvars.json - โ”œโ”€โ”€ 02-networking.auto.tfvars.json - โ””โ”€โ”€ 02-security.auto.tfvars.json +โ”‚ โ”œโ”€โ”€ 00-bootstrap.auto.tfvars.json +โ”‚ โ”œโ”€โ”€ 01-resman.auto.tfvars.json +โ”‚ โ”œโ”€โ”€ 02-networking.auto.tfvars.json +โ”‚ โ””โ”€โ”€ 02-security.auto.tfvars.json +โ””โ”€โ”€ workflows + โ””โ”€โ”€ [optional depending on the configured CI/CD repositories] ``` ### Running the stage @@ -322,6 +341,71 @@ If a different convention is needed, identify names via search/grep (e.g. with ` Names used in internal references (e.g. `module.foo-prod.id`) are only used by Terraform and do not influence resource naming, so they are best left untouched to avoid having to debug complex errors. +### Workload Identity Federation + +At any time during this stage's lifecycle you can configure a Workload Identity Federation pool, and one or more providers. These are part of this stage's interface, included in the automatically generated `.tfvars` files and accepted by the Resource Managent stage that follows. + +The variable maps each provider's `issuer` attribute with the definitions in the `identity-providers.tf` file. We currently support GitHub and Gitlab directly, and extending to definitions to support more providers is trivial (send us a PR if you do!). + +Provider key names are used by the `cicd_repositories` variable to configure authentication for CI/CD repositories, and generally from your Terraform code whenever you need to configure IAM access or impersonation for federated identities. + +This is a sample configuration of a GitHub provider, the `attribute_condition` attribute can be set to null if needed: + +```hcl +federated_identity_providers = { + github-sample = { + attribute_condition = "attribute.repository_owner==\"my-github-org\"" + issuer = "github" + } +} +``` + +### CI/CD repositories + +FAST is designed to directly support running in automated workflows from separate repositories for each stage. The `cicd_repositories` variable allows you to configure impersonation from external repositories leveraging Workload identity Federation, and pre-configures a FAST workflow file that can be used to validate and apply the code in each repository. + +The repository design we support is fairly simple, with a repository for modules that enables centralization and versioning, and one repository for each stage optionally configured from the previous stage. + +This is an example of configuring the bootstrap and resource management repositories in this stage. CI/CD configuration is optional, so the entire variable or any of its attributes can be set to null if not needed. + +```hcl +cicd_repositories = { + bootstrap = { + branch = null + identity_provider = "github-sample" + name = "my-gh-org/fast-bootstrap" + type = "github" + } + resman = { + branch = "main" + identity_provider = "github-sample" + name = "my-gh-org/fast-resman" + type = "github" + } +} +``` + +Once the stage is applied the generated output files will contain pre-configured workflow files for each repository, that will use Workload Identity Federation via a dedicated service account for each repository to impersonate the automation service account for the stage. + +The remaining configuration is manual, as it regards the repositories themselves: + +- create a repository for modules + - clone and populate it with the Fabric modules + - configure authentication to the modules repository + - for GitHub + - create a key pair + - create a [deploy key](https://docs.github.com/en/developers/overview/managing-deploy-keys#deploy-keys) in the modules repository with the public key + - create a `CICD_MODULES_KEY` secret with the private key in each of the repositories that need to access modules +- create one repository for each stage + - clone and populate them with the stage source + - edit the modules source to match your modules repository + - a simple way is using the "Replace in files" function of your editor + - search for `source\s*= "../../../modules/([^"]+)"` + - replace with `source = "git@github.com:my-org/fast-modules.git//$1?ref=v1.0"` + - copy the generated workflow file for the stage from the GCS output files bucket or from the local clone if enabled + - for GitHub, place it in a `.github/workflows` folder in the repository root + - for Gitlab, rename it to `.gitlab-ci.yml` and place it in the repository root + @@ -331,10 +415,14 @@ Names used in internal references (e.g. `module.foo-prod.id`) are only used by T |---|---|---|---| | [automation.tf](./automation.tf) | Automation project and resources. | gcs ยท iam-service-account ยท project | | | [billing.tf](./billing.tf) | Billing export project and dataset. | bigquery-dataset ยท organization ยท project | google_billing_account_iam_member ยท google_organization_iam_binding | +| [cicd.tf](./cicd.tf) | Workload Identity Federation configurations for CI/CD. | iam-service-account | | +| [identity-providers.tf](./identity-providers.tf) | Workload Identity Federation provider definitions. | | google_iam_workload_identity_pool ยท google_iam_workload_identity_pool_provider | | [log-export.tf](./log-export.tf) | Audit log project and sink. | bigquery-dataset ยท gcs ยท logging-bucket ยท project ยท pubsub | | | [main.tf](./main.tf) | Module-level locals and resources. | | | | [organization.tf](./organization.tf) | Organization-level IAM. | organization | google_organization_iam_binding | -| [outputs.tf](./outputs.tf) | Module outputs. | | local_file | +| [outputs-files.tf](./outputs-files.tf) | Output files persistence to local filesystem. | | local_file | +| [outputs-gcs.tf](./outputs-gcs.tf) | Output files persistence to automation GCS bucket. | | google_storage_bucket_object | +| [outputs.tf](./outputs.tf) | Module outputs. | | | | [variables.tf](./variables.tf) | Module variables. | | | ## Variables @@ -342,24 +430,31 @@ Names used in internal references (e.g. `module.foo-prod.id`) are only used by T | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| | [billing_account](variables.tf#L17) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | โœ“ | | | -| [organization](variables.tf#L96) | Organization details. | object({…}) | โœ“ | | | -| [prefix](variables.tf#L111) | Prefix used for resources that need unique names. Use 9 characters or less. | string | โœ“ | | | +| [organization](variables.tf#L146) | Organization details. | object({…}) | โœ“ | | | +| [prefix](variables.tf#L161) | Prefix used for resources that need unique names. Use 9 characters or less. | string | โœ“ | | | | [bootstrap_user](variables.tf#L25) | Email of the nominal user running this stage for the first time. | string | | null | | -| [custom_role_names](variables.tf#L31) | Names of custom roles defined at the org level. | object({…}) | | {…} | | -| [groups](variables.tf#L43) | Group names to grant organization-level permissions. | map(string) | | {…} | | -| [iam](variables.tf#L57) | Organization-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | | -| [iam_additive](variables.tf#L63) | Organization-level custom IAM settings in role => [principal] format for non-authoritative bindings. | map(list(string)) | | {} | | -| [log_sinks](variables.tf#L71) | Org-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | | -| [outputs_location](variables.tf#L105) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | | +| [cicd_repositories](variables.tf#L31) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | | +| [custom_role_names](variables.tf#L71) | Names of custom roles defined at the org level. | object({…}) | | {…} | | +| [federated_identity_providers](variables.tf#L83) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | map(object({…})) | | {} | | +| [groups](variables.tf#L93) | Group names to grant organization-level permissions. | map(string) | | {…} | | +| [iam](variables.tf#L107) | Organization-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | | +| [iam_additive](variables.tf#L113) | Organization-level custom IAM settings in role => [principal] format for non-authoritative bindings. | map(list(string)) | | {} | | +| [log_sinks](variables.tf#L121) | Org-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | | +| [outputs_location](variables.tf#L155) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | string | | null | | ## Outputs | name | description | sensitive | consumers | |---|---|:---:|---| -| [billing_dataset](outputs.tf#L58) | BigQuery dataset prepared for billing export. | | | -| [custom_roles](outputs.tf#L63) | Organization-level custom roles. | | | -| [project_ids](outputs.tf#L68) | Projects created by this stage. | | | -| [providers](outputs.tf#L79) | Terraform provider files for this stage and dependent stages. | โœ“ | stage-01 | -| [tfvars](outputs.tf#L88) | Terraform variable files for the following stages. | โœ“ | | +| [automation](outputs.tf#L89) | Automation resources. | | | +| [billing_dataset](outputs.tf#L94) | BigQuery dataset prepared for billing export. | | | +| [cicd_repositories](outputs.tf#L99) | CI/CD repository configurations. | | | +| [custom_roles](outputs.tf#L111) | Organization-level custom roles. | | | +| [federated_identity](outputs.tf#L116) | Workload Identity Federation pool and providers. | | | +| [outputs_bucket](outputs.tf#L126) | GCS bucket where generated output files are stored. | | | +| [project_ids](outputs.tf#L131) | Projects created by this stage. | | | +| [providers](outputs.tf#L150) | Terraform provider files for this stage and dependent stages. | โœ“ | stage-01 | +| [service_accounts](outputs.tf#L140) | Automation service accounts created by this stage. | | | +| [tfvars](outputs.tf#L159) | Terraform variable files for the following stages. | โœ“ | | diff --git a/fast/stages/00-bootstrap/automation.tf b/fast/stages/00-bootstrap/automation.tf index d5bf3a4a4..1caaf94ca 100644 --- a/fast/stages/00-bootstrap/automation.tf +++ b/fast/stages/00-bootstrap/automation.tf @@ -30,6 +30,7 @@ module "automation-project" { ] (local.groups.gcp-organization-admins) = [ "roles/iam.serviceAccountTokenCreator", + "roles/iam.workloadIdentityPoolAdmin" ] } # machine (service accounts) IAM bindings @@ -40,6 +41,9 @@ module "automation-project" { "roles/iam.serviceAccountAdmin" = [ module.automation-tf-resman-sa.iam_email ] + "roles/iam.workloadIdentityPoolAdmin" = [ + module.automation-tf-resman-sa.iam_email + ] "roles/storage.admin" = [ module.automation-tf-resman-sa.iam_email ] @@ -57,15 +61,28 @@ module "automation-project" { "compute.googleapis.com", "essentialcontacts.googleapis.com", "iam.googleapis.com", + "iamcredentials.googleapis.com", "pubsub.googleapis.com", "servicenetworking.googleapis.com", "serviceusage.googleapis.com", "stackdriver.googleapis.com", "storage-component.googleapis.com", "storage.googleapis.com", + "sts.googleapis.com" ] } +# outputt files bucket + +module "automation-tf-output-gcs" { + source = "../../../modules/gcs" + project_id = module.automation-project.project_id + name = "iac-core-outputs-0" + prefix = local.prefix + versioning = true + depends_on = [module.organization] +} + # this stage's bucket and service account module "automation-tf-bootstrap-gcs" { @@ -83,6 +100,14 @@ module "automation-tf-bootstrap-sa" { name = "bootstrap-0" description = "Terraform organization bootstrap service account." prefix = local.prefix + iam = { + "roles/iam.serviceAccountTokenCreator" = compact([ + try(module.automation-tf-cicd-sa["bootstrap"].iam_email, null) + ]) + } + iam_storage_roles = { + (module.automation-tf-output-gcs.name) = ["roles/storage.admin"] + } } # resource hierarchy stage's bucket and service account @@ -103,6 +128,14 @@ module "automation-tf-resman-sa" { source = "../../../modules/iam-service-account" project_id = module.automation-project.project_id name = "resman-0" - description = "Terraform organization bootstrap service account." + description = "Terraform stage 1 resman service account." prefix = local.prefix + iam = { + "roles/iam.serviceAccountTokenCreator" = compact([ + try(module.automation-tf-cicd-sa["resman"].iam_email, null) + ]) + } + iam_storage_roles = { + (module.automation-tf-output-gcs.name) = ["roles/storage.admin"] + } } diff --git a/fast/stages/00-bootstrap/cicd.tf b/fast/stages/00-bootstrap/cicd.tf new file mode 100644 index 000000000..f4c8c6c28 --- /dev/null +++ b/fast/stages/00-bootstrap/cicd.tf @@ -0,0 +1,63 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Workload Identity Federation configurations for CI/CD. + +locals { + # TODO: map null provider to Cloud Build once we add support for it + cicd_repositories = { + for k, v in coalesce(var.cicd_repositories, {}) : k => v + if( + v != null + && + contains(keys(local.identity_providers), v.identity_provider) + && + fileexists("${path.module}/templates/workflow-${v.type}.yaml") + ) + } + cicd_service_accounts = { + for k, v in module.automation-tf-cicd-sa : + k => v.iam_email + } +} + +module "automation-tf-cicd-sa" { + source = "../../../modules/iam-service-account" + for_each = local.cicd_repositories + project_id = module.automation-project.project_id + name = "${each.key}-1" + description = "Terraform CI/CD stage 1 ${each.key} service account." + prefix = local.prefix + iam = { + "roles/iam.workloadIdentityUser" = [ + each.value.branch == null + ? format( + local.identity_providers_defs[each.value.type].principalset_tpl, + google_iam_workload_identity_pool.default.0.name, + each.value.name + ) + : format( + local.identity_providers_defs[each.value.type].principal_tpl, + google_iam_workload_identity_pool.default.0.name, + each.value.name, + each.value.branch + ) + ] + } + iam_storage_roles = { + (module.automation-tf-output-gcs.name) = ["roles/storage.objectViewer"] + } +} diff --git a/fast/stages/00-bootstrap/identity-providers.tf b/fast/stages/00-bootstrap/identity-providers.tf new file mode 100644 index 000000000..925b8eed0 --- /dev/null +++ b/fast/stages/00-bootstrap/identity-providers.tf @@ -0,0 +1,75 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Workload Identity Federation provider definitions. + +locals { + identity_providers = { + for k, v in var.federated_identity_providers : k => merge( + v, lookup(local.identity_providers_defs, v.issuer, {}) + ) + } + identity_providers_defs = { + github = { + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.sub" = "assertion.sub" + "attribute.actor" = "assertion.actor" + "attribute.repository" = "assertion.repository" + "attribute.ref" = "assertion.ref" + } + issuer_uri = "https://token.actions.githubusercontent.com" + principal_tpl = "principal://iam.googleapis.com/%s/subject/repo:%s:ref:refs/heads/%s" + principalset_tpl = "principalSet://iam.googleapis.com/%s/attribute.repository/%s" + } + gitlab = { + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.sub" = "assertion.sub" + "attribute.actor" = "assertion.actor" + "attribute.repository" = "assertion.repository" + "attribute.ref" = "assertion.ref" + } + allowed_audiences = ["https://gitlab.com"] + issuer_uri = "https://gitlab.com" + principal_tpl = "principal://iam.googleapis.com/%s/subject/project_path:%s:ref_type:branch:ref:%s" + principalset_tpl = "principalSet://iam.googleapis.com/%s/attribute.repository/%s" + } + } +} + +resource "google_iam_workload_identity_pool" "default" { + provider = google-beta + count = length(local.identity_providers) > 0 ? 1 : 0 + project = module.automation-project.project_id + workload_identity_pool_id = "${var.prefix}-bootstrap" +} + +resource "google_iam_workload_identity_pool_provider" "default" { + provider = google-beta + for_each = local.identity_providers + project = module.automation-project.project_id + workload_identity_pool_id = ( + google_iam_workload_identity_pool.default.0.workload_identity_pool_id + ) + workload_identity_pool_provider_id = "${var.prefix}-bootstrap-${each.key}" + attribute_condition = each.value.attribute_condition + attribute_mapping = each.value.attribute_mapping + oidc { + allowed_audiences = try(each.value.allowed_audiences, null) + issuer_uri = each.value.issuer_uri + } +} diff --git a/fast/stages/00-bootstrap/outputs-files.tf b/fast/stages/00-bootstrap/outputs-files.tf new file mode 100644 index 000000000..3016c8e21 --- /dev/null +++ b/fast/stages/00-bootstrap/outputs-files.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Output files persistence to local filesystem. + +resource "local_file" "providers" { + for_each = var.outputs_location == null ? {} : local.providers + file_permission = "0644" + filename = "${pathexpand(var.outputs_location)}/providers/${each.key}-providers.tf" + content = each.value +} + +resource "local_file" "tfvars" { + for_each = var.outputs_location == null ? {} : { 1 = 1 } + file_permission = "0644" + filename = "${pathexpand(var.outputs_location)}/tfvars/00-bootstrap.auto.tfvars.json" + content = jsonencode(local.tfvars) +} + +resource "local_file" "tfvars_globals" { + for_each = var.outputs_location == null ? {} : { 1 = 1 } + file_permission = "0644" + filename = "${pathexpand(var.outputs_location)}/tfvars/globals.auto.tfvars.json" + content = jsonencode(local.tfvars_globals) +} + +resource "local_file" "workflows" { + for_each = local.cicd_workflows + file_permission = "0644" + filename = "${pathexpand(var.outputs_location)}/workflows/${each.key}-workflow.yaml" + content = each.value +} diff --git a/fast/stages/00-bootstrap/outputs-gcs.tf b/fast/stages/00-bootstrap/outputs-gcs.tf new file mode 100644 index 000000000..2c281d4cc --- /dev/null +++ b/fast/stages/00-bootstrap/outputs-gcs.tf @@ -0,0 +1,44 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Output files persistence to automation GCS bucket. + +resource "google_storage_bucket_object" "providers" { + for_each = local.providers + bucket = module.automation-tf-output-gcs.name + # provider suffix allows excluding via .gitignore when linked from stages + name = "providers/${each.key}-providers.tf" + content = each.value +} + +resource "google_storage_bucket_object" "tfvars" { + bucket = module.automation-tf-output-gcs.name + name = "tfvars/00-bootstrap.auto.tfvars.json" + content = jsonencode(local.tfvars) +} + +resource "google_storage_bucket_object" "tfvars_globals" { + bucket = module.automation-tf-output-gcs.name + name = "tfvars/globals.auto.tfvars.json" + content = jsonencode(local.tfvars_globals) +} + +resource "google_storage_bucket_object" "workflows" { + for_each = local.cicd_workflows + bucket = module.automation-tf-output-gcs.name + name = "workflows/${each.key}-workflow.yaml" + content = each.value +} diff --git a/fast/stages/00-bootstrap/outputs.tf b/fast/stages/00-bootstrap/outputs.tf index 5779d5b28..45595c9eb 100644 --- a/fast/stages/00-bootstrap/outputs.tf +++ b/fast/stages/00-bootstrap/outputs.tf @@ -15,56 +15,119 @@ */ locals { + _cicd_workflow_attrs = { + bootstrap = { + service_account = module.automation-tf-bootstrap-sa.email + tf_providers_file = "00-bootstrap-providers.tf" + tf_var_files = [] + } + resman = { + service_account = module.automation-tf-resman-sa.email + tf_providers_file = "01-resman-providers.tf" + tf_var_files = [ + "00-bootstrap.auto.tfvars.json", + "globals.auto.tfvars.json" + ] + } + } + _tpl_providers = "${path.module}/templates/providers.tf.tpl" + cicd_workflows = { + for k, v in local.cicd_repositories : k => templatefile( + "${path.module}/templates/workflow-${v.type}.yaml", + merge(local._cicd_workflow_attrs[k], { + identity_provider = local.wif_providers[v["identity_provider"]].name + outputs_bucket = module.automation-tf-output-gcs.name + stage_name = k + }) + ) + } custom_roles = { for k, v in var.custom_role_names : k => try(module.organization.custom_role_id[v], null) } providers = { - "00-bootstrap" = templatefile("${path.module}/../../assets/templates/providers.tpl", { + "00-bootstrap" = templatefile(local._tpl_providers, { bucket = module.automation-tf-bootstrap-gcs.name name = "bootstrap" sa = module.automation-tf-bootstrap-sa.email }) - "01-resman" = templatefile("${path.module}/../../assets/templates/providers.tpl", { + "01-resman" = templatefile(local._tpl_providers, { bucket = module.automation-tf-resman-gcs.name name = "resman" sa = module.automation-tf-resman-sa.email }) } tfvars = { - automation_project_id = module.automation-project.project_id - custom_roles = local.custom_roles + automation = { + federated_identity_pool = try( + google_iam_workload_identity_pool.default.0.name, null + ) + federated_identity_providers = local.wif_providers + outputs_bucket = module.automation-tf-output-gcs.name + project_id = module.automation-project.project_id + } + custom_roles = local.custom_roles + } + tfvars_globals = { + billing_account = var.billing_account + groups = var.groups + organization = var.organization + prefix = var.prefix + } + wif_providers = { + for k, v in google_iam_workload_identity_pool_provider.default : + k => { + issuer = local.identity_providers[k].issuer + issuer_uri = local.identity_providers[k].issuer_uri + name = v.name + principal_tpl = local.identity_providers[k].principal_tpl + principalset_tpl = local.identity_providers[k].principalset_tpl + } } } -# optionally generate providers and tfvars files for subsequent stages - -resource "local_file" "providers" { - for_each = var.outputs_location == null ? {} : local.providers - file_permission = "0644" - filename = "${pathexpand(var.outputs_location)}/providers/${each.key}-providers.tf" - content = each.value +output "automation" { + description = "Automation resources." + value = local.tfvars.automation } -resource "local_file" "tfvars" { - for_each = var.outputs_location == null ? {} : { 1 = 1 } - file_permission = "0644" - filename = "${pathexpand(var.outputs_location)}/tfvars/00-bootstrap.auto.tfvars.json" - content = jsonencode(local.tfvars) -} - -# outputs - output "billing_dataset" { description = "BigQuery dataset prepared for billing export." value = try(module.billing-export-dataset.0.id, null) } +output "cicd_repositories" { + description = "CI/CD repository configurations." + value = { + for k, v in local.cicd_repositories : k => { + branch = v.branch + name = v.name + provider = local.wif_providers[v.identity_provider].name + service_account = module.automation-tf-cicd-sa[k].email + } + } +} + output "custom_roles" { description = "Organization-level custom roles." value = local.custom_roles } +output "federated_identity" { + description = "Workload Identity Federation pool and providers." + value = { + pool = try( + google_iam_workload_identity_pool.default.0.name, null + ) + providers = local.wif_providers + } +} + +output "outputs_bucket" { + description = "GCS bucket where generated output files are stored." + value = module.automation-tf-output-gcs.name +} + output "project_ids" { description = "Projects created by this stage." value = { @@ -74,6 +137,14 @@ output "project_ids" { } } +output "service_accounts" { + description = "Automation service accounts created by this stage." + value = { + bootstrap = module.automation-tf-bootstrap-sa.email + resman = module.automation-tf-resman-sa.email + } +} + # ready to use provider configurations for subsequent stages when not using files output "providers" { diff --git a/fast/stages/00-bootstrap/templates b/fast/stages/00-bootstrap/templates new file mode 120000 index 000000000..bcb6967be --- /dev/null +++ b/fast/stages/00-bootstrap/templates @@ -0,0 +1 @@ +../../assets/templates \ No newline at end of file diff --git a/fast/stages/00-bootstrap/variables.tf b/fast/stages/00-bootstrap/variables.tf index 68d54650c..1eb12c866 100644 --- a/fast/stages/00-bootstrap/variables.tf +++ b/fast/stages/00-bootstrap/variables.tf @@ -28,6 +28,46 @@ variable "bootstrap_user" { default = null } +variable "cicd_repositories" { + # TODO: edit description once we add support for Cloud Build (null provider) + description = "CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed." + type = object({ + bootstrap = object({ + branch = string + identity_provider = string + name = string + type = string + }) + resman = object({ + branch = string + identity_provider = string + name = string + type = string + }) + }) + default = null + validation { + condition = alltrue([ + for k, v in coalesce(var.cicd_repositories, {}) : + v == null || ( + try(v.name, null) != null + && + try(v.identity_provider, null) != null + ) + ]) + error_message = "Non-null repositories need non-null name and providers." + } + validation { + condition = alltrue([ + for k, v in coalesce(var.cicd_repositories, {}) : + v == null || ( + contains(["github"], coalesce(try(v.type, null), "null")) + ) + ]) + error_message = "Invalid repository type, supported types: 'github'." + } +} + variable "custom_role_names" { description = "Names of custom roles defined at the org level." type = object({ @@ -40,6 +80,16 @@ variable "custom_role_names" { } } +variable "federated_identity_providers" { + description = "Workload Identity Federation pools. The `cicd_repositories` variable references keys here." + type = map(object({ + attribute_condition = string + issuer = string + })) + default = {} + nullable = false +} + variable "groups" { # https://cloud.google.com/docs/enterprise/setup-checklist description = "Group names to grant organization-level permissions." @@ -103,7 +153,7 @@ variable "organization" { } variable "outputs_location" { - description = "Path where providers and tfvars files for the following stages are written. Leave empty to disable." + description = "Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable" type = string default = null } diff --git a/fast/stages/01-resman/README.md b/fast/stages/01-resman/README.md index 3ac9c09e3..30a86830e 100644 --- a/fast/stages/01-resman/README.md +++ b/fast/stages/01-resman/README.md @@ -36,6 +36,12 @@ Additionally, a few critical benefits are directly provided by this design: For a discussion on naming, please refer to the [Bootstrap stage documentation](../00-bootstrap/README.md#naming), as the same approach is shared by all stages. +### Workload Identity Federation and CI/CD + +This stage also implements optional support for CI/CD, much in the same way as the bootstrap stage. The only difference is on Workload Identity Federation, which is only configured in bootstrap and made available here via stage interface variables (the automatically generated `.tfvars` files). + +For details on how to configure CI/CD please refer to the [relevant section in the bootstrap stage documentation](../00-bootstrap/README.md#cicd-repositories). + ## How to run this stage This stage is meant to be executed after the [bootstrap](../00-bootstrap) stage has run, as it leverages the automation service account and bucket created there. The relevant user groups must also exist, but that's one of the requirements for the previous stage too, so if you ran that successfully, you're good to go. @@ -156,37 +162,41 @@ Due to its simplicity, this stage lends itself easily to customizations: adding | [branch-networking.tf](./branch-networking.tf) | Networking stage resources. | folder ยท gcs ยท iam-service-account | | | [branch-sandbox.tf](./branch-sandbox.tf) | Sandbox stage resources. | folder ยท gcs ยท iam-service-account | | | [branch-security.tf](./branch-security.tf) | Security stage resources. | folder ยท gcs ยท iam-service-account | | -| [branch-teams.tf](./branch-teams.tf) | Team stages resources. | folder ยท gcs ยท iam-service-account | | +| [branch-teams.tf](./branch-teams.tf) | Team stage resources. | folder ยท gcs ยท iam-service-account | | | [main.tf](./main.tf) | Module-level locals and resources. | | | | [organization.tf](./organization.tf) | Organization policies. | organization | google_organization_iam_member | -| [outputs.tf](./outputs.tf) | Module outputs. | | local_file | +| [outputs-files.tf](./outputs-files.tf) | Output files persistence to local filesystem. | | local_file | +| [outputs-gcs.tf](./outputs-gcs.tf) | Output files persistence to automation GCS bucket. | | google_storage_bucket_object | +| [outputs.tf](./outputs.tf) | Module outputs. | | | | [variables.tf](./variables.tf) | Module variables. | | | ## Variables | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| -| [automation_project_id](variables.tf#L20) | Project id for the automation project created by the bootstrap stage. | string | โœ“ | | 00-bootstrap | -| [billing_account](variables.tf#L26) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | โœ“ | | 00-bootstrap | -| [organization](variables.tf#L59) | Organization details. | object({…}) | โœ“ | | 00-bootstrap | -| [prefix](variables.tf#L83) | Prefix used for resources that need unique names. Use 9 characters or less. | string | โœ“ | | 00-bootstrap | -| [custom_roles](variables.tf#L35) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap | -| [groups](variables.tf#L44) | Group names to grant organization-level permissions. | map(string) | | {…} | 00-bootstrap | -| [organization_policy_configs](variables.tf#L69) | Organization policies customization. | object({…}) | | null | | -| [outputs_location](variables.tf#L77) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | | -| [team_folders](variables.tf#L94) | Team folders to be created. Format is described in a code comment. | map(object({…})) | | null | | +| [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | object({…}) | โœ“ | | 00-bootstrap | +| [billing_account](variables.tf#L37) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | โœ“ | | 00-bootstrap | +| [organization](variables.tf#L133) | Organization details. | object({…}) | โœ“ | | 00-bootstrap | +| [prefix](variables.tf#L157) | Prefix used for resources that need unique names. Use 9 characters or less. | string | โœ“ | | 00-bootstrap | +| [cicd_repositories](variables.tf#L46) | CI/CD repository configuration. Identity providers reference keys in the `automation.federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | | +| [custom_roles](variables.tf#L109) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap | +| [groups](variables.tf#L118) | Group names to grant organization-level permissions. | map(string) | | {…} | 00-bootstrap | +| [organization_policy_configs](variables.tf#L143) | Organization policies customization. | object({…}) | | null | | +| [outputs_location](variables.tf#L151) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | string | | null | | +| [team_folders](variables.tf#L168) | Team folders to be created. Format is described in a code comment. | map(object({…})) | | null | | ## Outputs | name | description | sensitive | consumers | |---|---|:---:|---| -| [dataplatform](outputs.tf#L114) | Data for the Data Platform stage. | | | -| [networking](outputs.tf#L130) | Data for the networking stage. | | | -| [project_factories](outputs.tf#L139) | Data for the project factories stage. | | | -| [providers](outputs.tf#L155) | Terraform provider files for this stage and dependent stages. | โœ“ | 02-networking ยท 02-security ยท 03-dataplatform ยท xx-sandbox ยท xx-teams | -| [sandbox](outputs.tf#L162) | Data for the sandbox stage. | | xx-sandbox | -| [security](outputs.tf#L172) | Data for the networking stage. | | 02-security | -| [teams](outputs.tf#L182) | Data for the teams stage. | | | -| [tfvars](outputs.tf#L195) | Terraform variable files for the following stages. | โœ“ | | +| [cicd_repositories](outputs.tf#L156) | WIF configuration for CI/CD repositories. | | | +| [dataplatform](outputs.tf#L168) | Data for the Data Platform stage. | | | +| [networking](outputs.tf#L184) | Data for the networking stage. | | | +| [project_factories](outputs.tf#L193) | Data for the project factories stage. | | | +| [providers](outputs.tf#L209) | Terraform provider files for this stage and dependent stages. | โœ“ | 02-networking ยท 02-security ยท 03-dataplatform ยท xx-sandbox ยท xx-teams | +| [sandbox](outputs.tf#L216) | Data for the sandbox stage. | | xx-sandbox | +| [security](outputs.tf#L226) | Data for the networking stage. | | 02-security | +| [teams](outputs.tf#L236) | Data for the teams stage. | | | +| [tfvars](outputs.tf#L249) | Terraform variable files for the following stages. | โœ“ | | diff --git a/fast/stages/01-resman/branch-data-platform.tf b/fast/stages/01-resman/branch-data-platform.tf index d5eeafa0e..d518c9c15 100644 --- a/fast/stages/01-resman/branch-data-platform.tf +++ b/fast/stages/01-resman/branch-data-platform.tf @@ -16,8 +16,6 @@ # tfdoc:file:description Data Platform stages resources. -# top-level Data Platform folder and service account - module "branch-dp-folder" { source = "../../../modules/folder" parent = "organizations/${var.organization.id}" @@ -27,8 +25,6 @@ module "branch-dp-folder" { } } -# environment: development folder - module "branch-dp-dev-folder" { source = "../../../modules/folder" parent = module.branch-dp-folder.id @@ -47,27 +43,6 @@ module "branch-dp-dev-folder" { } } -module "branch-dp-dev-sa" { - source = "../../../modules/iam-service-account" - project_id = var.automation_project_id - name = "dev-resman-dp-0" - description = "Terraform Data Platform development service account." - prefix = var.prefix -} - -module "branch-dp-dev-gcs" { - source = "../../../modules/gcs" - project_id = var.automation_project_id - name = "dev-resman-dp-0" - prefix = var.prefix - versioning = true - iam = { - "roles/storage.objectAdmin" = [module.branch-dp-dev-sa.iam_email] - } -} - -# environment: production folder - module "branch-dp-prod-folder" { source = "../../../modules/folder" parent = module.branch-dp-folder.id @@ -86,17 +61,54 @@ module "branch-dp-prod-folder" { } } +# automation service accounts and buckets + +module "branch-dp-dev-sa" { + source = "../../../modules/iam-service-account" + project_id = var.automation.project_id + name = "dev-resman-dp-0" + description = "Terraform data platform development service account." + prefix = var.prefix + iam = { + "roles/iam.serviceAccountTokenCreator" = compact([ + try(module.branch-dp-dev-sa-cicd.0.iam_email, null) + ]) + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.admin"] + } +} + module "branch-dp-prod-sa" { source = "../../../modules/iam-service-account" - project_id = var.automation_project_id + project_id = var.automation.project_id name = "prod-resman-dp-0" - description = "Terraform Data Platform production service account." + description = "Terraform data platform production service account." prefix = var.prefix + iam = { + "roles/iam.serviceAccountTokenCreator" = compact([ + try(module.branch-dp-prod-sa-cicd.0.iam_email, null) + ]) + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.admin"] + } +} + +module "branch-dp-dev-gcs" { + source = "../../../modules/gcs" + project_id = var.automation.project_id + name = "dev-resman-dp-0" + prefix = var.prefix + versioning = true + iam = { + "roles/storage.objectAdmin" = [module.branch-dp-dev-sa.iam_email] + } } module "branch-dp-prod-gcs" { source = "../../../modules/gcs" - project_id = var.automation_project_id + project_id = var.automation.project_id name = "prod-resman-dp-0" prefix = var.prefix versioning = true @@ -104,3 +116,67 @@ module "branch-dp-prod-gcs" { "roles/storage.objectAdmin" = [module.branch-dp-prod-sa.iam_email] } } + +# ci/cd service accounts + +module "branch-dp-dev-sa-cicd" { + source = "../../../modules/iam-service-account" + for_each = ( + lookup(local.cicd_repositories, "dp_dev", null) == null + ? {} + : { 0 = local.cicd_repositories.dp_dev } + ) + project_id = var.automation.project_id + name = "dev-resman-dp-1" + description = "Terraform CI/CD data platform development service account." + prefix = var.prefix + iam = { + "roles/iam.workloadIdentityUser" = [ + each.value.branch == null + ? format( + local.identity_providers[each.value.identity_provider].principalset_tpl, + each.value.name + ) + : format( + local.identity_providers[each.value.identity_provider].principal_tpl, + each.value.name, + each.value.branch + ) + ] + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] + } +} + +module "branch-dp-prod-sa-cicd" { + source = "../../../modules/iam-service-account" + for_each = ( + lookup(local.cicd_repositories, "dp_prod", null) == null + ? {} + : { 0 = local.cicd_repositories.dp_prod } + ) + project_id = var.automation.project_id + name = "prod-resman-dp-1" + description = "Terraform CI/CD data platform production service account." + prefix = var.prefix + iam = { + "roles/iam.workloadIdentityUser" = [ + each.value.branch == null + ? format( + local.identity_providers[each.value.identity_provider].principalset_tpl, + var.automation.federated_identity_pool, + each.value.name + ) + : format( + local.identity_providers[each.value.identity_provider].principal_tpl, + var.automation.federated_identity_pool, + each.value.name, + each.value.branch + ) + ] + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] + } +} diff --git a/fast/stages/01-resman/branch-networking.tf b/fast/stages/01-resman/branch-networking.tf index 105c302fd..3d85f1be7 100644 --- a/fast/stages/01-resman/branch-networking.tf +++ b/fast/stages/01-resman/branch-networking.tf @@ -43,25 +43,6 @@ module "branch-network-folder" { } } -module "branch-network-sa" { - source = "../../../modules/iam-service-account" - project_id = var.automation_project_id - name = "prod-resman-net-0" - description = "Terraform resman networking service account." - prefix = var.prefix -} - -module "branch-network-gcs" { - source = "../../../modules/gcs" - project_id = var.automation_project_id - name = "prod-resman-net-0" - prefix = var.prefix - versioning = true - iam = { - "roles/storage.objectAdmin" = [module.branch-network-sa.iam_email] - } -} - module "branch-network-prod-folder" { source = "../../../modules/folder" parent = module.branch-network-folder.id @@ -91,3 +72,66 @@ module "branch-network-dev-folder" { environment = try(module.organization.tag_values["environment/development"].id, null) } } + +# automation service account and bucket + +module "branch-network-sa" { + source = "../../../modules/iam-service-account" + project_id = var.automation.project_id + name = "prod-resman-net-0" + description = "Terraform resman networking service account." + prefix = var.prefix + iam = { + "roles/iam.serviceAccountTokenCreator" = compact([ + try(module.branch-network-sa-cicd.0.iam_email, null) + ]) + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.admin"] + } +} + +module "branch-network-gcs" { + source = "../../../modules/gcs" + project_id = var.automation.project_id + name = "prod-resman-net-0" + prefix = var.prefix + versioning = true + iam = { + "roles/storage.objectAdmin" = [module.branch-network-sa.iam_email] + } +} + +# ci/cd service account + +module "branch-network-sa-cicd" { + source = "../../../modules/iam-service-account" + for_each = ( + lookup(local.cicd_repositories, "networking", null) == null + ? {} + : { 0 = local.cicd_repositories.networking } + ) + project_id = var.automation.project_id + name = "prod-resman-net-1" + description = "Terraform CI/CD stage 2 networking service account." + prefix = var.prefix + iam = { + "roles/iam.workloadIdentityUser" = [ + each.value.branch == null + ? format( + local.identity_providers[each.value.identity_provider].principalset_tpl, + var.automation.federated_identity_pool, + each.value.name + ) + : format( + local.identity_providers[each.value.identity_provider].principal_tpl, + var.automation.federated_identity_pool, + each.value.name, + each.value.branch + ) + ] + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] + } +} diff --git a/fast/stages/01-resman/branch-sandbox.tf b/fast/stages/01-resman/branch-sandbox.tf index 70772cbcb..dda4b1fcc 100644 --- a/fast/stages/01-resman/branch-sandbox.tf +++ b/fast/stages/01-resman/branch-sandbox.tf @@ -44,7 +44,7 @@ module "branch-sandbox-folder" { module "branch-sandbox-gcs" { source = "../../../modules/gcs" - project_id = var.automation_project_id + project_id = var.automation.project_id name = "dev-resman-sbox-0" prefix = var.prefix versioning = true @@ -55,7 +55,7 @@ module "branch-sandbox-gcs" { module "branch-sandbox-sa" { source = "../../../modules/iam-service-account" - project_id = var.automation_project_id + project_id = var.automation.project_id name = "dev-resman-sbox-0" description = "Terraform resman sandbox service account." prefix = var.prefix diff --git a/fast/stages/01-resman/branch-security.tf b/fast/stages/01-resman/branch-security.tf index 690247626..bba54b6cf 100644 --- a/fast/stages/01-resman/branch-security.tf +++ b/fast/stages/01-resman/branch-security.tf @@ -44,17 +44,27 @@ module "branch-security-folder" { } } +# automation service account and bucket + module "branch-security-sa" { source = "../../../modules/iam-service-account" - project_id = var.automation_project_id + project_id = var.automation.project_id name = "prod-resman-sec-0" description = "Terraform resman security service account." prefix = var.prefix + iam = { + "roles/iam.serviceAccountTokenCreator" = compact([ + try(module.branch-security-sa-cicd.0.iam_email, null) + ]) + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.admin"] + } } module "branch-security-gcs" { source = "../../../modules/gcs" - project_id = var.automation_project_id + project_id = var.automation.project_id name = "prod-resman-sec-0" prefix = var.prefix versioning = true @@ -62,3 +72,37 @@ module "branch-security-gcs" { "roles/storage.objectAdmin" = [module.branch-security-sa.iam_email] } } + +# ci/cd service account + +module "branch-security-sa-cicd" { + source = "../../../modules/iam-service-account" + for_each = ( + lookup(local.cicd_repositories, "security", null) == null + ? {} + : { 0 = local.cicd_repositories.security } + ) + project_id = var.automation.project_id + name = "prod-resman-sec-1" + description = "Terraform CI/CD stage 2 security service account." + prefix = var.prefix + iam = { + "roles/iam.workloadIdentityUser" = [ + each.value.branch == null + ? format( + local.identity_providers[each.value.identity_provider].principalset_tpl, + var.automation.federated_identity_pool, + each.value.name + ) + : format( + local.identity_providers[each.value.identity_provider].principal_tpl, + var.automation.federated_identity_pool, + each.value.name, + each.value.branch + ) + ] + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] + } +} diff --git a/fast/stages/01-resman/branch-teams.tf b/fast/stages/01-resman/branch-teams.tf index 81ba946a2..a5a16c768 100644 --- a/fast/stages/01-resman/branch-teams.tf +++ b/fast/stages/01-resman/branch-teams.tf @@ -14,9 +14,7 @@ * limitations under the License. */ -# tfdoc:file:description Team stages resources. - -# top-level teams folder and service account +# tfdoc:file:description Team stage resources. module "branch-teams-folder" { source = "../../../modules/folder" @@ -29,7 +27,7 @@ module "branch-teams-folder" { module "branch-teams-prod-sa" { source = "../../../modules/iam-service-account" - project_id = var.automation_project_id + project_id = var.automation.project_id name = "prod-resman-teams-0" description = "Terraform resman production service account." prefix = var.prefix @@ -48,7 +46,7 @@ module "branch-teams-team-folder" { module "branch-teams-team-sa" { source = "../../../modules/iam-service-account" for_each = coalesce(var.team_folders, {}) - project_id = var.automation_project_id + project_id = var.automation.project_id name = "prod-teams-${each.key}-0" description = "Terraform team ${each.key} service account." prefix = var.prefix @@ -64,7 +62,7 @@ module "branch-teams-team-sa" { module "branch-teams-team-gcs" { source = "../../../modules/gcs" for_each = coalesce(var.team_folders, {}) - project_id = var.automation_project_id + project_id = var.automation.project_id name = "prod-teams-${each.key}-0" prefix = var.prefix versioning = true @@ -73,7 +71,7 @@ module "branch-teams-team-gcs" { } } -# environment: development folder and project factory automation resources +# project factory per-team environment folders module "branch-teams-team-dev-folder" { source = "../../../modules/folder" @@ -96,28 +94,6 @@ module "branch-teams-team-dev-folder" { } } -module "branch-teams-dev-pf-sa" { - source = "../../../modules/iam-service-account" - project_id = var.automation_project_id - name = "dev-resman-pf-0" - # naming: environment in description - description = "Terraform project factory development service account." - prefix = var.prefix -} - -module "branch-teams-dev-pf-gcs" { - source = "../../../modules/gcs" - project_id = var.automation_project_id - name = "dev-resman-pf-0" - prefix = var.prefix - versioning = true - iam = { - "roles/storage.objectAdmin" = [module.branch-teams-dev-pf-sa.iam_email] - } -} - -# environment: production folder and project factory automation resources - module "branch-teams-team-prod-folder" { source = "../../../modules/folder" for_each = coalesce(var.team_folders, {}) @@ -139,18 +115,58 @@ module "branch-teams-team-prod-folder" { } } +# project factory per-team environment service accounts + +module "branch-teams-dev-pf-sa" { + source = "../../../modules/iam-service-account" + project_id = var.automation.project_id + name = "dev-resman-pf-0" + # naming: environment in description + description = "Terraform project factory development service account." + prefix = var.prefix + iam = { + "roles/iam.serviceAccountTokenCreator" = compact([ + try(module.branch-pf-dev-sa-cicd.0.iam_email, null) + ]) + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.admin"] + } +} + module "branch-teams-prod-pf-sa" { source = "../../../modules/iam-service-account" - project_id = var.automation_project_id + project_id = var.automation.project_id name = "prod-resman-pf-0" # naming: environment in description description = "Terraform project factory production service account." prefix = var.prefix + iam = { + "roles/iam.serviceAccountTokenCreator" = compact([ + try(module.branch-pf-prod-sa-cicd.0.iam_email, null) + ]) + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.admin"] + } +} + +# project factory per-team environment GCS buckets + +module "branch-teams-dev-pf-gcs" { + source = "../../../modules/gcs" + project_id = var.automation.project_id + name = "dev-resman-pf-0" + prefix = var.prefix + versioning = true + iam = { + "roles/storage.objectAdmin" = [module.branch-teams-dev-pf-sa.iam_email] + } } module "branch-teams-prod-pf-gcs" { source = "../../../modules/gcs" - project_id = var.automation_project_id + project_id = var.automation.project_id name = "prod-resman-pf-0" prefix = var.prefix versioning = true @@ -158,3 +174,67 @@ module "branch-teams-prod-pf-gcs" { "roles/storage.objectAdmin" = [module.branch-teams-prod-pf-sa.iam_email] } } + +# project factory per-team environment CI/CD service accounts + +module "branch-pf-dev-sa-cicd" { + source = "../../../modules/iam-service-account" + for_each = ( + lookup(local.cicd_repositories, "pf_dev", null) == null + ? {} + : { 0 = local.cicd_repositories.pf_dev } + ) + project_id = var.automation.project_id + name = "dev-resman-pf-1" + description = "Terraform CI/CD project factory development service account." + prefix = var.prefix + iam = { + "roles/iam.workloadIdentityUser" = [ + each.value.branch == null + ? format( + local.identity_providers[each.value.identity_provider].principalset_tpl, + each.value.name + ) + : format( + local.identity_providers[each.value.identity_provider].principal_tpl, + each.value.name, + each.value.branch + ) + ] + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] + } +} + +module "branch-pf-prod-sa-cicd" { + source = "../../../modules/iam-service-account" + for_each = ( + lookup(local.cicd_repositories, "pf_prod", null) == null + ? {} + : { 0 = local.cicd_repositories.pf_prod } + ) + project_id = var.automation.project_id + name = "prod-resman-pf-1" + description = "Terraform CI/CD project factory production service account." + prefix = var.prefix + iam = { + "roles/iam.workloadIdentityUser" = [ + each.value.branch == null + ? format( + local.identity_providers[each.value.identity_provider].principalset_tpl, + var.automation.federated_identity_pool, + each.value.name + ) + : format( + local.identity_providers[each.value.identity_provider].principal_tpl, + var.automation.federated_identity_pool, + each.value.name, + each.value.branch + ) + ] + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] + } +} diff --git a/fast/stages/01-resman/globals.auto.tfvars.json b/fast/stages/01-resman/globals.auto.tfvars.json new file mode 120000 index 000000000..29e127f3d --- /dev/null +++ b/fast/stages/01-resman/globals.auto.tfvars.json @@ -0,0 +1 @@ +/home/ludomagno/Desktop/dev/tf-playground/config/tfvars/globals.auto.tfvars.json \ No newline at end of file diff --git a/fast/stages/01-resman/main.tf b/fast/stages/01-resman/main.tf index 0cc1c6bbc..a0c58dc23 100644 --- a/fast/stages/01-resman/main.tf +++ b/fast/stages/01-resman/main.tf @@ -19,7 +19,17 @@ locals { billing_ext = var.billing_account.organization_id == null billing_org = var.billing_account.organization_id == var.organization.id billing_org_ext = !local.billing_ext && !local.billing_org - custom_roles = coalesce(var.custom_roles, {}) + cicd_repositories = { + for k, v in coalesce(var.cicd_repositories, {}) : k => v + if( + v != null + && + contains(keys(local.identity_providers), try(v.identity_provider, "")) + && + fileexists("${path.module}/templates/workflow-${try(v.type, "")}.yaml") + ) + } + custom_roles = coalesce(var.custom_roles, {}) groups = { for k, v in var.groups : k => "${v}@${var.organization.domain}" @@ -28,4 +38,7 @@ locals { for k, v in local.groups : k => "group:${v}" } + identity_providers = coalesce( + try(var.automation.federated_identity_providers, null), {} + ) } diff --git a/fast/stages/01-resman/organization.tf b/fast/stages/01-resman/organization.tf index 2e4cb35ef..4f4620592 100644 --- a/fast/stages/01-resman/organization.tf +++ b/fast/stages/01-resman/organization.tf @@ -137,6 +137,18 @@ module "organization" { # status = true # values = local.allowed_regions # } + # https://cloud.google.com/iam/docs/manage-workload-identity-pools-providers#restrict + # "constraints/iam.workloadIdentityPoolProviders" = merge( + # local.list_allow, { values = [ + # for k, v in coalesce(var.automation.federated_identity_providers, {}) : + # v.issuer_uri + # ] } + # ) + # "constraints/iam.workloadIdentityPoolAwsAccounts" = merge( + # local.list_allow, { values = [ + # + # ] } + # ) } tags = { context = { diff --git a/fast/stages/01-resman/outputs-files.tf b/fast/stages/01-resman/outputs-files.tf new file mode 100644 index 000000000..8f528fa78 --- /dev/null +++ b/fast/stages/01-resman/outputs-files.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Output files persistence to local filesystem. + +resource "local_file" "providers" { + for_each = var.outputs_location == null ? {} : local.providers + file_permission = "0644" + filename = "${pathexpand(var.outputs_location)}/providers/${each.key}-providers.tf" + content = each.value +} + +resource "local_file" "tfvars" { + for_each = var.outputs_location == null ? {} : { 1 = 1 } + file_permission = "0644" + filename = "${pathexpand(var.outputs_location)}/tfvars/01-resman.auto.tfvars.json" + content = jsonencode(local.tfvars) +} + +resource "local_file" "workflows" { + for_each = local.cicd_workflows + file_permission = "0644" + filename = "${pathexpand(var.outputs_location)}/workflows/${replace(each.key, "_", "-")}-workflow.yaml" + content = each.value +} diff --git a/fast/stages/01-resman/outputs-gcs.tf b/fast/stages/01-resman/outputs-gcs.tf new file mode 100644 index 000000000..f1db11ef5 --- /dev/null +++ b/fast/stages/01-resman/outputs-gcs.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Output files persistence to automation GCS bucket. + +resource "google_storage_bucket_object" "providers" { + for_each = local.providers + bucket = var.automation.outputs_bucket + name = "providers/${each.key}-providers.tf" + content = each.value +} + +resource "google_storage_bucket_object" "tfvars" { + bucket = var.automation.outputs_bucket + name = "tfvars/01-resman.auto.tfvars.json" + content = jsonencode(local.tfvars) +} + +resource "google_storage_bucket_object" "workflows" { + for_each = local.cicd_workflows + bucket = var.automation.outputs_bucket + name = "workflows/${replace(each.key, "_", "-")}-workflow.yaml" + content = each.value +} diff --git a/fast/stages/01-resman/outputs.tf b/fast/stages/01-resman/outputs.tf index d38d9a8ff..c9e68e660 100644 --- a/fast/stages/01-resman/outputs.tf +++ b/fast/stages/01-resman/outputs.tf @@ -15,6 +15,63 @@ */ locals { + _cicd_tf_var_files = { + stage_2 = [ + "00-bootstrap.auto.tfvars.json", + "01-resman.auto.tfvars.json", + "globals.auto.tfvars.json" + ] + stage_3 = [ + "00-bootstrap.auto.tfvars.json", + "01-resman.auto.tfvars.json", + "globals.auto.tfvars.json", + "02-networking.auto.tfvars.json", + "02-security.auto.tfvars.json" + ] + } + _tpl_providers = "${path.module}/templates/providers.tf.tpl" + cicd_workflow_attrs = { + data_platform_dev = { + service_account = try(module.branch-dp-dev-sa-cicd.0.email, null) + tf_providers_file = "03-data-platform-dev-providers.tf" + tf_var_files = local._cicd_tf_var_files.stage_3 + } + data_platform_prod = { + service_account = try(module.branch-dp-prod-sa-cicd.0.email, null) + tf_providers_file = "03-data-platform-prod-providers.tf" + tf_var_files = local._cicd_tf_var_files.stage_3 + } + networking = { + service_account = try(module.branch-network-sa-cicd.0.email, null) + tf_providers_file = "02-networking-providers.tf" + tf_var_files = local._cicd_tf_var_files.stage_2 + } + project_factory_dev = { + service_account = try(module.branch-pf-dev-sa-cicd.0.email, null) + tf_providers_file = "03-project-factory-dev-providers.tf" + tf_var_files = local._cicd_tf_var_files.stage_3 + } + project_factory_prod = { + service_account = try(module.branch-pf-prod-sa-cicd.0.email, null) + tf_providers_file = "03-project-factory-prod-providers.tf" + tf_var_files = local._cicd_tf_var_files.stage_3 + } + security = { + service_account = try(module.branch-security-sa-cicd.0.email, null) + tf_providers_file = "02-security-providers.tf" + tf_var_files = local._cicd_tf_var_files.stage_2 + } + } + cicd_workflows = { + for k, v in local.cicd_repositories : k => templatefile( + "${path.module}/templates/workflow-${v.type}.yaml", + merge(local.cicd_workflow_attrs[k], { + identity_provider = local.identity_providers[v.identity_provider].name + outputs_bucket = var.automation.outputs_bucket + stage_name = k + }) + ) + } folder_ids = merge( { data-platform = module.branch-dp-dev-folder.id @@ -26,47 +83,50 @@ locals { teams = module.branch-teams-folder.id }, { - for k, v in module.branch-teams-team-folder : "team-${k}" => v.id + for k, v in module.branch-teams-team-folder : + "team-${k}" => v.id }, { - for k, v in module.branch-teams-team-dev-folder : "team-${k}-dev" => v.id + for k, v in module.branch-teams-team-dev-folder : + "team-${k}-dev" => v.id }, { - for k, v in module.branch-teams-team-prod-folder : "team-${k}-prod" => v.id + for k, v in module.branch-teams-team-prod-folder : + "team-${k}-prod" => v.id } ) providers = { - "02-networking" = templatefile("${path.module}/../../assets/templates/providers.tpl", { + "02-networking" = templatefile(local._tpl_providers, { bucket = module.branch-network-gcs.name name = "networking" sa = module.branch-network-sa.email }) - "02-security" = templatefile("${path.module}/../../assets/templates/providers.tpl", { + "02-security" = templatefile(local._tpl_providers, { bucket = module.branch-security-gcs.name name = "security" sa = module.branch-security-sa.email }) - "03-data-platform-dev" = templatefile("${path.module}/../../assets/templates/providers.tpl", { + "03-data-platform-dev" = templatefile(local._tpl_providers, { bucket = module.branch-dp-dev-gcs.name name = "dp-dev" sa = module.branch-dp-dev-sa.email }) - "03-data-platform-prod" = templatefile("${path.module}/../../assets/templates/providers.tpl", { + "03-data-platform-prod" = templatefile(local._tpl_providers, { bucket = module.branch-dp-prod-gcs.name name = "dp-prod" sa = module.branch-dp-prod-sa.email }) - "03-project-factory-dev" = templatefile("${path.module}/../../assets/templates/providers.tpl", { + "03-project-factory-dev" = templatefile(local._tpl_providers, { bucket = module.branch-teams-dev-pf-gcs.name name = "team-dev" sa = module.branch-teams-dev-pf-sa.email }) - "03-project-factory-prod" = templatefile("${path.module}/../../assets/templates/providers.tpl", { + "03-project-factory-prod" = templatefile(local._tpl_providers, { bucket = module.branch-teams-prod-pf-gcs.name name = "team-prod" sa = module.branch-teams-prod-pf-sa.email }) - "99-sandbox" = templatefile("${path.module}/../../assets/templates/providers.tpl", { + "99-sandbox" = templatefile(local._tpl_providers, { bucket = module.branch-sandbox-gcs.name name = "sandbox" sa = module.branch-sandbox-sa.email @@ -93,24 +153,18 @@ locals { } } -# optionally generate providers and tfvars files for subsequent stages - -resource "local_file" "providers" { - for_each = var.outputs_location == null ? {} : local.providers - file_permission = "0644" - filename = "${pathexpand(var.outputs_location)}/providers/${each.key}-providers.tf" - content = each.value +output "cicd_repositories" { + description = "WIF configuration for CI/CD repositories." + value = { + for k, v in local.cicd_repositories : k => { + branch = v.branch + name = v.name + provider = local.identity_providers[v.identity_provider].name + service_account = local.cicd_workflow_attrs[k].service_account + } if v != null + } } -resource "local_file" "tfvars" { - for_each = var.outputs_location == null ? {} : { 1 = 1 } - file_permission = "0644" - filename = "${pathexpand(var.outputs_location)}/tfvars/01-resman.auto.tfvars.json" - content = jsonencode(local.tfvars) -} - -# outputs - output "dataplatform" { description = "Data for the Data Platform stage." value = { diff --git a/fast/stages/01-resman/templates b/fast/stages/01-resman/templates new file mode 120000 index 000000000..bcb6967be --- /dev/null +++ b/fast/stages/01-resman/templates @@ -0,0 +1 @@ +../../assets/templates \ No newline at end of file diff --git a/fast/stages/01-resman/variables.tf b/fast/stages/01-resman/variables.tf index 639aba6f6..b0a97cb04 100644 --- a/fast/stages/01-resman/variables.tf +++ b/fast/stages/01-resman/variables.tf @@ -17,10 +17,21 @@ # defaults for variables marked with global tfdoc annotations, can be set via # the tfvars file generated in stage 00 and stored in its outputs -variable "automation_project_id" { +variable "automation" { # tfdoc:variable:source 00-bootstrap - description = "Project id for the automation project created by the bootstrap stage." - type = string + description = "Automation resources created by the bootstrap stage." + type = object({ + outputs_bucket = string + project_id = string + federated_identity_pool = string + federated_identity_providers = map(object({ + issuer = string + issuer_uri = string + name = string + principal_tpl = string + principalset_tpl = string + })) + }) } variable "billing_account" { @@ -32,6 +43,69 @@ variable "billing_account" { }) } +variable "cicd_repositories" { + description = "CI/CD repository configuration. Identity providers reference keys in the `automation.federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed." + type = object({ + data_platform_dev = object({ + branch = string + identity_provider = string + name = string + type = string + }) + data_platform_prod = object({ + branch = string + identity_provider = string + name = string + type = string + }) + networking = object({ + branch = string + identity_provider = string + name = string + type = string + }) + project_factory_dev = object({ + branch = string + identity_provider = string + name = string + type = string + }) + project_factory_prod = object({ + branch = string + identity_provider = string + name = string + type = string + }) + security = object({ + branch = string + identity_provider = string + name = string + type = string + }) + }) + default = null + validation { + condition = alltrue([ + for k, v in coalesce(var.cicd_repositories, {}) : + v == null || ( + try(v.name, null) != null + && + try(v.identity_provider, null) != null + ) + ]) + error_message = "Non-null repositories need non-null name and providers." + } + validation { + condition = alltrue([ + for k, v in coalesce(var.cicd_repositories, {}) : + v == null || ( + contains(["github"], coalesce(try(v.type, null), "null")) + ) + ]) + error_message = "Invalid repository type, supported types: 'github'." + } +} + variable "custom_roles" { # tfdoc:variable:source 00-bootstrap description = "Custom roles defined at the org level, in key => id format." @@ -75,7 +149,7 @@ variable "organization_policy_configs" { } variable "outputs_location" { - description = "Path where providers and tfvars files for the following stages are written. Leave empty to disable." + description = "Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable" type = string default = null } diff --git a/fast/stages/02-networking-nva/README.md b/fast/stages/02-networking-nva/README.md index dec165af4..fd9b2831f 100644 --- a/fast/stages/02-networking-nva/README.md +++ b/fast/stages/02-networking-nva/README.md @@ -352,7 +352,7 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS | [main.tf](./main.tf) | Networking folder and hierarchical policy. | folder | | | [monitoring.tf](./monitoring.tf) | Network monitoring dashboards. | | google_monitoring_dashboard | | [nva.tf](./nva.tf) | None | compute-mig ยท compute-vm ยท net-ilb | | -| [outputs.tf](./outputs.tf) | Module outputs. | | local_file | +| [outputs.tf](./outputs.tf) | Module outputs. | | google_storage_bucket_object ยท local_file | | [spoke-dev.tf](./spoke-dev.tf) | Dev spoke VPC and related resources. | net-vpc ยท net-vpc-firewall ยท net-vpc-peering ยท project | google_project_iam_binding | | [spoke-prod.tf](./spoke-prod.tf) | Production spoke VPC and related resources. | net-vpc ยท net-vpc-firewall ยท net-vpc-peering ยท project | google_project_iam_binding | | [test-resources.tf](./test-resources.tf) | temporary instances for testing | compute-vm | | @@ -363,30 +363,31 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| -| [billing_account](variables.tf#L17) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | โœ“ | | 00-bootstrap | -| [folder_ids](variables.tf#L71) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | โœ“ | | 01-resman | -| [organization](variables.tf#L107) | Organization details. | object({…}) | โœ“ | | 00-bootstrap | -| [prefix](variables.tf#L123) | Prefix used for resources that need unique names. Use 9 characters or less. | string | โœ“ | | 00-bootstrap | -| [custom_adv](variables.tf#L26) | Custom advertisement definitions in name => range format. | map(string) | | {…} | | -| [custom_roles](variables.tf#L48) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap | -| [data_dir](variables.tf#L57) | Relative path for the folder storing configuration data for network resources. | string | | "data" | | -| [dns](variables.tf#L63) | Onprem DNS resolvers | map(list(string)) | | {…} | | -| [l7ilb_subnets](variables.tf#L81) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | | -| [onprem_cidr](variables.tf#L99) | Onprem addresses in name => range format. | map(string) | | {…} | | -| [outputs_location](variables.tf#L117) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | | -| [psa_ranges](variables.tf#L134) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | | -| [router_configs](variables.tf#L175) | Configurations for CRs and onprem routers. | map(object({…})) | | {…} | | -| [service_accounts](variables.tf#L198) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | -| [vpn_onprem_configs](variables.tf#L210) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | +| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | โœ“ | | 00-bootstrap | +| [billing_account](variables.tf#L25) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | โœ“ | | 00-bootstrap | +| [folder_ids](variables.tf#L79) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | โœ“ | | 01-resman | +| [organization](variables.tf#L115) | Organization details. | object({…}) | โœ“ | | 00-bootstrap | +| [prefix](variables.tf#L131) | Prefix used for resources that need unique names. Use 9 characters or less. | string | โœ“ | | 00-bootstrap | +| [custom_adv](variables.tf#L34) | Custom advertisement definitions in name => range format. | map(string) | | {…} | | +| [custom_roles](variables.tf#L56) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap | +| [data_dir](variables.tf#L65) | Relative path for the folder storing configuration data for network resources. | string | | "data" | | +| [dns](variables.tf#L71) | Onprem DNS resolvers | map(list(string)) | | {…} | | +| [l7ilb_subnets](variables.tf#L89) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | | +| [onprem_cidr](variables.tf#L107) | Onprem addresses in name => range format. | map(string) | | {…} | | +| [outputs_location](variables.tf#L125) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | | +| [psa_ranges](variables.tf#L142) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | | +| [router_configs](variables.tf#L183) | Configurations for CRs and onprem routers. | map(object({…})) | | {…} | | +| [service_accounts](variables.tf#L206) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | +| [vpn_onprem_configs](variables.tf#L218) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | ## Outputs | name | description | sensitive | consumers | |---|---|:---:|---| -| [host_project_ids](outputs.tf#L52) | Network project ids. | | | -| [host_project_numbers](outputs.tf#L57) | Network project numbers. | | | -| [shared_vpc_self_links](outputs.tf#L62) | Shared VPC host projects. | | | -| [tfvars](outputs.tf#L81) | Terraform variables file for the following stages. | โœ“ | | -| [vpn_gateway_endpoints](outputs.tf#L67) | External IP Addresses for the GCP VPN gateways. | | | +| [host_project_ids](outputs.tf#L58) | Network project ids. | | | +| [host_project_numbers](outputs.tf#L63) | Network project numbers. | | | +| [shared_vpc_self_links](outputs.tf#L68) | Shared VPC host projects. | | | +| [tfvars](outputs.tf#L87) | Terraform variables file for the following stages. | โœ“ | | +| [vpn_gateway_endpoints](outputs.tf#L73) | External IP Addresses for the GCP VPN gateways. | | | diff --git a/fast/stages/02-networking-nva/outputs.tf b/fast/stages/02-networking-nva/outputs.tf index 072cb9a76..b32807cce 100644 --- a/fast/stages/02-networking-nva/outputs.tf +++ b/fast/stages/02-networking-nva/outputs.tf @@ -38,7 +38,7 @@ locals { } } -# optionally generate tfvars file for subsequent stages +# generate tfvars file for subsequent stages resource "local_file" "tfvars" { for_each = var.outputs_location == null ? {} : { 1 = 1 } @@ -47,6 +47,12 @@ resource "local_file" "tfvars" { content = jsonencode(local.tfvars) } +resource "google_storage_bucket_object" "tfvars" { + bucket = var.automation.outputs_bucket + name = "tfvars/02-networking.auto.tfvars.json" + content = jsonencode(local.tfvars) +} + # outputs output "host_project_ids" { diff --git a/fast/stages/02-networking-nva/variables.tf b/fast/stages/02-networking-nva/variables.tf index 2cae9a6cd..bc06729bc 100644 --- a/fast/stages/02-networking-nva/variables.tf +++ b/fast/stages/02-networking-nva/variables.tf @@ -14,6 +14,14 @@ * limitations under the License. */ +variable "automation" { + # tfdoc:variable:source 00-bootstrap + description = "Automation resources created by the bootstrap stage." + type = object({ + outputs_bucket = string + }) +} + variable "billing_account" { # tfdoc:variable:source 00-bootstrap description = "Billing account id and organization id ('nnnnnnnn' or null)." diff --git a/fast/stages/02-networking-peering/README.md b/fast/stages/02-networking-peering/README.md index 8ce038e20..316caf7e9 100644 --- a/fast/stages/02-networking-peering/README.md +++ b/fast/stages/02-networking-peering/README.md @@ -274,7 +274,7 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS | [landing.tf](./landing.tf) | Landing VPC and related resources. | net-cloudnat ยท net-vpc ยท net-vpc-firewall ยท project | | | [main.tf](./main.tf) | Networking folder and hierarchical policy. | folder | | | [monitoring.tf](./monitoring.tf) | Network monitoring dashboards. | | google_monitoring_dashboard | -| [outputs.tf](./outputs.tf) | Module outputs. | | local_file | +| [outputs.tf](./outputs.tf) | Module outputs. | | google_storage_bucket_object ยท local_file | | [peerings.tf](./peerings.tf) | None | net-vpc-peering | | | [spoke-dev.tf](./spoke-dev.tf) | Dev spoke VPC and related resources. | net-cloudnat ยท net-vpc ยท net-vpc-firewall ยท project | google_project_iam_binding | | [spoke-prod.tf](./spoke-prod.tf) | Production spoke VPC and related resources. | net-cloudnat ยท net-vpc ยท net-vpc-firewall ยท project | google_project_iam_binding | @@ -287,31 +287,32 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| -| [billing_account](variables.tf#L17) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | โœ“ | | 00-bootstrap | -| [folder_ids](variables.tf#L66) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | โœ“ | | 01-resman | -| [organization](variables.tf#L94) | Organization details. | object({…}) | โœ“ | | 00-bootstrap | -| [prefix](variables.tf#L110) | Prefix used for resources that need unique names. Use 9 characters or less. | string | โœ“ | | 00-bootstrap | -| [custom_adv](variables.tf#L26) | Custom advertisement definitions in name => range format. | map(string) | | {…} | | -| [custom_roles](variables.tf#L43) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap | -| [data_dir](variables.tf#L52) | Relative path for the folder storing configuration data for network resources. | string | | "data" | | -| [dns](variables.tf#L58) | Onprem DNS resolvers. | map(list(string)) | | {…} | | -| [l7ilb_subnets](variables.tf#L76) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | | -| [outputs_location](variables.tf#L104) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | | +| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | โœ“ | | 00-bootstrap | +| [billing_account](variables.tf#L25) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | โœ“ | | 00-bootstrap | +| [folder_ids](variables.tf#L74) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | โœ“ | | 01-resman | +| [organization](variables.tf#L102) | Organization details. | object({…}) | โœ“ | | 00-bootstrap | +| [prefix](variables.tf#L118) | Prefix used for resources that need unique names. Use 9 characters or less. | string | โœ“ | | 00-bootstrap | +| [custom_adv](variables.tf#L34) | Custom advertisement definitions in name => range format. | map(string) | | {…} | | +| [custom_roles](variables.tf#L51) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap | +| [data_dir](variables.tf#L60) | Relative path for the folder storing configuration data for network resources. | string | | "data" | | +| [dns](variables.tf#L66) | Onprem DNS resolvers. | map(list(string)) | | {…} | | +| [l7ilb_subnets](variables.tf#L84) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | | +| [outputs_location](variables.tf#L112) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | | | [peering_configs](variables-peerings.tf#L19) | Peering configurations. | map(object({…})) | | {…} | | -| [psa_ranges](variables.tf#L121) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | | -| [router_onprem_configs](variables.tf#L158) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | | -| [service_accounts](variables.tf#L176) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | -| [vpn_onprem_configs](variables.tf#L188) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | +| [psa_ranges](variables.tf#L129) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | | +| [router_onprem_configs](variables.tf#L166) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | | +| [service_accounts](variables.tf#L184) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | +| [vpn_onprem_configs](variables.tf#L196) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | ## Outputs | name | description | sensitive | consumers | |---|---|:---:|---| -| [cloud_dns_inbound_policy](outputs.tf#L57) | IP Addresses for Cloud DNS inbound policy. | | | -| [host_project_ids](outputs.tf#L62) | Network project ids. | | | -| [host_project_numbers](outputs.tf#L67) | Network project numbers. | | | -| [shared_vpc_self_links](outputs.tf#L72) | Shared VPC host projects. | | | -| [tfvars](outputs.tf#L87) | Terraform variables file for the following stages. | โœ“ | | -| [vpn_gateway_endpoints](outputs.tf#L77) | External IP Addresses for the GCP VPN gateways. | | | +| [cloud_dns_inbound_policy](outputs.tf#L63) | IP Addresses for Cloud DNS inbound policy. | | | +| [host_project_ids](outputs.tf#L68) | Network project ids. | | | +| [host_project_numbers](outputs.tf#L73) | Network project numbers. | | | +| [shared_vpc_self_links](outputs.tf#L78) | Shared VPC host projects. | | | +| [tfvars](outputs.tf#L93) | Terraform variables file for the following stages. | โœ“ | | +| [vpn_gateway_endpoints](outputs.tf#L83) | External IP Addresses for the GCP VPN gateways. | | | diff --git a/fast/stages/02-networking-peering/outputs.tf b/fast/stages/02-networking-peering/outputs.tf index 3fe18d657..ec3f7191a 100644 --- a/fast/stages/02-networking-peering/outputs.tf +++ b/fast/stages/02-networking-peering/outputs.tf @@ -43,7 +43,7 @@ locals { } } -# optionally generate tfvars file for subsequent stages +# generate tfvars file for subsequent stages resource "local_file" "tfvars" { for_each = var.outputs_location == null ? {} : { 1 = 1 } @@ -52,6 +52,12 @@ resource "local_file" "tfvars" { content = jsonencode(local.tfvars) } +resource "google_storage_bucket_object" "tfvars" { + bucket = var.automation.outputs_bucket + name = "tfvars/02-networking.auto.tfvars.json" + content = jsonencode(local.tfvars) +} + # outputs output "cloud_dns_inbound_policy" { diff --git a/fast/stages/02-networking-peering/variables.tf b/fast/stages/02-networking-peering/variables.tf index 6549080c8..60bd8be1d 100644 --- a/fast/stages/02-networking-peering/variables.tf +++ b/fast/stages/02-networking-peering/variables.tf @@ -14,6 +14,14 @@ * limitations under the License. */ +variable "automation" { + # tfdoc:variable:source 00-bootstrap + description = "Automation resources created by the bootstrap stage." + type = object({ + outputs_bucket = string + }) +} + variable "billing_account" { # tfdoc:variable:source 00-bootstrap description = "Billing account id and organization id ('nnnnnnnn' or null)." diff --git a/fast/stages/02-networking-vpn/README.md b/fast/stages/02-networking-vpn/README.md index aa95d4005..e9dd8cab5 100644 --- a/fast/stages/02-networking-vpn/README.md +++ b/fast/stages/02-networking-vpn/README.md @@ -297,7 +297,7 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS | [landing.tf](./landing.tf) | Landing VPC and related resources. | net-cloudnat ยท net-vpc ยท net-vpc-firewall ยท project | | | [main.tf](./main.tf) | Networking folder and hierarchical policy. | folder | | | [monitoring.tf](./monitoring.tf) | Network monitoring dashboards. | | google_monitoring_dashboard | -| [outputs.tf](./outputs.tf) | Module outputs. | | local_file | +| [outputs.tf](./outputs.tf) | Module outputs. | | google_storage_bucket_object ยท local_file | | [spoke-dev.tf](./spoke-dev.tf) | Dev spoke VPC and related resources. | net-cloudnat ยท net-vpc ยท net-vpc-firewall ยท project | google_project_iam_binding | | [spoke-prod.tf](./spoke-prod.tf) | Production spoke VPC and related resources. | net-cloudnat ยท net-vpc ยท net-vpc-firewall ยท project | google_project_iam_binding | | [test-resources.tf](./test-resources.tf) | temporary instances for testing | compute-vm | | @@ -311,32 +311,33 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| -| [billing_account](variables.tf#L17) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | โœ“ | | 00-bootstrap | -| [folder_ids](variables.tf#L66) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | โœ“ | | 01-resman | -| [organization](variables.tf#L94) | Organization details. | object({…}) | โœ“ | | 00-bootstrap | -| [prefix](variables.tf#L110) | Prefix used for resources that need unique names. Use 9 characters or less. | string | โœ“ | | 00-bootstrap | -| [custom_adv](variables.tf#L26) | Custom advertisement definitions in name => range format. | map(string) | | {…} | | -| [custom_roles](variables.tf#L43) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap | -| [data_dir](variables.tf#L52) | Relative path for the folder storing configuration data for network resources. | string | | "data" | | -| [dns](variables.tf#L58) | Onprem DNS resolvers. | map(list(string)) | | {…} | | -| [l7ilb_subnets](variables.tf#L76) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | | -| [outputs_location](variables.tf#L104) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | | -| [psa_ranges](variables.tf#L121) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | | -| [router_onprem_configs](variables.tf#L158) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | | +| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | โœ“ | | 00-bootstrap | +| [billing_account](variables.tf#L25) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | โœ“ | | 00-bootstrap | +| [folder_ids](variables.tf#L74) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | โœ“ | | 01-resman | +| [organization](variables.tf#L102) | Organization details. | object({…}) | โœ“ | | 00-bootstrap | +| [prefix](variables.tf#L118) | Prefix used for resources that need unique names. Use 9 characters or less. | string | โœ“ | | 00-bootstrap | +| [custom_adv](variables.tf#L34) | Custom advertisement definitions in name => range format. | map(string) | | {…} | | +| [custom_roles](variables.tf#L51) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap | +| [data_dir](variables.tf#L60) | Relative path for the folder storing configuration data for network resources. | string | | "data" | | +| [dns](variables.tf#L66) | Onprem DNS resolvers. | map(list(string)) | | {…} | | +| [l7ilb_subnets](variables.tf#L84) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | | +| [outputs_location](variables.tf#L112) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | | +| [psa_ranges](variables.tf#L129) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | | +| [router_onprem_configs](variables.tf#L166) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | | | [router_spoke_configs](variables-vpn.tf#L18) | Configurations for routers used for internal connectivity. | map(object({…})) | | {…} | | -| [service_accounts](variables.tf#L176) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | -| [vpn_onprem_configs](variables.tf#L188) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | +| [service_accounts](variables.tf#L184) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | +| [vpn_onprem_configs](variables.tf#L196) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | | [vpn_spoke_configs](variables-vpn.tf#L37) | VPN gateway configuration for spokes. | map(object({…})) | | {…} | | ## Outputs | name | description | sensitive | consumers | |---|---|:---:|---| -| [cloud_dns_inbound_policy](outputs.tf#L57) | IP Addresses for Cloud DNS inbound policy. | | | -| [host_project_ids](outputs.tf#L62) | Network project ids. | | | -| [host_project_numbers](outputs.tf#L67) | Network project numbers. | | | -| [shared_vpc_self_links](outputs.tf#L72) | Shared VPC host projects. | | | -| [tfvars](outputs.tf#L87) | Terraform variables file for the following stages. | โœ“ | | -| [vpn_gateway_endpoints](outputs.tf#L77) | External IP Addresses for the GCP VPN gateways. | | | +| [cloud_dns_inbound_policy](outputs.tf#L63) | IP Addresses for Cloud DNS inbound policy. | | | +| [host_project_ids](outputs.tf#L68) | Network project ids. | | | +| [host_project_numbers](outputs.tf#L73) | Network project numbers. | | | +| [shared_vpc_self_links](outputs.tf#L78) | Shared VPC host projects. | | | +| [tfvars](outputs.tf#L93) | Terraform variables file for the following stages. | โœ“ | | +| [vpn_gateway_endpoints](outputs.tf#L83) | External IP Addresses for the GCP VPN gateways. | | | diff --git a/fast/stages/02-networking-vpn/outputs.tf b/fast/stages/02-networking-vpn/outputs.tf index 3fe18d657..ec3f7191a 100644 --- a/fast/stages/02-networking-vpn/outputs.tf +++ b/fast/stages/02-networking-vpn/outputs.tf @@ -43,7 +43,7 @@ locals { } } -# optionally generate tfvars file for subsequent stages +# generate tfvars file for subsequent stages resource "local_file" "tfvars" { for_each = var.outputs_location == null ? {} : { 1 = 1 } @@ -52,6 +52,12 @@ resource "local_file" "tfvars" { content = jsonencode(local.tfvars) } +resource "google_storage_bucket_object" "tfvars" { + bucket = var.automation.outputs_bucket + name = "tfvars/02-networking.auto.tfvars.json" + content = jsonencode(local.tfvars) +} + # outputs output "cloud_dns_inbound_policy" { diff --git a/fast/stages/02-networking-vpn/variables.tf b/fast/stages/02-networking-vpn/variables.tf index 6549080c8..60bd8be1d 100644 --- a/fast/stages/02-networking-vpn/variables.tf +++ b/fast/stages/02-networking-vpn/variables.tf @@ -14,6 +14,14 @@ * limitations under the License. */ +variable "automation" { + # tfdoc:variable:source 00-bootstrap + description = "Automation resources created by the bootstrap stage." + type = object({ + outputs_bucket = string + }) +} + variable "billing_account" { # tfdoc:variable:source 00-bootstrap description = "Billing account id and organization id ('nnnnnnnn' or null)." diff --git a/fast/stages/02-security/README.md b/fast/stages/02-security/README.md index 8da0c0169..50df26a10 100644 --- a/fast/stages/02-security/README.md +++ b/fast/stages/02-security/README.md @@ -277,7 +277,7 @@ Some references that might be useful in setting up this stage: | [core-dev.tf](./core-dev.tf) | None | kms ยท project | google_project_iam_member | | [core-prod.tf](./core-prod.tf) | None | kms ยท project | google_project_iam_member | | [main.tf](./main.tf) | Module-level locals and resources. | | | -| [outputs.tf](./outputs.tf) | Module outputs. | | local_file | +| [outputs.tf](./outputs.tf) | Module outputs. | | google_storage_bucket_object ยท local_file | | [variables.tf](./variables.tf) | Module variables. | | | | [vpc-sc.tf](./vpc-sc.tf) | None | vpc-sc | | @@ -285,29 +285,30 @@ Some references that might be useful in setting up this stage: | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| -| [billing_account](variables.tf#L17) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | โœ“ | | 00-bootstrap | -| [folder_ids](variables.tf#L26) | Folder name => id mappings, the 'security' folder name must exist. | object({…}) | โœ“ | | 01-resman | -| [organization](variables.tf#L81) | Organization details. | object({…}) | โœ“ | | 00-bootstrap | -| [prefix](variables.tf#L97) | Prefix used for resources that need unique names. Use 9 characters or less. | string | โœ“ | | 00-bootstrap | -| [service_accounts](variables.tf#L72) | Automation service accounts that can assign the encrypt/decrypt roles on keys. | object({…}) | โœ“ | | 01-resman | -| [groups](variables.tf#L34) | Group names to grant organization-level permissions. | map(string) | | {…} | 00-bootstrap | -| [kms_defaults](variables.tf#L49) | Defaults used for KMS keys. | object({…}) | | {…} | | -| [kms_keys](variables.tf#L61) | KMS keys to create, keyed by name. Null attributes will be interpolated with defaults. | map(object({…})) | | {} | | -| [outputs_location](variables.tf#L91) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [vpc_sc_access_levels](variables.tf#L108) | VPC SC access level definitions. | map(object({…})) | | {} | | -| [vpc_sc_egress_policies](variables.tf#L123) | VPC SC egress policy defnitions. | map(object({…})) | | {} | | -| [vpc_sc_ingress_policies](variables.tf#L141) | VPC SC ingress policy defnitions. | map(object({…})) | | {} | | -| [vpc_sc_perimeter_access_levels](variables.tf#L161) | VPC SC perimeter access_levels. | object({…}) | | null | | -| [vpc_sc_perimeter_egress_policies](variables.tf#L171) | VPC SC egress policies per perimeter, values reference keys defined in the `vpc_sc_ingress_policies` variable. | object({…}) | | null | | -| [vpc_sc_perimeter_ingress_policies](variables.tf#L181) | VPC SC ingress policies per perimeter, values reference keys defined in the `vpc_sc_ingress_policies` variable. | object({…}) | | null | | -| [vpc_sc_perimeter_projects](variables.tf#L191) | VPC SC perimeter resources. | object({…}) | | null | | +| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | โœ“ | | 00-bootstrap | +| [billing_account](variables.tf#L25) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | โœ“ | | 00-bootstrap | +| [folder_ids](variables.tf#L34) | Folder name => id mappings, the 'security' folder name must exist. | object({…}) | โœ“ | | 01-resman | +| [organization](variables.tf#L89) | Organization details. | object({…}) | โœ“ | | 00-bootstrap | +| [prefix](variables.tf#L105) | Prefix used for resources that need unique names. Use 9 characters or less. | string | โœ“ | | 00-bootstrap | +| [service_accounts](variables.tf#L80) | Automation service accounts that can assign the encrypt/decrypt roles on keys. | object({…}) | โœ“ | | 01-resman | +| [groups](variables.tf#L42) | Group names to grant organization-level permissions. | map(string) | | {…} | 00-bootstrap | +| [kms_defaults](variables.tf#L57) | Defaults used for KMS keys. | object({…}) | | {…} | | +| [kms_keys](variables.tf#L69) | KMS keys to create, keyed by name. Null attributes will be interpolated with defaults. | map(object({…})) | | {} | | +| [outputs_location](variables.tf#L99) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | +| [vpc_sc_access_levels](variables.tf#L116) | VPC SC access level definitions. | map(object({…})) | | {} | | +| [vpc_sc_egress_policies](variables.tf#L131) | VPC SC egress policy defnitions. | map(object({…})) | | {} | | +| [vpc_sc_ingress_policies](variables.tf#L149) | VPC SC ingress policy defnitions. | map(object({…})) | | {} | | +| [vpc_sc_perimeter_access_levels](variables.tf#L169) | VPC SC perimeter access_levels. | object({…}) | | null | | +| [vpc_sc_perimeter_egress_policies](variables.tf#L179) | VPC SC egress policies per perimeter, values reference keys defined in the `vpc_sc_ingress_policies` variable. | object({…}) | | null | | +| [vpc_sc_perimeter_ingress_policies](variables.tf#L189) | VPC SC ingress policies per perimeter, values reference keys defined in the `vpc_sc_ingress_policies` variable. | object({…}) | | null | | +| [vpc_sc_perimeter_projects](variables.tf#L199) | VPC SC perimeter resources. | object({…}) | | null | | ## Outputs | name | description | sensitive | consumers | |---|---|:---:|---| -| [kms_keys](outputs.tf#L53) | KMS key ids. | | | -| [stage_perimeter_projects](outputs.tf#L58) | Security project numbers. They can be added to perimeter resources. | | | -| [tfvars](outputs.tf#L68) | Terraform variable files for the following stages. | โœ“ | | +| [kms_keys](outputs.tf#L59) | KMS key ids. | | | +| [stage_perimeter_projects](outputs.tf#L64) | Security project numbers. They can be added to perimeter resources. | | | +| [tfvars](outputs.tf#L74) | Terraform variable files for the following stages. | โœ“ | | diff --git a/fast/stages/02-security/outputs.tf b/fast/stages/02-security/outputs.tf index ee2ac15e6..b7e42e492 100644 --- a/fast/stages/02-security/outputs.tf +++ b/fast/stages/02-security/outputs.tf @@ -39,7 +39,7 @@ locals { } } -# optionally generate files for subsequent stages +# generate files for subsequent stages resource "local_file" "tfvars" { for_each = var.outputs_location == null ? {} : { 1 = 1 } @@ -48,6 +48,12 @@ resource "local_file" "tfvars" { content = jsonencode(local.tfvars) } +resource "google_storage_bucket_object" "tfvars" { + bucket = var.automation.outputs_bucket + name = "tfvars/02-security.auto.tfvars.json" + content = jsonencode(local.tfvars) +} + # outputs output "kms_keys" { diff --git a/fast/stages/02-security/variables.tf b/fast/stages/02-security/variables.tf index 8ff52ffda..352f4f394 100644 --- a/fast/stages/02-security/variables.tf +++ b/fast/stages/02-security/variables.tf @@ -14,6 +14,14 @@ * limitations under the License. */ +variable "automation" { + # tfdoc:variable:source 00-bootstrap + description = "Automation resources created by the bootstrap stage." + type = object({ + outputs_bucket = string + }) +} + variable "billing_account" { # tfdoc:variable:source 00-bootstrap description = "Billing account id and organization id ('nnnnnnnn' or null)." diff --git a/fast/stages/README.md b/fast/stages/README.md index ba01e6b21..8b0814280 100644 --- a/fast/stages/README.md +++ b/fast/stages/README.md @@ -21,7 +21,7 @@ Refer to each stage's documentation for a detailed description of its purpose, t - [Bootstrap](00-bootstrap/README.md) Enables critical organization-level functionality that depends on broad permissions. It has two primary purposes. The first is to bootstrap the resources needed for automation of this and the following stages (service accounts, GCS buckets). And secondly, it applies the minimum amount of configuration needed at the organization level, to avoid the need of broad permissions later on, and to implement a minimum of security features like sinks and exports from the start.\ - Exports: automation project id, organization-level custom roles + Exports: automation variables, organization-level custom roles - [Resource Management](01-resman/README.md) Creates the base resource hierarchy (folders) and the automation resources required later to delegate deployment of each part of the hierarchy to separate stages. This stage also configures organization-level policies and any exceptions needed by different branches of the resource hierarchy.\ Exports: folder ids, automation service account emails diff --git a/modules/gcs/README.md b/modules/gcs/README.md index 322a4c277..3b8c52dbe 100644 --- a/modules/gcs/README.md +++ b/modules/gcs/README.md @@ -136,8 +136,8 @@ module "bucket-gcs-notification" { |---|---|:---:| | [bucket](outputs.tf#L17) | Bucket resource. | | | [name](outputs.tf#L22) | Bucket name. | | -| [notification](outputs.tf#L26) | GCS Notification self link. | | -| [topic](outputs.tf#L30) | Topic ID used by GCS. | | -| [url](outputs.tf#L34) | Bucket URL. | | +| [notification](outputs.tf#L30) | GCS Notification self link. | | +| [topic](outputs.tf#L34) | Topic ID used by GCS. | | +| [url](outputs.tf#L38) | Bucket URL. | | diff --git a/modules/gcs/outputs.tf b/modules/gcs/outputs.tf index 415b94639..3e1ca8746 100644 --- a/modules/gcs/outputs.tf +++ b/modules/gcs/outputs.tf @@ -21,7 +21,11 @@ output "bucket" { output "name" { description = "Bucket name." - value = google_storage_bucket.bucket.name + value = "${local.prefix}${lower(var.name)}" + depends_on = [ + google_storage_bucket.bucket, + google_storage_bucket_iam_binding.bindings + ] } output "notification" { description = "GCS Notification self link." diff --git a/modules/iam-service-account/README.md b/modules/iam-service-account/README.md index 30085dabf..bd2240b8a 100644 --- a/modules/iam-service-account/README.md +++ b/modules/iam-service-account/README.md @@ -31,7 +31,7 @@ module "myproject-default-service-accounts" { | name | description | resources | |---|---|---| -| [iam.tf](./iam.tf) | IAM bindings. | google_billing_account_iam_member ยท google_folder_iam_member ยท google_organization_iam_member ยท google_project_iam_member ยท google_service_account_iam_binding ยท google_storage_bucket_iam_member | +| [iam.tf](./iam.tf) | IAM bindings. | google_billing_account_iam_member ยท google_folder_iam_member ยท google_organization_iam_member ยท google_project_iam_member ยท google_service_account_iam_binding ยท google_service_account_iam_member ยท google_storage_bucket_iam_member | | [main.tf](./main.tf) | Module-level locals and resources. | google_service_account ยท google_service_account_key | | [outputs.tf](./outputs.tf) | Module outputs. | | | [variables.tf](./variables.tf) | Module variables. | | @@ -41,20 +41,21 @@ module "myproject-default-service-accounts" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L77) | Name of the service account to create. | string | โœ“ | | -| [project_id](variables.tf#L88) | Project id where service account will be created. | string | โœ“ | | +| [name](variables.tf#L84) | Name of the service account to create. | string | โœ“ | | +| [project_id](variables.tf#L95) | Project id where service account will be created. | string | โœ“ | | | [description](variables.tf#L17) | Optional description. | string | | null | | [display_name](variables.tf#L23) | Display name of the service account to create. | string | | "Terraform-managed." | | [generate_key](variables.tf#L29) | Generate a key for service account. | bool | | false | | [iam](variables.tf#L35) | IAM bindings on the service account in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_billing_roles](variables.tf#L42) | Billing account roles granted to the service account, by billing account id. Non-authoritative. | map(list(string)) | | {} | -| [iam_folder_roles](variables.tf#L49) | Folder roles granted to the service account, by folder id. Non-authoritative. | map(list(string)) | | {} | -| [iam_organization_roles](variables.tf#L56) | Organization roles granted to the service account, by organization id. Non-authoritative. | map(list(string)) | | {} | -| [iam_project_roles](variables.tf#L63) | Project roles granted to the service account, by project id. | map(list(string)) | | {} | -| [iam_storage_roles](variables.tf#L70) | Storage roles granted to the service account, by bucket name. | map(list(string)) | | {} | -| [prefix](variables.tf#L82) | Prefix applied to service account names. | string | | null | -| [public_keys_directory](variables.tf#L93) | Path to public keys data files to upload to the service account (should have `.pem` extension). | string | | "" | -| [service_account_create](variables.tf#L99) | Create service account. When set to false, uses a data source to reference an existing service account. | bool | | true | +| [iam_billing_roles](variables.tf#L42) | Billing account roles granted to this service account, by billing account id. Non-authoritative. | map(list(string)) | | {} | +| [iam_folder_roles](variables.tf#L49) | Folder roles granted to this service account, by folder id. Non-authoritative. | map(list(string)) | | {} | +| [iam_organization_roles](variables.tf#L56) | Organization roles granted to this service account, by organization id. Non-authoritative. | map(list(string)) | | {} | +| [iam_project_roles](variables.tf#L63) | Project roles granted to this service account, by project id. | map(list(string)) | | {} | +| [iam_sa_roles](variables.tf#L70) | Service account roles granted to this service account, by service account name. | map(list(string)) | | {} | +| [iam_storage_roles](variables.tf#L77) | Storage roles granted to this service account, by bucket name. | map(list(string)) | | {} | +| [prefix](variables.tf#L89) | Prefix applied to service account names. | string | | null | +| [public_keys_directory](variables.tf#L100) | Path to public keys data files to upload to the service account (should have `.pem` extension). | string | | "" | +| [service_account_create](variables.tf#L106) | Create service account. When set to false, uses a data source to reference an existing service account. | bool | | true | ## Outputs @@ -63,7 +64,8 @@ module "myproject-default-service-accounts" { | [email](outputs.tf#L17) | Service account email. | | | [iam_email](outputs.tf#L25) | IAM-format service account email. | | | [key](outputs.tf#L33) | Service account key. | โœ“ | -| [service_account](outputs.tf#L39) | Service account resource. | | -| [service_account_credentials](outputs.tf#L44) | Service account json credential templates for uploaded public keys data. | | +| [name](outputs.tf#L39) | Service account id. | | +| [service_account](outputs.tf#L44) | Service account resource. | | +| [service_account_credentials](outputs.tf#L49) | Service account json credential templates for uploaded public keys data. | | diff --git a/modules/iam-service-account/iam.tf b/modules/iam-service-account/iam.tf index b50fadecf..1aa260a76 100644 --- a/modules/iam-service-account/iam.tf +++ b/modules/iam-service-account/iam.tf @@ -45,6 +45,13 @@ locals { ] ] ]) + iam_sa_pairs = flatten([ + for entity, roles in var.iam_sa_roles : [ + for role in roles : [ + { entity = entity, role = role } + ] + ] + ]) iam_storage_pairs = flatten([ for entity, roles in var.iam_storage_roles : [ for role in roles : [ @@ -101,6 +108,16 @@ resource "google_project_iam_member" "project-roles" { member = local.resource_iam_email } +resource "google_service_account_iam_member" "additive" { + for_each = { + for pair in local.iam_sa_pairs : + "${pair.entity}-${pair.role}" => pair + } + service_account_id = each.value.entity + role = each.value.role + member = local.resource_iam_email +} + resource "google_storage_bucket_iam_member" "bucket-roles" { for_each = { for pair in local.iam_storage_pairs : diff --git a/modules/iam-service-account/main.tf b/modules/iam-service-account/main.tf index 329d676e4..37f8205b4 100644 --- a/modules/iam-service-account/main.tf +++ b/modules/iam-service-account/main.tf @@ -21,10 +21,14 @@ locals { ? google_service_account_key.key["1"] : map("", null) , {}) - prefix = var.prefix != null ? "${var.prefix}-" : "" - resource_email_static = "${local.prefix}${var.name}@${var.project_id}.iam.gserviceaccount.com" + prefix = var.prefix != null ? "${var.prefix}-" : "" + resource_email_static = "${local.prefix}${var.name}@${var.project_id}.iam.gserviceaccount.com" + resource_iam_email = ( + local.service_account != null + ? "serviceAccount:${local.service_account.email}" + : local.resource_iam_email_static + ) resource_iam_email_static = "serviceAccount:${local.resource_email_static}" - resource_iam_email = local.service_account != null ? "serviceAccount:${local.service_account.email}" : local.resource_iam_email_static service_account = ( var.service_account_create ? try(google_service_account.service_account.0, null) diff --git a/modules/iam-service-account/outputs.tf b/modules/iam-service-account/outputs.tf index 8653ccc7c..8234ed96c 100644 --- a/modules/iam-service-account/outputs.tf +++ b/modules/iam-service-account/outputs.tf @@ -36,6 +36,11 @@ output "key" { value = local.key } +output "name" { + description = "Service account id." + value = local.service_account.name +} + output "service_account" { description = "Service account resource." value = local.service_account diff --git a/modules/iam-service-account/variables.tf b/modules/iam-service-account/variables.tf index 93fc7fe17..ee1561343 100644 --- a/modules/iam-service-account/variables.tf +++ b/modules/iam-service-account/variables.tf @@ -40,35 +40,42 @@ variable "iam" { } variable "iam_billing_roles" { - description = "Billing account roles granted to the service account, by billing account id. Non-authoritative." + description = "Billing account roles granted to this service account, by billing account id. Non-authoritative." type = map(list(string)) default = {} nullable = false } variable "iam_folder_roles" { - description = "Folder roles granted to the service account, by folder id. Non-authoritative." + description = "Folder roles granted to this service account, by folder id. Non-authoritative." type = map(list(string)) default = {} nullable = false } variable "iam_organization_roles" { - description = "Organization roles granted to the service account, by organization id. Non-authoritative." + description = "Organization roles granted to this service account, by organization id. Non-authoritative." type = map(list(string)) default = {} nullable = false } variable "iam_project_roles" { - description = "Project roles granted to the service account, by project id." + description = "Project roles granted to this service account, by project id." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_sa_roles" { + description = "Service account roles granted to this service account, by service account name." type = map(list(string)) default = {} nullable = false } variable "iam_storage_roles" { - description = "Storage roles granted to the service account, by bucket name." + description = "Storage roles granted to this service account, by bucket name." type = map(list(string)) default = {} nullable = false diff --git a/tests/fast/stages/s01_resman/fixture/main.tf b/tests/fast/stages/s01_resman/fixture/main.tf index e4e1bdf35..ddb9aafef 100644 --- a/tests/fast/stages/s01_resman/fixture/main.tf +++ b/tests/fast/stages/s01_resman/fixture/main.tf @@ -15,8 +15,13 @@ */ module "stage" { - source = "../../../../../fast/stages/01-resman" - automation_project_id = "fast-prod-automation" + source = "../../../../../fast/stages/01-resman" + automation = { + federated_identity_pool = null + federated_identity_providers = null + project_id = "fast-prod-automation" + outputs_bucket = "test" + } billing_account = { id = "000000-111111-222222" organization_id = 123456789012 diff --git a/tests/fast/stages/s02_networking_nva/fixture/main.tf b/tests/fast/stages/s02_networking_nva/fixture/main.tf index e978cf9ef..f0ff8ad03 100644 --- a/tests/fast/stages/s02_networking_nva/fixture/main.tf +++ b/tests/fast/stages/s02_networking_nva/fixture/main.tf @@ -17,6 +17,9 @@ module "stage" { source = "../../../../../fast/stages/02-networking-nva" data_dir = "../../../../../fast/stages/02-networking-nva/data/" + automation = { + outputs_bucket = "test" + } billing_account = { id = "000000-111111-222222" organization_id = 123456789012 diff --git a/tests/fast/stages/s02_networking_peering/fixture/main.tf b/tests/fast/stages/s02_networking_peering/fixture/main.tf index b06bad39f..420409590 100644 --- a/tests/fast/stages/s02_networking_peering/fixture/main.tf +++ b/tests/fast/stages/s02_networking_peering/fixture/main.tf @@ -17,6 +17,9 @@ module "stage" { source = "../../../../../fast/stages/02-networking-peering" data_dir = "../../../../../fast/stages/02-networking-peering/data/" + automation = { + outputs_bucket = "test" + } billing_account = { id = "000000-111111-222222" organization_id = 123456789012 diff --git a/tests/fast/stages/s02_networking_vpn/fixture/main.tf b/tests/fast/stages/s02_networking_vpn/fixture/main.tf index 9a736685d..6d7b8840f 100644 --- a/tests/fast/stages/s02_networking_vpn/fixture/main.tf +++ b/tests/fast/stages/s02_networking_vpn/fixture/main.tf @@ -17,6 +17,9 @@ module "stage" { source = "../../../../../fast/stages/02-networking-vpn" data_dir = "../../../../../fast/stages/02-networking-vpn/data/" + automation = { + outputs_bucket = "test" + } billing_account = { id = "000000-111111-222222" organization_id = 123456789012 diff --git a/tests/fast/stages/s02_security/fixture/main.tf b/tests/fast/stages/s02_security/fixture/main.tf index 14e2eb5b5..9947fc49e 100644 --- a/tests/fast/stages/s02_security/fixture/main.tf +++ b/tests/fast/stages/s02_security/fixture/main.tf @@ -16,6 +16,9 @@ module "stage" { source = "../../../../../fast/stages/02-security" + automation = { + outputs_bucket = "test" + } billing_account = { id = "000000-111111-222222" organization_id = 123456789012 diff --git a/tools/check_links.py b/tools/check_links.py index e96db72e8..77dc61739 100755 --- a/tools/check_links.py +++ b/tools/check_links.py @@ -79,19 +79,21 @@ def check_docs(dir_name, external=False): help='Whether to test external links.') def main(dirs, external): 'Checks links in Markdown files contained in dirs.' - errors = 0 + errors = [] for dir_name in dirs: print(f'----- {dir_name} -----') for doc in check_docs(dir_name, external): state = 'โœ“' if all(l.valid for l in doc.links) else 'โœ—' print(f'[{state}] {doc.relpath} ({len(doc.links)})') if state == 'โœ—': - errors += 1 + error = [f'{dir_name}{doc.relpath}'] for l in doc.links: if not l.valid: + error.append(f' - {l.dest}') print(f' {l.dest}') + errors.append('\n'.join(error)) if errors: - raise SystemExit('Errors found.') + raise SystemExit('Errors found:\n{}'.format('\n'.join(errors))) if __name__ == '__main__': diff --git a/tools/state_iam.py b/tools/state_iam.py index 42f9f76ea..a428f782f 100755 --- a/tools/state_iam.py +++ b/tools/state_iam.py @@ -65,7 +65,13 @@ def get_bindings(resources, prefix=None, folders=None): member_type, _, member_id = member.partition(':') if member_type == 'user': continue - member_id, member_domain = member_id.split('@', 1) + try: + member_id, member_domain = member_id.split('@', 1) + except ValueError: + if member_type == 'domain': + member_id = 'GCP organization domain' + member_domain = '' + # raise SystemExit(f'Cannot parse binding {member_id}') # Handle Cloud Services Service Account if member_domain == 'cloudservices.gserviceaccount.com': member_id = "PROJECT_CLOUD_SERVICES"