diff --git a/fast/stages/0-org-setup/schemas/project.schema.json b/fast/stages/0-org-setup/schemas/project.schema.json index d0df5f57a..fa9f09ad1 100644 --- a/fast/stages/0-org-setup/schemas/project.schema.json +++ b/fast/stages/0-org-setup/schemas/project.schema.json @@ -530,6 +530,10 @@ "name": { "type": "string" }, + "create": { + "type": "boolean", + "default": true + }, "description": { "type": "string" }, diff --git a/modules/project-factory/README.md b/modules/project-factory/README.md index b33547d31..c0e417066 100644 --- a/modules/project-factory/README.md +++ b/modules/project-factory/README.md @@ -31,7 +31,9 @@ The code is meant to be executed by a high level service account with powerful p - [Factory-wide project defaults, merges, optionals](#factory-wide-project-defaults-merges-optionals) - [Project templates](#project-templates) - [Service accounts and buckets](#service-accounts-and-buckets) - - [Automation project and resources](#automation-project-and-resources) + - [Automation resources](#automation-resources) + - [Prefix handling](#prefix-handling) + - [Complete automation example](#complete-automation-example) - [Billing budgets](#billing-budgets) - [Context-based interpolation](#context-based-interpolation) - [Folder context ids](#folder-context-ids) @@ -118,17 +120,60 @@ buckets: - $iam_principals:service_accounts/my-project/terraform-rw ``` -### Automation project and resources +### Automation resources -Other than creating automation resources within the project via the `service_accounts` and `buckets` attributes, this module also support management of automation resources created in a separate controlling project. This allows grating broad roles on the project, while still making sure that the automation resources used for Terraform cannot be manipulated from the same identities. +Other than creating automation resources within the project via the `service_accounts` and `buckets` attributes, this module also supports management of automation resources created in a separate controlling project. + +This allows granting broad roles on the project while ensuring that the automation resources used for Terraform are under a separate span of control. It also allows grouping together in a single file all resources specific to the same task, making template distribution easier. Automation resources are defined via the `automation` attribute in project configurations, which supports: - a mandatory `project` attribute to define the external controlling project; this attribute does not support interpolation and needs to be explicit - an optional `service_accounts` list where each element defines a service account in the controlling project -- an optional `bucket` which defines a bucket in the controlling project, and the map of roles/principals in the corresponding value assigned on the created bucket; principals can refer to the created service accounts by key +- an optional `bucket` which defines a bucket and/org managed folders in the controlling project; bucket names cannot use interpolation so where bucket creation is not needed, they need to be explicit -Service accounts and buckets are prefixed with the project name. Service accounts use the key specified in the YAML file as a suffix, while buckets use a default `tf-state` suffix. +#### Prefix handling + +To easily distinguish automation resources in the controlling project, service account and bucket names use a prefix that embeds the "local" project name to the default prefix. Due to the difference in maximum length and name uniqueness, service accounts and buckets treat the prefix differently. + +For service accounts the global prefix is ignored, and the "local" project name is used as a prefix. For example, a project defined in a `prod-app-example-0.yaml` file where the prefix is `foo` will have the `rw` automation service account resulting in the `prod-app-example-0-rw` name. + +For GCS buckets the global prefix is kept to ensure name uniqueness, and the "local" project name is appended. For example, a project defined in a `prod-app-example-0.yaml` file where the prefix is `foo` will have the `tf-state` automation bucket resulting in the `foo-prod-app-example-0-tf-state` name. + +This behaviour changes when bucket creation is set to `false`, which is the pattern used when GCS managed folders are used for each project automation. In these cases the prefix for the bucket is not suffixed with the local project name, to make it possible to refer to the pre-existing bucket. + +The difference in the two behaviours is shown in the snippets below. + +```yaml +# file/project name: prod-example-app-0 +# prefix via factory defaults: foo + +automation: + project: $project_ids:iac-core-0 + bucket: + name: tf-state + +# bucket is created, name is foo-prod-example-app-0-tf-state +``` + +```yaml +# file/project name: prod-example-app-0 +# prefix via factory defaults: foo +# pre-existing bucket: foo-prod-iac-core-0-shared-tf-state + +automation: + project: $project_ids:iac-core-0 + bucket: + name: prod-iac-core-0-shared-tf-state + create: false + managed_folders: + prod-example-app-0: {} + +# managed folder prod-example-app-0 is created +# in bucket foo-prod-iac-core-0-shared-tf-state +``` + +#### Complete automation example ```yaml # file name: prod-app-example-0 diff --git a/modules/project-factory/automation.tf b/modules/project-factory/automation.tf index a23b2f47a..11a52070f 100644 --- a/modules/project-factory/automation.tf +++ b/modules/project-factory/automation.tf @@ -18,23 +18,20 @@ locals { _automation = merge( { for k, v in local.folders_input : k => { - bucket = try(v.automation.bucket, {}) - # name = replace(k, "/", "-") + bucket = try(v.automation.bucket, {}) + parent_name = replace(k, "/", "-") parent_type = "folder" - prefix = try(v.automation.prefix, null) + prefix = coalesce(try(v.automation.prefix, null), v.prefix) project = try(v.automation.project, null) service_accounts = try(v.automation.service_accounts, {}) } if try(v.automation.bucket, null) != null }, { for k, v in local.projects_input : k => { - bucket = try(v.automation.bucket, {}) - # name = v.name - parent_type = "project" - prefix = coalesce( - try(v.automation.prefix, null), - v.prefix == null ? v.name : "${v.prefix}-${v.name}" - ) + bucket = try(v.automation.bucket, {}) + parent_name = k + parent_type = "project" + prefix = coalesce(try(v.automation.prefix, null), v.prefix) project = try(v.automation.project, null) service_accounts = try(v.automation.service_accounts, {}) } if try(v.automation.bucket, null) != null @@ -43,12 +40,12 @@ locals { _automation_buckets = { for k, v in local._automation : k => merge(v.bucket, { automation_project = v.project - source_project = k - name = lookup(v, "name", "tf-state") - # project automation always has a prefix + parent_name = v.parent_name + name = lookup(v.bucket, "name", "tf-state") + create = lookup(v.bucket, "create", true) prefix = try(coalesce( - v.prefix, local.data_defaults.overrides.prefix, + v.prefix, local.data_defaults.defaults.prefix ), null) }) @@ -57,10 +54,9 @@ locals { for k, v in local._automation : [ for sk, sv in v.service_accounts : merge(sv, { automation_project = v.project - source_project = k name = sk parent = k - prefix = v.prefix + parent_name = v.parent_name }) ] ])) @@ -79,11 +75,16 @@ locals { } module "automation-bucket" { - source = "../gcs" - for_each = local.automation_buckets - project_id = each.value.automation_project - prefix = each.value.prefix + source = "../gcs" + for_each = local.automation_buckets + project_id = each.value.automation_project + prefix = ( + each.value.create == false + ? each.value.prefix + : "${each.value.prefix}-${each.value.parent_name}" + ) name = each.value.name + bucket_create = each.value.create encryption_key = lookup(each.value, "encryption_key", null) force_destroy = try(coalesce( local.data_defaults.overrides.bucket.force_destroy, @@ -99,7 +100,7 @@ module "automation-bucket" { iam_bindings_additive = lookup(each.value, "iam_bindings_additive", {}) labels = lookup(each.value, "labels", {}) managed_folders = lookup(each.value, "managed_folders", {}) - location = coalesce( + location = each.value.create == false ? null : coalesce( local.data_defaults.overrides.storage_location, lookup(each.value, "location", null), local.data_defaults.defaults.storage_location @@ -119,7 +120,7 @@ module "automation-service-accounts" { source = "../iam-service-account" for_each = local.automation_sas project_id = each.value.automation_project - prefix = each.value.prefix + prefix = each.value.parent_name name = each.value.name description = lookup(each.value, "description", null) display_name = lookup( diff --git a/modules/project-factory/outputs.tf b/modules/project-factory/outputs.tf index d0858b9bc..d80effa8b 100644 --- a/modules/project-factory/outputs.tf +++ b/modules/project-factory/outputs.tf @@ -16,10 +16,10 @@ locals { _outputs_automation_buckets = { - for k, v in local.automation_buckets : v.source_project => k + for k, v in local.automation_buckets : v.parent_name => k } _outputs_automation_sas = { - for k, v in local.automation_sas : v.source_project => k... + for k, v in local.automation_sas : v.parent_name => k... } outputs_projects = { for k, v in local.projects_input : k => { diff --git a/modules/project-factory/projects-buckets.tf b/modules/project-factory/projects-buckets.tf index b87686aed..e200c4cd4 100644 --- a/modules/project-factory/projects-buckets.tf +++ b/modules/project-factory/projects-buckets.tf @@ -21,6 +21,7 @@ locals { project_key = k project_name = v.name name = name + create = lookup(opts, "create", true) description = lookup(opts, "description", "Terraform-managed.") encryption_key = lookup(opts, "encryption_key", null) force_destroy = try(coalesce( @@ -62,6 +63,7 @@ module "buckets" { project_id = module.projects[each.value.project_key].project_id prefix = each.value.prefix name = "${each.value.project_name}-${each.value.name}" + bucket_create = each.value.create encryption_key = each.value.encryption_key force_destroy = each.value.force_destroy context = merge(local.ctx, { diff --git a/modules/project-factory/schemas/project.schema.json b/modules/project-factory/schemas/project.schema.json index d0df5f57a..fa9f09ad1 100644 --- a/modules/project-factory/schemas/project.schema.json +++ b/modules/project-factory/schemas/project.schema.json @@ -530,6 +530,10 @@ "name": { "type": "string" }, + "create": { + "type": "boolean", + "default": true + }, "description": { "type": "string" }, diff --git a/tests/modules/project_factory/examples/example.yaml b/tests/modules/project_factory/examples/example.yaml index 4af807cd3..a69cf903d 100644 --- a/tests/modules/project_factory/examples/example.yaml +++ b/tests/modules/project_factory/examples/example.yaml @@ -44,7 +44,7 @@ values: : bucket: test-pf-dev-tb-app0-0-tf-state condition: [] members: - - serviceAccount:test-pf-dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com + - serviceAccount:dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com role: roles/storage.objectCreator ? module.project-factory.module.automation-bucket["dev-tb-app0-0/automation/tf-state"].google_storage_bucket_iam_binding.authoritative["roles/storage.objectViewer"] : bucket: test-pf-dev-tb-app0-0-tf-state @@ -52,27 +52,27 @@ values: members: - group:gcp-devops@example.org - group:team-b-admins@example.org - - serviceAccount:test-pf-dev-tb-app0-0-ro@test-pf-teams-iac-0.iam.gserviceaccount.com - - serviceAccount:test-pf-dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com + - serviceAccount:dev-tb-app0-0-ro@test-pf-teams-iac-0.iam.gserviceaccount.com + - serviceAccount:dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com role: roles/storage.objectViewer ? module.project-factory.module.automation-service-accounts["dev-tb-app0-0/automation/ro"].google_service_account.service_account[0] - : account_id: test-pf-dev-tb-app0-0-ro + : account_id: dev-tb-app0-0-ro create_ignore_already_exists: null description: Team B app 0 read-only automation sa. disabled: false display_name: Service account ro for dev-tb-app0-0. - email: test-pf-dev-tb-app0-0-ro@test-pf-teams-iac-0.iam.gserviceaccount.com - member: serviceAccount:test-pf-dev-tb-app0-0-ro@test-pf-teams-iac-0.iam.gserviceaccount.com + email: dev-tb-app0-0-ro@test-pf-teams-iac-0.iam.gserviceaccount.com + member: serviceAccount:dev-tb-app0-0-ro@test-pf-teams-iac-0.iam.gserviceaccount.com project: test-pf-teams-iac-0 timeouts: null ? module.project-factory.module.automation-service-accounts["dev-tb-app0-0/automation/rw"].google_service_account.service_account[0] - : account_id: test-pf-dev-tb-app0-0-rw + : account_id: dev-tb-app0-0-rw create_ignore_already_exists: null description: Team B app 0 read/write automation sa. disabled: false display_name: Service account rw for dev-tb-app0-0. - email: test-pf-dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com - member: serviceAccount:test-pf-dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com + email: dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com + member: serviceAccount:dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com project: test-pf-teams-iac-0 timeouts: null module.project-factory.module.billing-budgets[0].google_billing_budget.default["test-100"]: @@ -195,13 +195,13 @@ values: module.project-factory.module.projects-iam["dev-tb-app0-0"].google_project_iam_binding.authoritative["roles/owner"]: condition: [] members: - - serviceAccount:test-pf-dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com + - serviceAccount:dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com project: test-pf-dev-tb-app0-0 role: roles/owner module.project-factory.module.projects-iam["dev-tb-app0-0"].google_project_iam_binding.authoritative["roles/viewer"]: condition: [] members: - - serviceAccount:test-pf-dev-tb-app0-0-ro@test-pf-teams-iac-0.iam.gserviceaccount.com + - serviceAccount:dev-tb-app0-0-ro@test-pf-teams-iac-0.iam.gserviceaccount.com project: test-pf-dev-tb-app0-0 role: roles/viewer module.project-factory.module.projects-iam["dev-tb-app0-1"].google_project_iam_binding.authoritative["roles/run.admin"]: @@ -571,7 +571,7 @@ values: ? module.project-factory.module.service_accounts-iam["dev-tb-app0-0/vm-default"].google_service_account_iam_binding.authoritative["roles/iam.serviceAccountTokenCreator"] : condition: [] members: - - serviceAccount:test-pf-dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com + - serviceAccount:dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com role: roles/iam.serviceAccountTokenCreator module.project-factory.terraform_data.defaults_preconditions: input: null