From 505ee02fef40f9a16e17551010a17b69e9e6a9dc Mon Sep 17 00:00:00 2001 From: kovagoadam Date: Wed, 1 Oct 2025 12:12:45 +0200 Subject: [PATCH] Add support for billing export in 0-org-setup (#3347) * Add support for billing export in 0-org-setup * Merge branch 'master' into add-billing-export-support * Refactored billing export with adding support for bigquery_datasets in project_factory * Renamed bigquery_dataset to datasets * Fixed defaults.schema.md * Fixed default.schema.md again * Cleanup md's * Fixed boilerplate * Fixed JSON schema * reword README, rename project file * Moved dataset_id to the key of the map --------- Co-authored-by: Julio Castillo Co-authored-by: Ludovico Magnocavallo --- fast/stages/0-org-setup/README.md | 6 +- fast/stages/0-org-setup/data/defaults.yaml | 1 + .../data/projects/core/billing-0.yaml | 29 ++++++ .../0-org-setup/schemas/defaults.schema.json | 6 ++ .../0-org-setup/schemas/defaults.schema.md | 2 + .../0-org-setup/schemas/project.schema.json | 19 ++++ .../0-org-setup/schemas/project.schema.md | 6 ++ modules/project-factory/README.md | 9 +- modules/project-factory/projects-bigquery.tf | 44 +++++++++ modules/project-factory/projects-defaults.tf | 2 + .../schemas/project.schema.json | 19 ++++ .../project-factory/schemas/project.schema.md | 6 ++ modules/project-factory/variables.tf | 2 + .../s0_org_setup/data-simple/defaults.yaml | 1 + .../fast/stages/s0_org_setup/not-simple.yaml | 97 +++++++++++++++++-- 15 files changed, 234 insertions(+), 15 deletions(-) create mode 100644 fast/stages/0-org-setup/data/projects/core/billing-0.yaml create mode 100644 modules/project-factory/projects-bigquery.tf diff --git a/fast/stages/0-org-setup/README.md b/fast/stages/0-org-setup/README.md index a75ae03dd..fc6db3051 100644 --- a/fast/stages/0-org-setup/README.md +++ b/fast/stages/0-org-setup/README.md @@ -19,7 +19,7 @@ - [Context interpolation](#context-interpolation) - [Factory data](#factory-data) - [Defaults configuration](#defaults-configuration) - - [Billing account IAM](#billing-account-iam) + - [Billing account IAM and billing export](#billing-account-iam-and-billing-export) - [Context-based replacement in the billing account factory](#context-based-replacement-in-the-billing-account-factory) - [Organization configuration](#organization-configuration) - [Context-based replacement in organization factories](#context-based-replacement-in-organization-factories) @@ -355,7 +355,7 @@ context: gcp-organization-admins: group:fabric-fast-owners@example.com ``` -### Billing account IAM +### Billing account IAM and billing export FAST traditionally supports three different billing configurations: @@ -371,6 +371,8 @@ This stage allows the same flexibility, and even makes it possible to mix and ma The default dataset assumes an externally managed billing account is used, and configures its IAM accordingly via the billing account factory. The example below shows some of the IAM bindings configured at the billing account level, and how context-based interpolation is used there. +Where billing exports need to be configured as part of a FAST installation, the default dataset includes a dedicated project and a BigQuery dataset that can be used as part of [manual process to set up exports](https://cloud.google.com/billing/docs/how-to/export-data-bigquery-setup#enable-bq-export). +
Context-based replacement examples for the billing accounts factory diff --git a/fast/stages/0-org-setup/data/defaults.yaml b/fast/stages/0-org-setup/data/defaults.yaml index 04d4e9324..6512f758a 100644 --- a/fast/stages/0-org-setup/data/defaults.yaml +++ b/fast/stages/0-org-setup/data/defaults.yaml @@ -30,6 +30,7 @@ projects: # prefix must be unique and less than 9 characters prefix: test00 storage_location: europe-west1 + bigquery_location: europe-west1 overrides: {} context: # you can populate context variables here for use in YAML replacements diff --git a/fast/stages/0-org-setup/data/projects/core/billing-0.yaml b/fast/stages/0-org-setup/data/projects/core/billing-0.yaml new file mode 100644 index 000000000..e419b3f28 --- /dev/null +++ b/fast/stages/0-org-setup/data/projects/core/billing-0.yaml @@ -0,0 +1,29 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../../schemas/project.schema.json + +name: prod-billing-exp-0 +iam_by_principals: + $iam_principals:service_accounts/iac-0/iac-org-ro: + - roles/viewer + $iam_principals:service_accounts/iac-0/iac-org-rw: + - roles/owner +services: + - bigquery.googleapis.com + - bigquerydatatransfer.googleapis.com + - storage.googleapis.com +datasets: + billing_export: + friendly_name: Billing export \ No newline at end of file diff --git a/fast/stages/0-org-setup/schemas/defaults.schema.json b/fast/stages/0-org-setup/schemas/defaults.schema.json index ea0af8c6d..73f699afc 100644 --- a/fast/stages/0-org-setup/schemas/defaults.schema.json +++ b/fast/stages/0-org-setup/schemas/defaults.schema.json @@ -342,6 +342,9 @@ "required": [ "perimeter_name" ] + }, + "bigquery_location": { + "type": "string" } } }, @@ -504,6 +507,9 @@ "required": [ "perimeter_name" ] + }, + "bigquery_location": { + "type": "string" } } } diff --git a/fast/stages/0-org-setup/schemas/defaults.schema.md b/fast/stages/0-org-setup/schemas/defaults.schema.md index bc2b41476..c5b4fdf39 100644 --- a/fast/stages/0-org-setup/schemas/defaults.schema.md +++ b/fast/stages/0-org-setup/schemas/defaults.schema.md @@ -85,6 +85,7 @@ - **is_dry_run**: *boolean* - **logging_data_access**: *object* *additional properties: Object* + - **bigquery_location**: *string* - **overrides**: *object*
*additional properties: false* - **billing_account**: *string* @@ -113,6 +114,7 @@ - **is_dry_run**: *boolean* - **logging_data_access**: *object* *additional properties: Object* + - **bigquery_location**: *string* - **context**: *object*
*additional properties: false* - **iam_principals**: *object* diff --git a/fast/stages/0-org-setup/schemas/project.schema.json b/fast/stages/0-org-setup/schemas/project.schema.json index fa9f09ad1..098ea9d8a 100644 --- a/fast/stages/0-org-setup/schemas/project.schema.json +++ b/fast/stages/0-org-setup/schemas/project.schema.json @@ -520,6 +520,25 @@ "type": "boolean" } } + }, + "datasets": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "friendly_name": { + "type": "string" + }, + "location": { + "type": "string" + } + + } + } + } } }, "$defs": { diff --git a/fast/stages/0-org-setup/schemas/project.schema.md b/fast/stages/0-org-setup/schemas/project.schema.md index f8aa04735..8f88ebd99 100644 --- a/fast/stages/0-org-setup/schemas/project.schema.md +++ b/fast/stages/0-org-setup/schemas/project.schema.md @@ -143,6 +143,12 @@ - **vpc_sc**: *object* - ⁺**perimeter_name**: *string* - **is_dry_run**: *boolean* +- **datasets**: *object* +
*additional properties: false* + - **`^[a-z0-9_]+$`**: *object* +
*additional properties: false* + - **friendly_name**: *string* + - **location**: *string* ## Definitions diff --git a/modules/project-factory/README.md b/modules/project-factory/README.md index 92b05cb72..d083981bc 100644 --- a/modules/project-factory/README.md +++ b/modules/project-factory/README.md @@ -678,6 +678,7 @@ service_accounts: | [folders.tf](./folders.tf) | Folder hierarchy factory resources. | folder | | | [main.tf](./main.tf) | Projects and billing budgets factory resources. | | terraform_data | | [outputs.tf](./outputs.tf) | Module outputs. | | | +| [projects-bigquery.tf](./projects-bigquery.tf) | None | bigquery-dataset | | | [projects-buckets.tf](./projects-buckets.tf) | None | gcs | | | [projects-defaults.tf](./projects-defaults.tf) | None | | | | [projects-log-buckets.tf](./projects-log-buckets.tf) | None | logging-bucket | | @@ -692,11 +693,11 @@ service_accounts: | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [factories_config](variables.tf#L171) | Path to folder with YAML resource description data files. | object({…}) | ✓ | | +| [factories_config](variables.tf#L173) | Path to folder with YAML resource description data files. | object({…}) | ✓ | | | [context](variables.tf#L17) | Context-specific interpolations. | object({…}) | | {} | -| [data_defaults](variables.tf#L36) | Optional default values used when corresponding project or folder data from files are missing. | object({…}) | | {} | -| [data_merges](variables.tf#L107) | Optional values that will be merged with corresponding data from files. Combines with `data_defaults`, file data, and `data_overrides`. | object({…}) | | {} | -| [data_overrides](variables.tf#L126) | Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`. | object({…}) | | {} | +| [data_defaults](variables.tf#L36) | Optional default values used when corresponding project or folder data from files are missing. | object({…}) | | {} | +| [data_merges](variables.tf#L108) | Optional values that will be merged with corresponding data from files. Combines with `data_defaults`, file data, and `data_overrides`. | object({…}) | | {} | +| [data_overrides](variables.tf#L127) | Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`. | object({…}) | | {} | | [folders](variables-folders.tf#L17) | Folders data merged with factory data. | map(object({…})) | | {} | | [notification_channels](variables-billing.tf#L17) | Notification channels used by budget alerts. | map(object({…})) | | {} | | [projects](variables-projects.tf#L17) | Projects data merged with factory data. | map(object({…})) | | {} | diff --git a/modules/project-factory/projects-bigquery.tf b/modules/project-factory/projects-bigquery.tf new file mode 100644 index 000000000..650261b0c --- /dev/null +++ b/modules/project-factory/projects-bigquery.tf @@ -0,0 +1,44 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + projects_bigquery_datasets = flatten([ + for k, v in local.projects_input : [ + for name, opts in lookup(v, "datasets", {}) : { + project_key = k + project_name = v.name + id = name + friendly_name = lookup(opts, "friendly_name", null) + location = lookup(opts, "location", null) + } + ] + ]) +} + +module "bigquery-datasets" { + source = "../bigquery-dataset" + for_each = { + for k in local.projects_bigquery_datasets : "${k.project_key}/${k.id}" => k + } + project_id = module.projects[each.value.project_key].project_id + id = each.value.id + friendly_name = each.value.friendly_name + location = coalesce( + local.data_defaults.overrides.bigquery_location, + lookup(each.value, "location", null), + local.data_defaults.defaults.bigquery_location + ) +} \ No newline at end of file diff --git a/modules/project-factory/projects-defaults.tf b/modules/project-factory/projects-defaults.tf index 14bff2a81..42f90801d 100644 --- a/modules/project-factory/projects-defaults.tf +++ b/modules/project-factory/projects-defaults.tf @@ -100,6 +100,7 @@ locals { ) ) logging_data_access = {} + bigquery_location = null }, try( local._data_defaults.defaults, {} @@ -144,6 +145,7 @@ locals { null ) logging_data_access = null + bigquery_location = null }, try( local._data_defaults.overrides, {} diff --git a/modules/project-factory/schemas/project.schema.json b/modules/project-factory/schemas/project.schema.json index fa9f09ad1..098ea9d8a 100644 --- a/modules/project-factory/schemas/project.schema.json +++ b/modules/project-factory/schemas/project.schema.json @@ -520,6 +520,25 @@ "type": "boolean" } } + }, + "datasets": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "friendly_name": { + "type": "string" + }, + "location": { + "type": "string" + } + + } + } + } } }, "$defs": { diff --git a/modules/project-factory/schemas/project.schema.md b/modules/project-factory/schemas/project.schema.md index 900523134..baf3df987 100644 --- a/modules/project-factory/schemas/project.schema.md +++ b/modules/project-factory/schemas/project.schema.md @@ -146,6 +146,12 @@ - **vpc_sc**: *object* - ⁺**perimeter_name**: *string* - **is_dry_run**: *boolean* +- **datasets**: *object* +
*additional properties: false* + - **`^[a-z0-9_]+$`**: *object* +
*additional properties: false* + - **friendly_name**: *string* + - **location**: *string* ## Definitions diff --git a/modules/project-factory/variables.tf b/modules/project-factory/variables.tf index cad34971f..e3462fd8e 100644 --- a/modules/project-factory/variables.tf +++ b/modules/project-factory/variables.tf @@ -99,6 +99,7 @@ variable "data_defaults" { perimeter_name = string is_dry_run = optional(bool, false) })) + bigquery_location = optional(string) }) nullable = false default = {} @@ -163,6 +164,7 @@ variable "data_overrides" { perimeter_name = string is_dry_run = optional(bool, false) })) + bigquery_location = optional(string) }) nullable = false default = {} diff --git a/tests/fast/stages/s0_org_setup/data-simple/defaults.yaml b/tests/fast/stages/s0_org_setup/data-simple/defaults.yaml index 968e1d5b6..c1efefac0 100644 --- a/tests/fast/stages/s0_org_setup/data-simple/defaults.yaml +++ b/tests/fast/stages/s0_org_setup/data-simple/defaults.yaml @@ -29,6 +29,7 @@ projects: defaults: prefix: ft0 storage_location: europe-west1 + bigquery_location: europe-west1 overrides: {} output_files: local_path: /tmp/fast-config diff --git a/tests/fast/stages/s0_org_setup/not-simple.yaml b/tests/fast/stages/s0_org_setup/not-simple.yaml index 159108d9f..fb1b2efbf 100644 --- a/tests/fast/stages/s0_org_setup/not-simple.yaml +++ b/tests/fast/stages/s0_org_setup/not-simple.yaml @@ -472,6 +472,26 @@ values: condition: [] member: serviceAccount:iac-security-rw@ft0-prod-iac-core-0.iam.gserviceaccount.com role: roles/billing.user + module.factory.module.bigquery-datasets["billing-0/billing_export"].google_bigquery_dataset.default: + dataset_id: billing_export + default_encryption_configuration: [] + default_partition_expiration_ms: null + default_table_expiration_ms: null + delete_contents_on_destroy: false + description: Terraform managed. + effective_labels: + goog-terraform-provisioned: 'true' + external_catalog_dataset_options: [] + external_dataset_reference: [] + friendly_name: Billing export + labels: null + location: europe-west1 + max_time_travel_hours: '168' + project: ft0-prod-billing-exp-0 + resource_tags: null + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null module.billing-accounts["default"].google_billing_account_iam_member.bindings["billing_viewer_org_ro"]: billing_account_id: 012345-012345-012345 condition: [] @@ -1013,6 +1033,18 @@ values: locked: null project: ft0-prod-audit-logs-0 retention_days: 31 + module.factory.module.projects-iam["billing-0"].google_project_iam_binding.authoritative["roles/owner"]: + condition: [] + members: + - serviceAccount:iac-org-rw@ft0-prod-iac-core-0.iam.gserviceaccount.com + project: ft0-prod-billing-exp-0 + role: roles/owner + module.factory.module.projects-iam["billing-0"].google_project_iam_binding.authoritative["roles/viewer"]: + condition: [] + members: + - serviceAccount:iac-org-ro@ft0-prod-iac-core-0.iam.gserviceaccount.com + project: ft0-prod-billing-exp-0 + role: roles/viewer module.factory.module.projects-iam["iac-0"].google_project_iam_binding.authoritative["$custom_roles:storage_viewer"]: condition: [] members: @@ -1098,6 +1130,52 @@ values: - serviceAccount:iac-org-ro@ft0-prod-iac-core-0.iam.gserviceaccount.com project: ft0-prod-audit-logs-0 role: roles/viewer + module.factory.module.projects["billing-0"].data.google_bigquery_default_service_account.bq_sa[0]: + project: ft0-prod-billing-exp-0 + module.factory.module.projects["billing-0"].data.google_storage_project_service_account.gcs_sa[0]: + project: ft0-prod-billing-exp-0 + user_project: null + module.factory.module.projects["billing-0"].google_project.project[0]: + auto_create_network: false + billing_account: 012345-012345-012345 + deletion_policy: DELETE + effective_labels: + goog-terraform-provisioned: 'true' + folder_id: null + labels: null + name: ft0-prod-billing-exp-0 + org_id: '1234567890' + project_id: ft0-prod-billing-exp-0 + tags: null + terraform_labels: + goog-terraform-provisioned: 'true' + timeouts: null + module.factory.module.projects["billing-0"].google_project_iam_member.service_agents["bigquerydatatransfer"]: + condition: [] + project: ft0-prod-billing-exp-0 + role: roles/bigquerydatatransfer.serviceAgent + module.factory.module.projects["billing-0"].google_project_service.project_services["bigquery.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: ft0-prod-billing-exp-0 + service: bigquery.googleapis.com + timeouts: null + module.factory.module.projects["billing-0"].google_project_service.project_services["bigquerydatatransfer.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: ft0-prod-billing-exp-0 + service: bigquerydatatransfer.googleapis.com + timeouts: null + module.factory.module.projects["billing-0"].google_project_service.project_services["storage.googleapis.com"]: + disable_dependent_services: false + disable_on_destroy: false + project: ft0-prod-billing-exp-0 + service: storage.googleapis.com + timeouts: null + module.factory.module.projects["billing-0"].google_project_service_identity.default["bigquerydatatransfer.googleapis.com"]: + project: ft0-prod-billing-exp-0 + service: bigquerydatatransfer.googleapis.com + timeouts: null module.factory.module.projects["iac-0"].data.google_bigquery_default_service_account.bq_sa[0]: project: ft0-prod-iac-core-0 module.factory.module.projects["iac-0"].data.google_storage_project_service_account.gcs_sa[0]: @@ -2682,7 +2760,8 @@ values: output: null triggers_replace: null counts: - google_bigquery_default_service_account: 1 + google_bigquery_dataset: 1 + google_bigquery_default_service_account: 2 google_billing_account_iam_member: 6 google_folder: 8 google_folder_iam_binding: 44 @@ -2695,11 +2774,11 @@ counts: google_org_policy_policy: 37 google_organization_iam_binding: 35 google_organization_iam_custom_role: 7 - google_project: 2 - google_project_iam_binding: 14 - google_project_iam_member: 14 - google_project_service: 30 - google_project_service_identity: 8 + google_project: 3 + google_project_iam_binding: 16 + google_project_iam_member: 15 + google_project_service: 33 + google_project_service_identity: 9 google_service_account: 16 google_service_account_iam_member: 4 google_storage_bucket: 3 @@ -2707,12 +2786,12 @@ counts: google_storage_bucket_object: 9 google_storage_managed_folder: 5 google_storage_managed_folder_iam_binding: 10 - google_storage_project_service_account: 2 + google_storage_project_service_account: 3 google_tags_tag_binding: 5 google_tags_tag_key: 3 google_tags_tag_value: 5 google_tags_tag_value_iam_binding: 4 local_file: 9 - modules: 43 - resources: 297 + modules: 46 + resources: 308 terraform_data: 2 \ No newline at end of file