From fb21f6aaf866092ca2190e948a445d8129eed661 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 11 Feb 2026 16:29:49 +0100 Subject: [PATCH] Change factories_config type in FAST and project/vpc factory modules, add YAML schema validation (#3728) * stage 0 * stage 1 * networking * security * pf stage * tfdoc * align schemas * inventory * fix observability * pf module * pf module budgets * align fast stages * align project subfactories * tfdoc * schema validation * add missing schemas * Fix observability types --------- Co-authored-by: Julio Castillo --- .github/workflows/linting.yml | 5 + fast/stages/0-org-setup/README.md | 55 +++-- fast/stages/0-org-setup/billing.tf | 7 +- .../folders/security/dev/.config.yaml | 2 +- .../folders/security/prod/.config.yaml | 2 +- .../classic/folders/security/dev/.config.yaml | 2 +- .../folders/security/prod/.config.yaml | 2 +- .../observability/iac-0/impersonation.yaml | 8 +- .../datasets/classic/projects/core/iac-0.yaml | 2 +- .../hardened/billing-accounts/default.yaml | 46 ++++ .../folders/security/dev/.config.yaml | 2 +- .../folders/security/prod/.config.yaml | 2 +- .../observability/auditConfigChanges.yaml | 2 +- .../binaryAuthorizationPolicyChanges.yaml | 2 +- .../cloudsqlInstanceChanges.yaml | 2 +- .../observability/customRoleChanges.yaml | 2 +- .../firewallPolicyRuleChanges.yaml | 2 +- .../observability/firewallRuleChanges.yaml | 2 +- .../observability/networkChanges.yaml | 2 +- .../observability/networkRouteChanges.yaml | 2 +- .../observability/projectOwnershipChange.yaml | 2 +- .../observability/storageIamChanges.yaml | 2 +- .../hardened/projects/core/log-0.yaml | 2 +- fast/stages/0-org-setup/factory.tf | 6 +- fast/stages/0-org-setup/main.tf | 10 +- fast/stages/0-org-setup/observability.tf | 2 +- .../schemas/observability.schema.json | 35 +-- .../0-org-setup/schemas/project.schema.json | 2 +- fast/stages/0-org-setup/variables.tf | 19 +- fast/stages/1-vpcsc/README.md | 12 +- fast/stages/1-vpcsc/main.tf | 12 +- fast/stages/1-vpcsc/variables.tf | 15 +- fast/stages/2-networking/README.md | 2 +- .../dev/firewall-rules/default-ingress.yaml | 2 +- .../vpcs/hub/vpns/onprem.yaml | 2 + .../prod/firewall-rules/default-ingress.yaml | 2 +- .../dns/zones/fwd-root.yaml | 2 +- .../dns/zones/peer-root.yaml | 2 +- .../dns/zones/pvt-dev-test.yaml | 2 +- .../dns/zones/pvt-prod-test.yaml | 2 +- .../dns/zones/pvt-test.yaml | 2 +- .../dev/firewall-rules/default-ingress.yaml | 2 +- .../dmz/firewall-rules/default-ingress.yaml | 2 +- .../hub/firewall-rules/default-ingress.yaml | 2 +- .../prod/firewall-rules/default-ingress.yaml | 2 +- .../dev/firewall-rules/default-ingress.yaml | 2 +- .../vpcs/dev/vpns/to-hub.yaml | 2 + .../hub/firewall-rules/default-ingress.yaml | 2 +- .../vpcs/hub/vpns/onprem.yaml | 2 + .../vpcs/hub/vpns/to-dev.yaml | 2 + .../vpcs/hub/vpns/to-prod.yaml | 2 + .../prod/firewall-rules/default-ingress.yaml | 2 +- .../vpcs/prod/vpns/to-hub.yaml | 2 + fast/stages/2-networking/factory-dns.tf | 12 +- .../2-networking/factory-firewall-policies.tf | 9 +- fast/stages/2-networking/factory-ncc.tf | 5 +- fast/stages/2-networking/factory-nva.tf | 5 +- fast/stages/2-networking/factory-projects.tf | 7 +- fast/stages/2-networking/factory-vpcs.tf | 11 +- fast/stages/2-networking/factory-vpns.tf | 4 +- fast/stages/2-networking/main.tf | 10 +- .../2-networking/schemas/project.schema.json | 2 +- fast/stages/2-networking/variables.tf | 21 +- fast/stages/2-project-factory/README.md | 6 +- .../{data => datasets/classic}/defaults.yaml | 2 +- .../classic}/folders/team-a/.config.yaml | 2 +- .../classic}/folders/team-a/dev/.config.yaml | 2 +- .../classic}/folders/team-a/prod/.config.yaml | 2 +- .../classic}/projects/dev-app-a-0.yaml | 2 +- .../classic}/projects/prod-app-a-0.yaml | 2 +- fast/stages/2-project-factory/main.tf | 22 +- .../schemas/project.schema.json | 2 +- fast/stages/2-project-factory/variables.tf | 14 +- fast/stages/2-security/README.md | 24 ++- .../certificate-authorities/prod-ca-0.yaml | 2 +- .../{data => datasets/classic}/defaults.yaml | 2 +- .../keyrings/dev-primary-default.yaml | 2 +- .../keyrings/prod-primary-default.yaml | 2 +- .../classic}/projects/dev-sec-core-0.yaml | 2 +- .../classic}/projects/prod-sec-core-0.yaml | 2 +- fast/stages/2-security/factory-cas.tf | 7 +- fast/stages/2-security/factory-keyrings.tf | 7 +- fast/stages/2-security/factory-projects.tf | 7 +- fast/stages/2-security/main.tf | 10 +- .../2-security/schemas/project.schema.json | 2 +- fast/stages/2-security/variables.tf | 13 +- modules/net-vpc-factory/data/defaults.yaml | 2 +- .../net-vpc-factory/data/example/.config.yaml | 6 +- .../firewall-rules/default-ingress.yaml | 2 +- .../subnets/example-default-secondary.yaml | 2 +- .../schemas/defaults.schema.json | 59 +++++ modules/project-factory/README.md | 39 ++-- modules/project-factory/budgets.tf | 10 +- modules/project-factory/folders.tf | 7 +- modules/project-factory/main.tf | 11 +- modules/project-factory/projects-defaults.tf | 11 +- modules/project-factory/projects.tf | 24 +-- .../schemas/project.schema.json | 2 +- modules/project-factory/variables-projects.tf | 10 + modules/project-factory/variables.tf | 15 +- modules/project/README.md | 10 +- modules/project/alerts.tf | 48 +++-- .../project/schemas/observability.schema.json | 35 +-- modules/project/variables-observability.tf | 12 +- .../fast/stages/s0_org_setup/hardened.tfvars | 11 +- tests/fast/stages/s0_org_setup/hardened.yaml | 145 +++++++++---- tests/fast/stages/s0_org_setup/simple.tfvars | 6 +- .../access-levels/geo_it.yaml | 2 +- .../access-levels/identity_me.yaml | 2 +- .../egress-policies/test.yaml | 2 +- .../ingress-policies/fast-org-log-sinks.yaml | 2 +- .../ingress-policies/test.yaml | 2 +- .../perimeters/default.yaml | 2 +- .../restricted-services.yaml | 0 tests/fast/stages/s1_vpcsc/factory.tfvars | 10 +- tests/fast/stages/s1_vpcsc/hardened.tfvars | 27 +++ tests/fast/stages/s1_vpcsc/hardened.yaml | 203 ++++++++++++++++++ tests/fast/stages/s1_vpcsc/simple.tfvars | 8 +- tests/fast/stages/s1_vpcsc/tftest.yaml | 8 + tests/fast/stages/s2_networking/ncc.tfvars | 12 +- tests/fast/stages/s2_networking/nva.tfvars | 12 +- tests/fast/stages/s2_networking/simple.tfvars | 12 +- tests/fast/stages/s2_networking/vpns.tfvars | 12 +- tests/fast/stages/s2_security/simple.tfvars | 6 +- tools/check_yaml_schema.py | 131 +++++++++++ 125 files changed, 1024 insertions(+), 432 deletions(-) create mode 100644 fast/stages/0-org-setup/datasets/hardened/billing-accounts/default.yaml rename fast/stages/2-project-factory/{data => datasets/classic}/defaults.yaml (94%) rename fast/stages/2-project-factory/{data => datasets/classic}/folders/team-a/.config.yaml (90%) rename fast/stages/2-project-factory/{data => datasets/classic}/folders/team-a/dev/.config.yaml (90%) rename fast/stages/2-project-factory/{data => datasets/classic}/folders/team-a/prod/.config.yaml (89%) rename fast/stages/2-project-factory/{data => datasets/classic}/projects/dev-app-a-0.yaml (92%) rename fast/stages/2-project-factory/{data => datasets/classic}/projects/prod-app-a-0.yaml (93%) rename fast/stages/2-security/{data => datasets/classic}/certificate-authorities/prod-ca-0.yaml (89%) rename fast/stages/2-security/{data => datasets/classic}/defaults.yaml (94%) rename fast/stages/2-security/{data => datasets/classic}/keyrings/dev-primary-default.yaml (92%) rename fast/stages/2-security/{data => datasets/classic}/keyrings/prod-primary-default.yaml (91%) rename fast/stages/2-security/{data => datasets/classic}/projects/dev-sec-core-0.yaml (95%) rename fast/stages/2-security/{data => datasets/classic}/projects/prod-sec-core-0.yaml (94%) create mode 100644 modules/net-vpc-factory/schemas/defaults.schema.json rename tests/fast/stages/s1_vpcsc/{data/vpc-sc => data-simple}/access-levels/geo_it.yaml (85%) rename tests/fast/stages/s1_vpcsc/{data/vpc-sc => data-simple}/access-levels/identity_me.yaml (86%) rename tests/fast/stages/s1_vpcsc/{data/vpc-sc => data-simple}/egress-policies/test.yaml (89%) rename tests/fast/stages/s1_vpcsc/{data/vpc-sc => data-simple}/ingress-policies/fast-org-log-sinks.yaml (88%) rename tests/fast/stages/s1_vpcsc/{data/vpc-sc => data-simple}/ingress-policies/test.yaml (89%) rename tests/fast/stages/s1_vpcsc/{data/vpc-sc => data-simple}/perimeters/default.yaml (90%) rename tests/fast/stages/s1_vpcsc/{data/vpc-sc => data-simple}/restricted-services.yaml (100%) create mode 100644 tests/fast/stages/s1_vpcsc/hardened.tfvars create mode 100644 tests/fast/stages/s1_vpcsc/hardened.yaml create mode 100755 tools/check_yaml_schema.py diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 54a3c8d23..e7cf8773a 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -90,6 +90,11 @@ jobs: run: | python3 tools/check_names.py --prefix-length=10 --failed-only fast/stages + - name: Check YAML schemas + id: yaml-schema + run: | + python3 tools/check_yaml_schema.py modules fast + - name: Check python formatting id: yapf uses: pre-commit/action@1b06ec171f2f6faa71ed760c4042bd969e4f8b43 diff --git a/fast/stages/0-org-setup/README.md b/fast/stages/0-org-setup/README.md index 75e629647..564f19857 100644 --- a/fast/stages/0-org-setup/README.md +++ b/fast/stages/0-org-setup/README.md @@ -78,20 +78,33 @@ The admin principal is typically a group that includes the user running the firs ### Select/configure a factory dataset -The `factories_config` variable points to several paths containing the YAML configuration files used by this stage. The default variable configuration points to the legacy FAST compatible fileset in the `data` folder. +The `factories_config` variable configures the location for the dataset, and the individual factories within it. Its default values point to the classic FAST compatible fileset in the `datasets/classic` folder. If this configuration matches requirements, no changes are necessary at this stage. To select a different setup create a `tfvars` file and set paths to the desired data folder, like shown in the example below. The different configurations produced by each fileset are described [later in this document](#default-factory-datasets). +This is how you would use a different dataset. + ```bash # create a file named 0-org-setup.auto.tfvars containing the following # and replace paths by pointing them to the desired data folder factories_config = { - billing_accounts = "datasets/classic/billing-accounts" - cicd = "datasets/classic/cicd.yaml" - defaults = "datasets/classic/defaults.yaml" - folders = "datasets/classic/folders" - organization = "datasets/classic/organization" - projects = "datasets/classic/projects" + dataset = "datasets/hardened" +} +``` + +While changing the paths used by the inner factories, or disabling some of them by pointing to a non-existent path, is done via the `paths` attribute (which can of course be combined with `dataset`). + +```bash +factories_config = { + # optional + # dataset = "datasets/hardened" + # optional + paths = { + # absolute paths ignores the dataset base path + billing_accounts = "/mydata/foo/billing-accounts" + # and if the path does not exist this specific factory is skipped + cicd = "/tmp/fake/cicd.yaml" + } } ``` @@ -386,30 +399,30 @@ This is a simple reference table of available interpolation namespaces, refer to The resources created by this stage are controlled by several factories, which point to YAML configuration files and folders. Data locations for each factory are controlled via the `var.factories_config` variable, and each factory path can be overridden individually. -The default paths point to the dataset in the `data` folder which deploys a FAST-compliant configuration. These are the available factories in this stage, with file-level factories based on a single YAML file, and folder-level factories based on sets of YAML files contained within a filesystem folder: +The default paths point to the dataset in the `datasets/classic` folder which deploys a FAST-compliant configuration. These are the available factories in this stage, with file-level factories based on a single YAML file, and folder-level factories based on sets of YAML files contained within a filesystem folder: -- **defaults** (`datasets/classic/defaults.yaml`) \ +- **defaults** (`[dataset]/defaults.yaml`) \ file-level factory to define stage defaults (organization id, locations, prefix, etc.) and static context mappings -- **billing_accounts** (`datasets/classic/billing-accounts`) \ +- **billing_accounts** (`[dataset]/billing-accounts`) \ folder-level factory where each YAML file defines billing-account level IAM for one billing account; only used for externally managed accounts -- **organization** (`datasets/classic/organization/.config.yaml`) \ +- **organization** (`[dataset]/organization/.config.yaml`) \ file-level factory to define organization IAM and log sinks - - **custom roles** (`datasets/classic/organization/custom-roles`) \ + - **custom roles** (`[dataset]/organization/custom-roles`) \ folder-level factory to define organization-level custom roles - - **org policies** (`datasets/classic/organization/org-policies`) \ + - **org policies** (`[dataset]/organization/org-policies`) \ folder-level factory to define organization-level org policies - - **tags** (`datasets/classic/organization/tags`) \ + - **tags** (`[dataset]/organization/tags`) \ folder-level factory to define organization-level resource management tags -- **folders** (`datasets/classic/folders`) \ +- **folders** (`[dataset]/folders`) \ folder-level factory to define the resource management hierarchy and individual folder attributes (IAM, org policies, tag bindings, etc.); also supports defining folder-level IaC resources -- **projects** (`datasets/classic/projects`) \ +- **projects** (`[dataset]/projects`) \ folder-level factory to define projects and their attributes (projejct factory) -- **cicd** (`datasets/classic/cicd-workflows.yaml`) \ +- **cicd** (`[dataset]/cicd-workflows.yaml`) \ file-level factory to define CI/CD configurations for this and subsequent stages ### Defaults configuration -The prerequisite configuration for this stage is done via a `defaults.yaml` file, which implements part or all of the [relevant JSON schema](./schemas/defaults.schema.json). The location of the file defaults to `datasets/classic/defaults.yaml` but can be easily changed via the `factories_config.defaults` variable. +The prerequisite configuration for this stage is done via a `defaults.yaml` file, which implements part or all of the [relevant JSON schema](./schemas/defaults.schema.json). The location of the file defaults to `[dataset]/defaults.yaml` but can be easily changed via the `factories_config.paths.defaults` variable. This is a commented example of a defaults file, showing a minimal working configuration. Refer to the YAML schema for all available options. @@ -663,7 +676,7 @@ The provided project configurations also create several key resources for the st CI/CD support is implemented via two different sets of configurations: - [Workload Identity](https://docs.cloud.google.com/iam/docs/workload-identity-federation) providers are defined in project configurations -- CI/CD service accounts and templated workflow generation are defined in a dedicated configuration (`var.factories_config.cicd_workflows`). +- CI/CD service accounts and templated workflow generation are defined in a dedicated configuration (`var.factories_config.paths.cicd_workflows`). The default approach is to define a Workload Identity provider in the `iac-0` project, or in an additional project dedicated to this task. This is achieved by adding a `workload_identity_pools` block to the project configuration, like in the following example. @@ -873,8 +886,8 @@ Define values for the `var.environments` variable in a tfvars file. | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [context](variables.tf#L17) | Context-specific interpolations. | object({…}) | | {} | -| [factories_config](variables.tf#L40) | Configuration for the resource factories or external data. | object({…}) | | {} | -| [org_policies_imports](variables.tf#L56) | List of org policies to import. These need to also be defined in data files. | list(string) | | [] | +| [factories_config](variables.tf#L40) | Configuration for the resource factories or external data. | object({…}) | | {} | +| [org_policies_imports](variables.tf#L59) | List of org policies to import. These need to also be defined in data files. | list(string) | | [] | ## Outputs diff --git a/fast/stages/0-org-setup/billing.tf b/fast/stages/0-org-setup/billing.tf index 1e098327f..c3eeb3a00 100644 --- a/fast/stages/0-org-setup/billing.tf +++ b/fast/stages/0-org-setup/billing.tf @@ -15,14 +15,11 @@ */ locals { - _billing_accounts_path = try( - pathexpand(var.factories_config.billing_accounts), null - ) _billing_accounts_raw = { - for f in try(fileset(local._billing_accounts_path, "*.yaml"), []) : + for f in try(fileset(local.paths.billing_accounts, "*.yaml"), []) : trimsuffix(f, ".yaml") => merge( { id = null }, - yamldecode(file("${local._billing_accounts_path}/${f}")) + yamldecode(file("${local.paths.billing_accounts}/${f}")) ) } billing_accounts = { diff --git a/fast/stages/0-org-setup/datasets/classic-gcd/folders/security/dev/.config.yaml b/fast/stages/0-org-setup/datasets/classic-gcd/folders/security/dev/.config.yaml index 630b4245a..1b8b08d6d 100644 --- a/fast/stages/0-org-setup/datasets/classic-gcd/folders/security/dev/.config.yaml +++ b/fast/stages/0-org-setup/datasets/classic-gcd/folders/security/dev/.config.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../../../0-org-setup/schemas/folder.schema.json +# yaml-language-server: $schema=../../../../../schemas/folder.schema.json name: Development parent: $folder_ids:security diff --git a/fast/stages/0-org-setup/datasets/classic-gcd/folders/security/prod/.config.yaml b/fast/stages/0-org-setup/datasets/classic-gcd/folders/security/prod/.config.yaml index 3dc1b6a76..c5bfe6258 100644 --- a/fast/stages/0-org-setup/datasets/classic-gcd/folders/security/prod/.config.yaml +++ b/fast/stages/0-org-setup/datasets/classic-gcd/folders/security/prod/.config.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../../../0-org-setup/schemas/folder.schema.json +# yaml-language-server: $schema=../../../../../schemas/folder.schema.json name: Production parent: $folder_ids:security diff --git a/fast/stages/0-org-setup/datasets/classic/folders/security/dev/.config.yaml b/fast/stages/0-org-setup/datasets/classic/folders/security/dev/.config.yaml index 630b4245a..1b8b08d6d 100644 --- a/fast/stages/0-org-setup/datasets/classic/folders/security/dev/.config.yaml +++ b/fast/stages/0-org-setup/datasets/classic/folders/security/dev/.config.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../../../0-org-setup/schemas/folder.schema.json +# yaml-language-server: $schema=../../../../../schemas/folder.schema.json name: Development parent: $folder_ids:security diff --git a/fast/stages/0-org-setup/datasets/classic/folders/security/prod/.config.yaml b/fast/stages/0-org-setup/datasets/classic/folders/security/prod/.config.yaml index 3dc1b6a76..c5bfe6258 100644 --- a/fast/stages/0-org-setup/datasets/classic/folders/security/prod/.config.yaml +++ b/fast/stages/0-org-setup/datasets/classic/folders/security/prod/.config.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../../../0-org-setup/schemas/folder.schema.json +# yaml-language-server: $schema=../../../../../schemas/folder.schema.json name: Production parent: $folder_ids:security diff --git a/fast/stages/0-org-setup/datasets/classic/observability/iac-0/impersonation.yaml b/fast/stages/0-org-setup/datasets/classic/observability/iac-0/impersonation.yaml index 632be9ef2..a02f7ffad 100644 --- a/fast/stages/0-org-setup/datasets/classic/observability/iac-0/impersonation.yaml +++ b/fast/stages/0-org-setup/datasets/classic/observability/iac-0/impersonation.yaml @@ -54,10 +54,10 @@ alerts: trigger: count: 1 aggregations: - alignment_period: 60s - per_series_aligner: ALIGN_COUNT - cross_series_reducer: REDUCE_SUM - group_by_fields: ["metric.label.email_id"] + - alignment_period: 60s + per_series_aligner: ALIGN_COUNT + cross_series_reducer: REDUCE_SUM + group_by_fields: ["metric.label.email_id"] notification_channels: - email-security enabled: true diff --git a/fast/stages/0-org-setup/datasets/classic/projects/core/iac-0.yaml b/fast/stages/0-org-setup/datasets/classic/projects/core/iac-0.yaml index cb1ba257e..f648827e0 100644 --- a/fast/stages/0-org-setup/datasets/classic/projects/core/iac-0.yaml +++ b/fast/stages/0-org-setup/datasets/classic/projects/core/iac-0.yaml @@ -84,7 +84,7 @@ org_policies: - https://gitlab.com - https://app.terraform.io factories_config: - observability: datasets/classic/observability/iac-0 + observability: observability/iac-0 data_access_logs: storage.googleapis.com: DATA_READ: {} diff --git a/fast/stages/0-org-setup/datasets/hardened/billing-accounts/default.yaml b/fast/stages/0-org-setup/datasets/hardened/billing-accounts/default.yaml new file mode 100644 index 000000000..64b9f1c84 --- /dev/null +++ b/fast/stages/0-org-setup/datasets/hardened/billing-accounts/default.yaml @@ -0,0 +1,46 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../../schemas/billing-account.schema.json + +id: $defaults:billing_account +# authoritative IAM must be used with care here, as it resets permissions +# three modes of managing billing IAM +# - at the org level (don't set anything here, set roles in the org) +# - at the billing account level (set the roles below) +# - no admin access to billing account (don't set roles here or in org) +iam_bindings_additive: + billing_admin_org_admins: + role: roles/billing.admin + member: $iam_principals:gcp-organization-admins + billing_admin_org_sa: + role: roles/billing.admin + member: $iam_principals:service_accounts/iac-0/iac-org-rw + billing_viewer_org_ro: + role: roles/billing.viewer + member: $iam_principals:service_accounts/iac-0/iac-org-ro + billing_user_security_sa: + role: roles/billing.user + member: $iam_principals:service_accounts/iac-0/iac-security-rw + billing_user_networking_sa: + role: roles/billing.user + member: $iam_principals:service_accounts/iac-0/iac-networking-rw + billing_user_pf_sa: + role: roles/billing.user + member: $iam_principals:service_accounts/iac-0/iac-pf-rw +# logging_sinks: +# test: +# description: Test sink +# destination: $project_ids:log-0 +# type: project diff --git a/fast/stages/0-org-setup/datasets/hardened/folders/security/dev/.config.yaml b/fast/stages/0-org-setup/datasets/hardened/folders/security/dev/.config.yaml index 42690a165..23775d521 100644 --- a/fast/stages/0-org-setup/datasets/hardened/folders/security/dev/.config.yaml +++ b/fast/stages/0-org-setup/datasets/hardened/folders/security/dev/.config.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../../../0-org-setup/schemas/folder.schema.json +# yaml-language-server: $schema=../../../../../schemas/folder.schema.json name: Development parent: $folder_ids:security diff --git a/fast/stages/0-org-setup/datasets/hardened/folders/security/prod/.config.yaml b/fast/stages/0-org-setup/datasets/hardened/folders/security/prod/.config.yaml index 3dc1b6a76..c5bfe6258 100644 --- a/fast/stages/0-org-setup/datasets/hardened/folders/security/prod/.config.yaml +++ b/fast/stages/0-org-setup/datasets/hardened/folders/security/prod/.config.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../../../0-org-setup/schemas/folder.schema.json +# yaml-language-server: $schema=../../../../../schemas/folder.schema.json name: Production parent: $folder_ids:security diff --git a/fast/stages/0-org-setup/datasets/hardened/observability/auditConfigChanges.yaml b/fast/stages/0-org-setup/datasets/hardened/observability/auditConfigChanges.yaml index 2b46eba13..476b6a818 100644 --- a/fast/stages/0-org-setup/datasets/hardened/observability/auditConfigChanges.yaml +++ b/fast/stages/0-org-setup/datasets/hardened/observability/auditConfigChanges.yaml @@ -50,7 +50,7 @@ alerts: mime_type: text/markdown logging_metrics: auditConfigChanges: - bucket_name: log-0/audit-logs + bucket_name: $log_buckets:log-0/audit-logs description: Audit Configuration Changes filter: protoPayload.methodName="SetIamPolicy" AND protoPayload.serviceData.policyDelta.auditConfigDeltas:* label_extractors: diff --git a/fast/stages/0-org-setup/datasets/hardened/observability/binaryAuthorizationPolicyChanges.yaml b/fast/stages/0-org-setup/datasets/hardened/observability/binaryAuthorizationPolicyChanges.yaml index 699f92c5b..eef9f0b63 100644 --- a/fast/stages/0-org-setup/datasets/hardened/observability/binaryAuthorizationPolicyChanges.yaml +++ b/fast/stages/0-org-setup/datasets/hardened/observability/binaryAuthorizationPolicyChanges.yaml @@ -51,7 +51,7 @@ alerts: mime_type: text/markdown logging_metrics: binaryAuthorizationPolicyChanges: - bucket_name: log-0/audit-logs + bucket_name: $log_buckets:log-0/audit-logs description: Binary Authorization Policy Changes filter: protoPayload.methodName="google.cloud.binaryauthorization.v1.BinauthzManagementServiceV1.UpdatePolicy" AND diff --git a/fast/stages/0-org-setup/datasets/hardened/observability/cloudsqlInstanceChanges.yaml b/fast/stages/0-org-setup/datasets/hardened/observability/cloudsqlInstanceChanges.yaml index 5d5145e98..61c2d4016 100644 --- a/fast/stages/0-org-setup/datasets/hardened/observability/cloudsqlInstanceChanges.yaml +++ b/fast/stages/0-org-setup/datasets/hardened/observability/cloudsqlInstanceChanges.yaml @@ -48,7 +48,7 @@ alerts: mime_type: text/markdown logging_metrics: cloudsqlInstanceChanges: - bucket_name: log-0/audit-logs + bucket_name: $log_buckets:log-0/audit-logs description: Cloud SQL Instance Configuration Changes filter: protoPayload.methodName="cloudsql.instances.update" label_extractors: diff --git a/fast/stages/0-org-setup/datasets/hardened/observability/customRoleChanges.yaml b/fast/stages/0-org-setup/datasets/hardened/observability/customRoleChanges.yaml index 6692dcdf3..4b4fac5c8 100644 --- a/fast/stages/0-org-setup/datasets/hardened/observability/customRoleChanges.yaml +++ b/fast/stages/0-org-setup/datasets/hardened/observability/customRoleChanges.yaml @@ -53,7 +53,7 @@ alerts: mime_type: text/markdown logging_metrics: customRoleChanges: - bucket_name: log-0/audit-logs + bucket_name: $log_buckets:log-0/audit-logs description: Custom Role Changes filter: |- resource.type="iam_role" AND ( diff --git a/fast/stages/0-org-setup/datasets/hardened/observability/firewallPolicyRuleChanges.yaml b/fast/stages/0-org-setup/datasets/hardened/observability/firewallPolicyRuleChanges.yaml index 2317f6db9..91842df48 100644 --- a/fast/stages/0-org-setup/datasets/hardened/observability/firewallPolicyRuleChanges.yaml +++ b/fast/stages/0-org-setup/datasets/hardened/observability/firewallPolicyRuleChanges.yaml @@ -50,7 +50,7 @@ alerts: mime_type: text/markdown logging_metrics: firewallPolicyRuleChanges: - bucket_name: log-0/audit-logs + bucket_name: $log_buckets:log-0/audit-logs description: Network Firewall Policy Rule Changes filter: |- resource.labels.method:"compute.networkFirewallPolicies" AND ( diff --git a/fast/stages/0-org-setup/datasets/hardened/observability/firewallRuleChanges.yaml b/fast/stages/0-org-setup/datasets/hardened/observability/firewallRuleChanges.yaml index 340573e4a..1e48d9717 100644 --- a/fast/stages/0-org-setup/datasets/hardened/observability/firewallRuleChanges.yaml +++ b/fast/stages/0-org-setup/datasets/hardened/observability/firewallRuleChanges.yaml @@ -51,7 +51,7 @@ alerts: mime_type: text/markdown logging_metrics: firewallRuleChanges: - bucket_name: log-0/audit-logs + bucket_name: $log_buckets:log-0/audit-logs description: VPC Network Firewall Rule Changes filter: resource.type="gce_firewall_rule" AND (protoPayload.methodName:"compute.firewalls.patch" OR diff --git a/fast/stages/0-org-setup/datasets/hardened/observability/networkChanges.yaml b/fast/stages/0-org-setup/datasets/hardened/observability/networkChanges.yaml index 03fffedaa..30fba3dd6 100644 --- a/fast/stages/0-org-setup/datasets/hardened/observability/networkChanges.yaml +++ b/fast/stages/0-org-setup/datasets/hardened/observability/networkChanges.yaml @@ -55,7 +55,7 @@ alerts: mime_type: text/markdown logging_metrics: networkChanges: - bucket_name: log-0/audit-logs + bucket_name: $log_buckets:log-0/audit-logs description: VPC Network Changes filter: |- resource.type="gce_network" AND ( diff --git a/fast/stages/0-org-setup/datasets/hardened/observability/networkRouteChanges.yaml b/fast/stages/0-org-setup/datasets/hardened/observability/networkRouteChanges.yaml index 5b1696ed7..e69337f60 100644 --- a/fast/stages/0-org-setup/datasets/hardened/observability/networkRouteChanges.yaml +++ b/fast/stages/0-org-setup/datasets/hardened/observability/networkRouteChanges.yaml @@ -50,7 +50,7 @@ alerts: mime_type: text/markdown logging_metrics: networkRouteChanges: - bucket_name: log-0/audit-logs + bucket_name: $log_buckets:log-0/audit-logs description: VPC Network Route Changes filter: |- resource.type="gce_route" AND ( diff --git a/fast/stages/0-org-setup/datasets/hardened/observability/projectOwnershipChange.yaml b/fast/stages/0-org-setup/datasets/hardened/observability/projectOwnershipChange.yaml index 8c500f653..7797695a1 100644 --- a/fast/stages/0-org-setup/datasets/hardened/observability/projectOwnershipChange.yaml +++ b/fast/stages/0-org-setup/datasets/hardened/observability/projectOwnershipChange.yaml @@ -58,7 +58,7 @@ alerts: mime_type: text/markdown logging_metrics: projectOwnershipChange: - bucket_name: log-0/audit-logs + bucket_name: $log_buckets:log-0/audit-logs description: Project Ownership Changes filter: |- (protoPayload.serviceName="cloudresourcemanager.googleapis.com") AND diff --git a/fast/stages/0-org-setup/datasets/hardened/observability/storageIamChanges.yaml b/fast/stages/0-org-setup/datasets/hardened/observability/storageIamChanges.yaml index f98acba62..81794bc65 100644 --- a/fast/stages/0-org-setup/datasets/hardened/observability/storageIamChanges.yaml +++ b/fast/stages/0-org-setup/datasets/hardened/observability/storageIamChanges.yaml @@ -48,7 +48,7 @@ alerts: mime_type: text/markdown logging_metrics: storageIamChanges: - bucket_name: log-0/audit-logs + bucket_name: l$log_buckets:log-0/audit-logs description: Cloud Storage IAM Permission Changes filter: resource.type="gcs_bucket" AND protoPayload.methodName="storage.setIamPermissions" label_extractors: diff --git a/fast/stages/0-org-setup/datasets/hardened/projects/core/log-0.yaml b/fast/stages/0-org-setup/datasets/hardened/projects/core/log-0.yaml index 43e9cba56..2c59e2623 100644 --- a/fast/stages/0-org-setup/datasets/hardened/projects/core/log-0.yaml +++ b/fast/stages/0-org-setup/datasets/hardened/projects/core/log-0.yaml @@ -65,7 +65,7 @@ log_buckets: enable: true retention: 90 factories_config: - observability: datasets/hardened/organization/observability/ + observability: organization/observability/ org_policies: gcp.restrictCmekCryptoKeyProjects: rules: diff --git a/fast/stages/0-org-setup/factory.tf b/fast/stages/0-org-setup/factory.tf index 25554c76f..4a6acf9cd 100644 --- a/fast/stages/0-org-setup/factory.tf +++ b/fast/stages/0-org-setup/factory.tf @@ -63,8 +63,8 @@ module "factory" { ) }) factories_config = { - folders = var.factories_config.folders - project_templates = var.factories_config.project_templates - projects = var.factories_config.projects + basepath = var.factories_config.dataset + budgets = local.factory_billing + paths = var.factories_config.paths } } diff --git a/fast/stages/0-org-setup/main.tf b/fast/stages/0-org-setup/main.tf index ee51ab4fc..129c2f6e1 100644 --- a/fast/stages/0-org-setup/main.tf +++ b/fast/stages/0-org-setup/main.tf @@ -15,9 +15,6 @@ */ locals { - paths = { - for k, v in var.factories_config : k => try(pathexpand(v), null) - } _ctx = { for k, v in var.context : k => merge( v, @@ -53,6 +50,13 @@ locals { storage_bucket = try(local._defaults.output_files.storage_bucket, null) providers = try(local._defaults.output_files.providers, {}) } + paths = { + for k, v in var.factories_config.paths : k => try(pathexpand( + startswith(v, "/") || startswith(v, ".") + ? v : + "${var.factories_config.dataset}/${v}" + ), null) + } project_defaults = { defaults = try(local._defaults.projects.defaults, {}) overrides = try(local._defaults.projects.overrides, {}) diff --git a/fast/stages/0-org-setup/observability.tf b/fast/stages/0-org-setup/observability.tf index 460c2cb9f..3d2fca0ff 100644 --- a/fast/stages/0-org-setup/observability.tf +++ b/fast/stages/0-org-setup/observability.tf @@ -58,7 +58,7 @@ module "projects-observability" { project_ids = module.factory.project_ids }) factories_config = { - observability = var.factories_config.observability + observability = local.paths.observability } } diff --git a/fast/stages/0-org-setup/schemas/observability.schema.json b/fast/stages/0-org-setup/schemas/observability.schema.json index cf3eb2f0a..b33bb72b3 100644 --- a/fast/stages/0-org-setup/schemas/observability.schema.json +++ b/fast/stages/0-org-setup/schemas/observability.schema.json @@ -478,23 +478,26 @@ ] }, "aggregations": { - "type": "object", - "additionalProperties": false, - "properties": { - "per_series_aligner": { - "type": "string" - }, - "group_by_fields": { - "type": "array", - "items": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "per_series_aligner": { + "type": "string" + }, + "group_by_fields": { + "type": "array", + "items": { + "type": "string" + } + }, + "cross_series_reducer": { + "type": "string" + }, + "alignment_period": { "type": "string" } - }, - "cross_series_reducer": { - "type": "string" - }, - "alignment_period": { - "type": "string" } } }, @@ -511,4 +514,4 @@ } } } -} +} \ No newline at end of file diff --git a/fast/stages/0-org-setup/schemas/project.schema.json b/fast/stages/0-org-setup/schemas/project.schema.json index 4881ece7c..8d437b134 100644 --- a/fast/stages/0-org-setup/schemas/project.schema.json +++ b/fast/stages/0-org-setup/schemas/project.schema.json @@ -425,7 +425,7 @@ "type": "object", "additionalProperties": false, "patternProperties": { - "^[a-z0-9-]+$": { + "^[a-zA-Z0-9_-]+$": { "$ref": "#/$defs/log_bucket" } } diff --git a/fast/stages/0-org-setup/variables.tf b/fast/stages/0-org-setup/variables.tf index fcb679900..facd77314 100644 --- a/fast/stages/0-org-setup/variables.tf +++ b/fast/stages/0-org-setup/variables.tf @@ -40,14 +40,17 @@ variable "context" { variable "factories_config" { description = "Configuration for the resource factories or external data." type = object({ - billing_accounts = optional(string, "datasets/classic/billing-accounts") - cicd_workflows = optional(string) - defaults = optional(string, "datasets/classic/defaults.yaml") - folders = optional(string, "datasets/classic/folders") - observability = optional(string, "datasets/classic/observability") - organization = optional(string, "datasets/classic/organization") - project_templates = optional(string, "datasets/classic/templates") - projects = optional(string, "datasets/classic/projects") + dataset = optional(string, "datasets/classic") + paths = optional(object({ + billing_accounts = optional(string, "billing-accounts") + cicd_workflows = optional(string) + defaults = optional(string, "defaults.yaml") + folders = optional(string, "folders") + observability = optional(string, "observability") + organization = optional(string, "organization") + project_templates = optional(string, "templates") + projects = optional(string, "projects") + }), {}) }) nullable = false default = {} diff --git a/fast/stages/1-vpcsc/README.md b/fast/stages/1-vpcsc/README.md index c599bc7c6..5895ffc71 100644 --- a/fast/stages/1-vpcsc/README.md +++ b/fast/stages/1-vpcsc/README.md @@ -53,7 +53,7 @@ Restricted services, access levels, ingress and egress policies and perimeters c The default setup only contains a single access level and an initial list of restricted services in the `datasets/classic/access-levels` folder and the `datasets/classic/restricted-services.yaml` file. -To configure ingress and egress policies simply add `ingress_policies` and/or `egress_policies` folders under `data`, or point the factories to your own folders by changing the `factories_config` variable values. +To configure ingress and egress policies simply add `ingress_policies` and/or `egress_policies` folders under `data`, or point the factories to your own folders by changing the `factories_config` variable values. For more details on how the `factories_config` interface works, refer to the [documentation for stage 0](../0-org-setup/README.md#selectconfigure-a-factory-dataset). Examples on how to write access level and policy YAML files are provided further down in this document. @@ -100,6 +100,7 @@ projects: ``` And then apply `0-org-setup` stage again. For later stages (such as networking, project factory), add the perimeter in a similar way, but there you can use context to provide perimeter: + ```yaml projects: overrides: @@ -109,6 +110,7 @@ projects: > [!CAUTION] > Do not add any resources to the perimeter in this stage when using the second approach. Any resources added in this stage will not be properly removed from perimeter, if the `terraform apply` is also changing the perimeter definition. +> #### Resource discovery If the first approach is desired in combination with resource discovery, you can simply tweak exclusions via the `resource_discovery` variable as the feature is enabled by default. @@ -459,13 +461,13 @@ Some references that might be useful in setting up this stage: | [access_policy](variables.tf#L67) | Access policy id (used for tenant-level VPC-SC configurations). | number | | null | | | [context](variables.tf#L73) | External context used in replacements. | object({…}) | | {} | | | [egress_policies](variables.tf#L87) | Egress policy definitions that can be referenced in perimeters. | map(object({…})) | | {} | | -| [factories_config](variables.tf#L130) | Paths to folders that enable factory functionality. | object({…}) | | {} | | +| [factories_config](variables.tf#L130) | Paths to folders that enable factory functionality. | object({…}) | | {} | | | [iam_principals](variables-fast.tf#L17) | Org-level IAM principals. | map(string) | | {} | 0-org-setup | -| [ingress_policies](variables.tf#L144) | Ingress policy definitions that can be referenced in perimeters. | map(object({…})) | | {} | | +| [ingress_policies](variables.tf#L147) | Ingress policy definitions that can be referenced in perimeters. | map(object({…})) | | {} | | | [logging](variables-fast.tf#L25) | Log writer identities for organization / folders. | object({…}) | | null | 0-org-setup | -| [perimeters](variables.tf#L186) | Perimeter definitions. | map(object({…})) | | {} | | +| [perimeters](variables.tf#L189) | Perimeter definitions. | map(object({…})) | | {} | | | [project_numbers](variables-fast.tf#L46) | Project numbers. | map(number) | | {} | 0-org-setup | -| [resource_discovery](variables.tf#L220) | Automatic discovery of perimeter projects. | object({…}) | | {} | | +| [resource_discovery](variables.tf#L223) | Automatic discovery of perimeter projects. | object({…}) | | {} | | | [root_node](variables-fast.tf#L54) | Root node for the hierarchy, if running in tenant mode. | string | | null | 0-org-setup | | [service_accounts](variables-fast.tf#L68) | Org-level service accounts. | map(string) | | {} | 0-org-setup | | [storage_buckets](variables-fast.tf#L76) | Storage buckets created in the bootstrap stage. | map(string) | | {} | 0-org-setup | diff --git a/fast/stages/1-vpcsc/main.tf b/fast/stages/1-vpcsc/main.tf index a1a885bc9..0c9969e34 100644 --- a/fast/stages/1-vpcsc/main.tf +++ b/fast/stages/1-vpcsc/main.tf @@ -15,9 +15,6 @@ */ locals { - paths = { - for k, v in var.factories_config : k => try(pathexpand(v), null) - } _ctx = { for k, v in var.context : k => merge( v, try(local._defaults.context[k], {}) @@ -78,6 +75,13 @@ locals { null ) } + paths = { + for k, v in var.factories_config.paths : k => try(pathexpand( + startswith(v, "/") || startswith(v, ".") + ? v : + "${var.factories_config.dataset}/${v}" + ), null) + } } module "vpc-sc-discovery" { @@ -100,7 +104,7 @@ module "vpc-sc" { access_levels = var.access_levels egress_policies = var.egress_policies context = local.ctx - factories_config = var.factories_config + factories_config = local.paths ingress_policies = var.ingress_policies perimeters = var.perimeters project_id_search_scope = "organizations/${var.organization.id}" diff --git a/fast/stages/1-vpcsc/variables.tf b/fast/stages/1-vpcsc/variables.tf index b634ad51e..877d058b9 100644 --- a/fast/stages/1-vpcsc/variables.tf +++ b/fast/stages/1-vpcsc/variables.tf @@ -130,12 +130,15 @@ variable "egress_policies" { variable "factories_config" { description = "Paths to folders that enable factory functionality." type = object({ - access_levels = optional(string, "datasets/classic/access-levels") - defaults = optional(string, "datasets/classic/defaults.yaml") - egress_policies = optional(string, "datasets/classic/egress-policies") - ingress_policies = optional(string, "datasets/classic/ingress-policies") - perimeters = optional(string, "datasets/classic/perimeters") - restricted_services = optional(string, "datasets/classic/restricted-services.yaml") + dataset = optional(string, "datasets/classic") + paths = optional(object({ + access_levels = optional(string, "access-levels") + defaults = optional(string, "defaults.yaml") + egress_policies = optional(string, "egress-policies") + ingress_policies = optional(string, "ingress-policies") + perimeters = optional(string, "perimeters") + restricted_services = optional(string, "restricted-services.yaml") + }), {}) }) nullable = false default = {} diff --git a/fast/stages/2-networking/README.md b/fast/stages/2-networking/README.md index c738ce36b..2e1fda2c6 100644 --- a/fast/stages/2-networking/README.md +++ b/fast/stages/2-networking/README.md @@ -318,7 +318,7 @@ Internally created resources are mapped to context namespaces, and use specific | [prefix](variables-fast.tf#L75) | Prefix used for resources that need unique names. Use a maximum of 9 chars for organizations, and 11 chars for tenants. | string | ✓ | | | [context](variables.tf#L17) | Context-specific interpolations. | object({…}) | | {} | | [custom_roles](variables-fast.tf#L25) | Custom roles defined at the org level, in key => id format. | map(string) | | {} | -| [factories_config](variables.tf#L37) | Configuration for the resource factories or external data. | object({…}) | | {} | +| [factories_config](variables.tf#L37) | Configuration for the resource factories or external data. | object({…}) | | {} | | [folder_ids](variables-fast.tf#L33) | Folders created in the bootstrap stage. | map(string) | | {} | | [iam_principals](variables-fast.tf#L41) | IAM-format principals. | map(string) | | {} | | [kms_keys](variables-fast.tf#L50) | KMS key ids. | map(string) | | {} | diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-ncc/vpcs/dev/firewall-rules/default-ingress.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-ncc/vpcs/dev/firewall-rules/default-ingress.yaml index 77cbf3327..0d2a658c6 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-ncc/vpcs/dev/firewall-rules/default-ingress.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-ncc/vpcs/dev/firewall-rules/default-ingress.yaml @@ -2,7 +2,7 @@ --- # start of document (---) avoids errors if the file only contains comments -# yaml-language-server: $schema=../../../schemas/firewall-rules.schema.json +# yaml-language-server: $schema=../../../../../schemas/firewall-rules.schema.json ingress: ingress-default-dev-deny: diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-ncc/vpcs/hub/vpns/onprem.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-ncc/vpcs/hub/vpns/onprem.yaml index d85cd25b9..36c555f1a 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-ncc/vpcs/hub/vpns/onprem.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-ncc/vpcs/hub/vpns/onprem.yaml @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# yaml-language-server: $schema=../../../../../schemas/vpn.schema.json + name: to-onprem region: $locations:primary peer_gateways: diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-ncc/vpcs/prod/firewall-rules/default-ingress.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-ncc/vpcs/prod/firewall-rules/default-ingress.yaml index cd99ce114..8cf7a3f5a 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-ncc/vpcs/prod/firewall-rules/default-ingress.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-ncc/vpcs/prod/firewall-rules/default-ingress.yaml @@ -2,7 +2,7 @@ --- # start of document (---) avoids errors if the file only contains comments -# yaml-language-server: $schema=../../../schemas/firewall-rules.schema.json +# yaml-language-server: $schema=../../../../../schemas/firewall-rules.schema.json ingress: ingress-default-prod-deny: diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/fwd-root.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/fwd-root.yaml index 6db3e36b8..effa5ee78 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/fwd-root.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/fwd-root.yaml @@ -2,7 +2,7 @@ --- # start of document (---) avoids errors if the file only contains comments -# yaml-language-server: $schema=../../../../../schemas/dns.schema.json +# yaml-language-server: $schema=../../../../schemas/dns.schema.json project_id: $project_ids:net-core-0 domain: onprem. diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/peer-root.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/peer-root.yaml index 383e91a0a..bf3961126 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/peer-root.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/peer-root.yaml @@ -2,7 +2,7 @@ --- # start of document (---) avoids errors if the file only contains comments -# yaml-language-server: $schema=../../../../../schemas/dns.schema.json +# yaml-language-server: $schema=../../../../schemas/dns.schema.json project_id: $project_ids:net-core-0 domain: . diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/pvt-dev-test.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/pvt-dev-test.yaml index b190c6f55..4db64ef93 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/pvt-dev-test.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/pvt-dev-test.yaml @@ -2,7 +2,7 @@ --- # start of document (---) avoids errors if the file only contains comments -# yaml-language-server: $schema=../../../../../schemas/dns.schema.json +# yaml-language-server: $schema=../../../../schemas/dns.schema.json project_id: $project_ids:net-core-0 domain: dev.test. diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/pvt-prod-test.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/pvt-prod-test.yaml index 1a2514931..f7668e053 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/pvt-prod-test.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/pvt-prod-test.yaml @@ -2,7 +2,7 @@ --- # start of document (---) avoids errors if the file only contains comments -# yaml-language-server: $schema=../../../../../schemas/dns.schema.json +# yaml-language-server: $schema=../../../../schemas/dns.schema.json project_id: $project_ids:net-core-0 domain: prod.test. diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/pvt-test.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/pvt-test.yaml index 0369f0c42..4d1938284 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/pvt-test.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-nva/dns/zones/pvt-test.yaml @@ -2,7 +2,7 @@ --- # start of document (---) avoids errors if the file only contains comments -# yaml-language-server: $schema=../../../../../schemas/dns.schema.json +# yaml-language-server: $schema=../../../../schemas/dns.schema.json project_id: $project_ids:net-core-0 domain: test. diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-nva/vpcs/dev/firewall-rules/default-ingress.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-nva/vpcs/dev/firewall-rules/default-ingress.yaml index 77cbf3327..0d2a658c6 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-nva/vpcs/dev/firewall-rules/default-ingress.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-nva/vpcs/dev/firewall-rules/default-ingress.yaml @@ -2,7 +2,7 @@ --- # start of document (---) avoids errors if the file only contains comments -# yaml-language-server: $schema=../../../schemas/firewall-rules.schema.json +# yaml-language-server: $schema=../../../../../schemas/firewall-rules.schema.json ingress: ingress-default-dev-deny: diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-nva/vpcs/dmz/firewall-rules/default-ingress.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-nva/vpcs/dmz/firewall-rules/default-ingress.yaml index 209e9d0c3..e8f1bc1da 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-nva/vpcs/dmz/firewall-rules/default-ingress.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-nva/vpcs/dmz/firewall-rules/default-ingress.yaml @@ -2,7 +2,7 @@ --- # start of document (---) avoids errors if the file only contains comments -# yaml-language-server: $schema=../../../schemas/firewall-rules.schema.json +# yaml-language-server: $schema=../../../../../schemas/firewall-rules.schema.json ingress: ingress-default-dmz-deny: diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-nva/vpcs/hub/firewall-rules/default-ingress.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-nva/vpcs/hub/firewall-rules/default-ingress.yaml index 6b24210d5..2050960bd 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-nva/vpcs/hub/firewall-rules/default-ingress.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-nva/vpcs/hub/firewall-rules/default-ingress.yaml @@ -2,7 +2,7 @@ --- # start of document (---) avoids errors if the file only contains comments -# yaml-language-server: $schema=../../../schemas/firewall-rules.schema.json +# yaml-language-server: $schema=../../../../../schemas/firewall-rules.schema.json ingress: ingress-default-landing-deny: diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-nva/vpcs/prod/firewall-rules/default-ingress.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-nva/vpcs/prod/firewall-rules/default-ingress.yaml index cd99ce114..8cf7a3f5a 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-nva/vpcs/prod/firewall-rules/default-ingress.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-nva/vpcs/prod/firewall-rules/default-ingress.yaml @@ -2,7 +2,7 @@ --- # start of document (---) avoids errors if the file only contains comments -# yaml-language-server: $schema=../../../schemas/firewall-rules.schema.json +# yaml-language-server: $schema=../../../../../schemas/firewall-rules.schema.json ingress: ingress-default-prod-deny: diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/dev/firewall-rules/default-ingress.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/dev/firewall-rules/default-ingress.yaml index 77cbf3327..0d2a658c6 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/dev/firewall-rules/default-ingress.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/dev/firewall-rules/default-ingress.yaml @@ -2,7 +2,7 @@ --- # start of document (---) avoids errors if the file only contains comments -# yaml-language-server: $schema=../../../schemas/firewall-rules.schema.json +# yaml-language-server: $schema=../../../../../schemas/firewall-rules.schema.json ingress: ingress-default-dev-deny: diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/dev/vpns/to-hub.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/dev/vpns/to-hub.yaml index 104c273d3..b21d8bc2a 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/dev/vpns/to-hub.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/dev/vpns/to-hub.yaml @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# yaml-language-server: $schema=../../../../../schemas/vpn.schema.json + name: to-hub region: $locations:primary peer_gateways: diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/hub/firewall-rules/default-ingress.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/hub/firewall-rules/default-ingress.yaml index 6b24210d5..2050960bd 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/hub/firewall-rules/default-ingress.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/hub/firewall-rules/default-ingress.yaml @@ -2,7 +2,7 @@ --- # start of document (---) avoids errors if the file only contains comments -# yaml-language-server: $schema=../../../schemas/firewall-rules.schema.json +# yaml-language-server: $schema=../../../../../schemas/firewall-rules.schema.json ingress: ingress-default-landing-deny: diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/hub/vpns/onprem.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/hub/vpns/onprem.yaml index 5d70ca435..89f224395 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/hub/vpns/onprem.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/hub/vpns/onprem.yaml @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# yaml-language-server: $schema=../../../../../schemas/vpn.schema.json + name: to-onprem region: $locations:primary peer_gateways: diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/hub/vpns/to-dev.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/hub/vpns/to-dev.yaml index 8c5143b7b..eb3cd17eb 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/hub/vpns/to-dev.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/hub/vpns/to-dev.yaml @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# yaml-language-server: $schema=../../../../../schemas/vpn.schema.json + name: to-dev region: $locations:primary peer_gateways: diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/hub/vpns/to-prod.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/hub/vpns/to-prod.yaml index 2d7a3e79d..78ffd89b8 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/hub/vpns/to-prod.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/hub/vpns/to-prod.yaml @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# yaml-language-server: $schema=../../../../../schemas/vpn.schema.json + name: to-prod region: $locations:primary peer_gateways: diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/prod/firewall-rules/default-ingress.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/prod/firewall-rules/default-ingress.yaml index cd99ce114..8cf7a3f5a 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/prod/firewall-rules/default-ingress.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/prod/firewall-rules/default-ingress.yaml @@ -2,7 +2,7 @@ --- # start of document (---) avoids errors if the file only contains comments -# yaml-language-server: $schema=../../../schemas/firewall-rules.schema.json +# yaml-language-server: $schema=../../../../../schemas/firewall-rules.schema.json ingress: ingress-default-prod-deny: diff --git a/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/prod/vpns/to-hub.yaml b/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/prod/vpns/to-hub.yaml index a9500eda7..3e35b9623 100644 --- a/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/prod/vpns/to-hub.yaml +++ b/fast/stages/2-networking/datasets/hub-and-spokes-vpns/vpcs/prod/vpns/to-hub.yaml @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# yaml-language-server: $schema=../../../../../schemas/vpn.schema.json + name: to-hub region: $locations:primary peer_gateways: diff --git a/fast/stages/2-networking/factory-dns.tf b/fast/stages/2-networking/factory-dns.tf index 754dcc29f..373fa7b98 100644 --- a/fast/stages/2-networking/factory-dns.tf +++ b/fast/stages/2-networking/factory-dns.tf @@ -17,10 +17,9 @@ # tfdoc:file:description DNS zones and RPZ factory. locals { - _dns_path = try(pathexpand(var.factories_config.dns), null) - _dns_files = try(fileset(local._dns_path, "**/*.yaml"), []) + _dns_files = try(fileset(local.paths.dns, "**/*.yaml"), []) _dns_preprocess = [ - for f in local._dns_files : merge(yamldecode(file("${coalesce(local._dns_path, "-")}/${f}")), { + for f in local._dns_files : merge(yamldecode(file("${coalesce(local.paths.dns, "-")}/${f}")), { key = replace(f, ".yaml", "") }) ] @@ -77,16 +76,13 @@ locals { ) } # DNS response policies - _dns_response_policies_path = try( - pathexpand(var.factories_config.dns-response-policies), null - ) _dns_response_policies_files = try( - fileset(local._dns_response_policies_path, "**/*.yaml"), [] + fileset(local.paths.dns_response_policies, "**/*.yaml"), [] ) _dns_response_policies_preprocess = [ for f in local._dns_response_policies_files : merge( - yamldecode(file("${coalesce(local._dns_response_policies_path, "-")}/${f}")), + yamldecode(file("${coalesce(local.paths.dns_response_policies, "-")}/${f}")), { key = replace(f, ".yaml", "") } diff --git a/fast/stages/2-networking/factory-firewall-policies.tf b/fast/stages/2-networking/factory-firewall-policies.tf index be047a126..e35e2e2e4 100644 --- a/fast/stages/2-networking/factory-firewall-policies.tf +++ b/fast/stages/2-networking/factory-firewall-policies.tf @@ -17,15 +17,12 @@ # tfdoc:file:description Firewall policies factory. locals { - _firewall_policies_path = try( - pathexpand(var.factories_config.firewall-policies), null - ) - _firewall_policies_files = local._firewall_policies_path == null ? [] : fileset( - local._firewall_policies_path, "**/*.yaml" + _firewall_policies_files = local.paths.firewall_policies == null ? [] : fileset( + local.paths.firewall_policies, "**/*.yaml" ) _firewall_policies_data = { for f in local._firewall_policies_files : replace(f, ".yaml", "") => yamldecode( - file("${local._firewall_policies_path}/${f}") + file("${local.paths.firewall_policies}/${f}") ) } firewall_policies = { diff --git a/fast/stages/2-networking/factory-ncc.tf b/fast/stages/2-networking/factory-ncc.tf index 1b469fa6e..2e4830f21 100644 --- a/fast/stages/2-networking/factory-ncc.tf +++ b/fast/stages/2-networking/factory-ncc.tf @@ -17,11 +17,10 @@ # tfdoc:file:description NCC Hubs and Groups factory locals { - _ncc_path = try(pathexpand(var.factories_config.ncc-hubs), null) - _ncc_files = try(fileset(local._ncc_path, "**/*.yaml"), []) + _ncc_files = try(fileset(local.paths.ncc_hubs, "**/*.yaml"), []) _ncc_preprocess = [ for f in local._ncc_files : yamldecode( - file("${coalesce(local._ncc_path, "-")}/${f}") + file("${coalesce(local.paths.ncc_hubs, "-")}/${f}") ) ] # Since NCC groups depend on NCC hubs, two different lookup maps avoid circular dependencies. diff --git a/fast/stages/2-networking/factory-nva.tf b/fast/stages/2-networking/factory-nva.tf index 33ce59dcf..169989ad4 100644 --- a/fast/stages/2-networking/factory-nva.tf +++ b/fast/stages/2-networking/factory-nva.tf @@ -17,11 +17,10 @@ # tfdoc:file:description NVA factory locals { - _nva_path = try(pathexpand(var.factories_config.nvas), null) - _nva_files = try(fileset(local._nva_path, "**/*.yaml"), []) + _nva_files = try(fileset(local.paths.nvas, "**/*.yaml"), []) _nva_configs = [ for f in local._nva_files : merge( - yamldecode(file("${coalesce(local._nva_path, "-")}/${f}")), + yamldecode(file("${coalesce(local.paths.nvas, "-")}/${f}")), { filename = replace(f, ".yaml", "") } ) ] diff --git a/fast/stages/2-networking/factory-projects.tf b/fast/stages/2-networking/factory-projects.tf index d88ecefc5..7711e6369 100644 --- a/fast/stages/2-networking/factory-projects.tf +++ b/fast/stages/2-networking/factory-projects.tf @@ -41,7 +41,10 @@ module "projects" { ) context = local.ctx factories_config = { - folders = var.factories_config.folders - projects = var.factories_config.projects + basepath = var.factories_config.dataset + budgets = { + billing_account = var.billing_account.id + } + paths = var.factories_config.paths } } diff --git a/fast/stages/2-networking/factory-vpcs.tf b/fast/stages/2-networking/factory-vpcs.tf index b51481fc0..fc320727d 100644 --- a/fast/stages/2-networking/factory-vpcs.tf +++ b/fast/stages/2-networking/factory-vpcs.tf @@ -17,19 +17,16 @@ # tfdoc:file:description VPC and firewall rules factory. locals { - _vpcs_path = try( - pathexpand(var.factories_config.vpcs), null - ) _vpcs_files = try( - fileset(local._vpcs_path, "**/.config.yaml"), + fileset(local.paths.vpcs, "**/.config.yaml"), [] ) _vpcs_preprocess = [ for f in local._vpcs_files : merge( - yamldecode(file("${coalesce(local._vpcs_path, "-")}/${f}")), + yamldecode(file("${coalesce(local.paths.vpcs, "-")}/${f}")), { factory_dirname = dirname(f) - factory_basepath = "${local._vpcs_path}/${dirname(f)}" + factory_basepath = "${local.paths.vpcs}/${dirname(f)}" } ) ] @@ -93,7 +90,7 @@ moved { module "vpc-factory" { source = "../../../modules/net-vpc-factory" - factories_config = var.factories_config + factories_config = local.paths context = { project_ids = local.ctx_projects.project_ids locations = local.ctx.locations diff --git a/fast/stages/2-networking/factory-vpns.tf b/fast/stages/2-networking/factory-vpns.tf index 1d5702216..eccb4d7f4 100644 --- a/fast/stages/2-networking/factory-vpns.tf +++ b/fast/stages/2-networking/factory-vpns.tf @@ -18,12 +18,12 @@ locals { _vpns_files = try( - fileset(local._vpcs_path, "**/vpns/*.yaml"), + fileset(local.paths.vpcs, "**/vpns/*.yaml"), [] ) _vpns_preprocess = [ for f in local._vpns_files : merge( - yamldecode(file("${coalesce(local._vpcs_path, "-")}/${f}")), + yamldecode(file("${coalesce(local.paths.vpcs, "-")}/${f}")), { factory_basepath = dirname(dirname(f)) } diff --git a/fast/stages/2-networking/main.tf b/fast/stages/2-networking/main.tf index 2b31f1f4d..776691af1 100644 --- a/fast/stages/2-networking/main.tf +++ b/fast/stages/2-networking/main.tf @@ -14,9 +14,6 @@ * limitations under the License. */ locals { - paths = { - for k, v in var.factories_config : k => try(pathexpand(v), null) - } _ctx = { for k, v in var.context : k => merge( v, try(local._defaults.context[k], {}) @@ -59,6 +56,13 @@ locals { null ) } + paths = { + for k, v in var.factories_config.paths : k => try(pathexpand( + startswith(v, "/") || startswith(v, ".") + ? v : + "${var.factories_config.dataset}/${v}" + ), null) + } project_defaults = { defaults = merge( { diff --git a/fast/stages/2-networking/schemas/project.schema.json b/fast/stages/2-networking/schemas/project.schema.json index 4881ece7c..8d437b134 100644 --- a/fast/stages/2-networking/schemas/project.schema.json +++ b/fast/stages/2-networking/schemas/project.schema.json @@ -425,7 +425,7 @@ "type": "object", "additionalProperties": false, "patternProperties": { - "^[a-z0-9-]+$": { + "^[a-zA-Z0-9_-]+$": { "$ref": "#/$defs/log_bucket" } } diff --git a/fast/stages/2-networking/variables.tf b/fast/stages/2-networking/variables.tf index 82afda8c1..6e95dc209 100644 --- a/fast/stages/2-networking/variables.tf +++ b/fast/stages/2-networking/variables.tf @@ -37,15 +37,18 @@ variable "context" { variable "factories_config" { description = "Configuration for the resource factories or external data." type = object({ - defaults = optional(string, "datasets/hub-and-spokes-peerings/defaults.yaml") - dns = optional(string, "datasets/hub-and-spokes-peerings/dns/zones") - dns-response-policies = optional(string, "datasets/hub-and-spokes-peerings/dns/response-policies") - firewall-policies = optional(string, "datasets/hub-and-spokes-peerings/firewall-policies") - folders = optional(string, "datasets/hub-and-spokes-peerings/folders") - ncc-hubs = optional(string, "datasets/hub-and-spokes-peerings/ncc-hubs") - nvas = optional(string, "datasets/hub-and-spokes-peerings/nvas") - projects = optional(string, "datasets/hub-and-spokes-peerings/projects") - vpcs = optional(string, "datasets/hub-and-spokes-peerings/vpcs") + dataset = optional(string, "datasets/hub-and-spokes-peerings") + paths = optional(object({ + defaults = optional(string, "defaults.yaml") + dns = optional(string, "dns/zones") + dns_response_policies = optional(string, "dns/response-policies") + firewall_policies = optional(string, "firewall-policies") + folders = optional(string, "folders") + ncc_hubs = optional(string, "ncc-hubs") + nvas = optional(string, "nvas") + projects = optional(string, "projects") + vpcs = optional(string, "vpcs") + }), {}) }) nullable = false default = {} diff --git a/fast/stages/2-project-factory/README.md b/fast/stages/2-project-factory/README.md index e0dcd5b0a..11c14c32d 100644 --- a/fast/stages/2-project-factory/README.md +++ b/fast/stages/2-project-factory/README.md @@ -220,7 +220,7 @@ An example defaults file is provided in the `data` folder, and the relevant sche ### Folder and hierarchy management -The project factory manages its folder hierarchy via a filesystem tree, rooted in the path defined via the `factories_config.folders` variable. +The project factory manages its folder hierarchy via a filesystem tree, rooted in the path defined via the `factories_config.paths.folders` variable. Filesystem folders which contain a `.config.yaml` file are mapped to folders in the resource management hierarchy. Their YAML configuration files allow defining folder attributes like descriptive name, IAM bindings, organization policies, tag bindings. @@ -288,7 +288,7 @@ parent: $folder_ids:teams Project YAML files can be created in two different filesystem paths: -- in the filesystem folder defined via the `factories_config.projects` variable, and then explicitly setting their `parent` attribute in YAML files, or +- in the filesystem folder defined via the `factories_config.path.projects` variable, and then explicitly setting their `parent` attribute in YAML files, or - in the filesystem hierarchy discussed above, so that their `parent` attribute is automatically derived from the containing folder The two approaches can be mixed and matched, but the first approach is safer as is avoids potentially dangerous situations when folders are deleted with project configuration files still inside. @@ -487,7 +487,7 @@ Pattern-based files make specific assumptions: | [data_defaults](variables-projects.tf#L17) | Optional default values used when corresponding project or folder data from files are missing. | object({…}) | | {} | | | [data_merges](variables-projects.tf#L93) | Optional values that will be merged with corresponding data from files. Combines with `data_defaults`, file data, and `data_overrides`. | object({…}) | | {} | | | [data_overrides](variables-projects.tf#L112) | Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`. | object({…}) | | {} | | -| [factories_config](variables.tf#L37) | Path to folder with YAML resource description data files. | object({…}) | | {} | | +| [factories_config](variables.tf#L37) | Path to folder with YAML resource description data files. | object({…}) | | {} | | | [folder_ids](variables-fast.tf#L42) | Folders created in the bootstrap stage. | map(string) | | {} | 0-org-setup | | [host_project_ids](variables-fast.tf#L58) | Host project for the shared VPC. | map(string) | | {} | 2-networking | | [iam_principals](variables-fast.tf#L50) | IAM-format principals. | map(string) | | {} | 0-org-setup | diff --git a/fast/stages/2-project-factory/data/defaults.yaml b/fast/stages/2-project-factory/datasets/classic/defaults.yaml similarity index 94% rename from fast/stages/2-project-factory/data/defaults.yaml rename to fast/stages/2-project-factory/datasets/classic/defaults.yaml index f99600e62..6f2e29399 100644 --- a/fast/stages/2-project-factory/data/defaults.yaml +++ b/fast/stages/2-project-factory/datasets/classic/defaults.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../schemas/defaults.schema.json +# yaml-language-server: $schema=../../schemas/defaults.schema.json # environment-specific project defaults, merges and overrides can be defined here diff --git a/fast/stages/2-project-factory/data/folders/team-a/.config.yaml b/fast/stages/2-project-factory/datasets/classic/folders/team-a/.config.yaml similarity index 90% rename from fast/stages/2-project-factory/data/folders/team-a/.config.yaml rename to fast/stages/2-project-factory/datasets/classic/folders/team-a/.config.yaml index c5207f058..9ae89a578 100644 --- a/fast/stages/2-project-factory/data/folders/team-a/.config.yaml +++ b/fast/stages/2-project-factory/datasets/classic/folders/team-a/.config.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../../schemas/folder.schema.json +# yaml-language-server: $schema=../../../../schemas/folder.schema.json name: Team A parent: $folder_ids:teams diff --git a/fast/stages/2-project-factory/data/folders/team-a/dev/.config.yaml b/fast/stages/2-project-factory/datasets/classic/folders/team-a/dev/.config.yaml similarity index 90% rename from fast/stages/2-project-factory/data/folders/team-a/dev/.config.yaml rename to fast/stages/2-project-factory/datasets/classic/folders/team-a/dev/.config.yaml index db8ba5325..60bf6e2ba 100644 --- a/fast/stages/2-project-factory/data/folders/team-a/dev/.config.yaml +++ b/fast/stages/2-project-factory/datasets/classic/folders/team-a/dev/.config.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../../../schemas/folder.schema.json +# yaml-language-server: $schema=../../../../../schemas/folder.schema.json name: Development tag_bindings: diff --git a/fast/stages/2-project-factory/data/folders/team-a/prod/.config.yaml b/fast/stages/2-project-factory/datasets/classic/folders/team-a/prod/.config.yaml similarity index 89% rename from fast/stages/2-project-factory/data/folders/team-a/prod/.config.yaml rename to fast/stages/2-project-factory/datasets/classic/folders/team-a/prod/.config.yaml index 4b0b318d4..79ad373e6 100644 --- a/fast/stages/2-project-factory/data/folders/team-a/prod/.config.yaml +++ b/fast/stages/2-project-factory/datasets/classic/folders/team-a/prod/.config.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../../../schemas/folder.schema.json +# yaml-language-server: $schema=../../../../../schemas/folder.schema.json name: Production tag_bindings: diff --git a/fast/stages/2-project-factory/data/projects/dev-app-a-0.yaml b/fast/stages/2-project-factory/datasets/classic/projects/dev-app-a-0.yaml similarity index 92% rename from fast/stages/2-project-factory/data/projects/dev-app-a-0.yaml rename to fast/stages/2-project-factory/datasets/classic/projects/dev-app-a-0.yaml index 45a5abae5..e0327cab8 100644 --- a/fast/stages/2-project-factory/data/projects/dev-app-a-0.yaml +++ b/fast/stages/2-project-factory/datasets/classic/projects/dev-app-a-0.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../schemas/project.schema.json +# yaml-language-server: $schema=../../../schemas/project.schema.json parent: $folder_ids:team-a/dev services: diff --git a/fast/stages/2-project-factory/data/projects/prod-app-a-0.yaml b/fast/stages/2-project-factory/datasets/classic/projects/prod-app-a-0.yaml similarity index 93% rename from fast/stages/2-project-factory/data/projects/prod-app-a-0.yaml rename to fast/stages/2-project-factory/datasets/classic/projects/prod-app-a-0.yaml index 493adba8f..461dacb6a 100644 --- a/fast/stages/2-project-factory/data/projects/prod-app-a-0.yaml +++ b/fast/stages/2-project-factory/datasets/classic/projects/prod-app-a-0.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../schemas/project.schema.json +# yaml-language-server: $schema=../../../schemas/project.schema.json parent: $folder_ids:team-a/prod services: diff --git a/fast/stages/2-project-factory/main.tf b/fast/stages/2-project-factory/main.tf index 780bd9b99..ba7f4e765 100644 --- a/fast/stages/2-project-factory/main.tf +++ b/fast/stages/2-project-factory/main.tf @@ -23,7 +23,7 @@ locals { context = merge(local._context, { vpc_sc_perimeters = merge(var.perimeters, local._context.vpc_sc_perimeters) }) - defaults = yamldecode(file(pathexpand(var.factories_config.defaults))) + defaults = yamldecode(file(pathexpand(local.paths.defaults))) fast_defaults = { billing_account = try(coalesce( var.data_defaults.billing_account, @@ -33,6 +33,13 @@ locals { var.data_defaults.prefix, var.prefix ) } + paths = { + for k, v in var.factories_config.paths : k => try(pathexpand( + startswith(v, "/") || startswith(v, ".") + ? v : + "${var.factories_config.dataset}/${v}" + ), null) + } project_defaults = { defaults = { for k, v in var.data_defaults : k => try( @@ -101,14 +108,11 @@ module "factory" { data_defaults = local.project_defaults.defaults data_merges = local.project_defaults.merges data_overrides = local.project_defaults.overrides - factories_config = merge(var.factories_config, { + factories_config = { + basepath = var.factories_config.dataset budgets = { - billing_account_id = try( - var.factories_config.budgets.billing_account_id, var.billing_account.id - ) - data = try( - var.factories_config.budgets.data, "data/budgets" - ) + billing_account = var.billing_account.id } - }) + paths = var.factories_config.paths + } } diff --git a/fast/stages/2-project-factory/schemas/project.schema.json b/fast/stages/2-project-factory/schemas/project.schema.json index 4881ece7c..8d437b134 100644 --- a/fast/stages/2-project-factory/schemas/project.schema.json +++ b/fast/stages/2-project-factory/schemas/project.schema.json @@ -425,7 +425,7 @@ "type": "object", "additionalProperties": false, "patternProperties": { - "^[a-z0-9-]+$": { + "^[a-zA-Z0-9_-]+$": { "$ref": "#/$defs/log_bucket" } } diff --git a/fast/stages/2-project-factory/variables.tf b/fast/stages/2-project-factory/variables.tf index c4aab0cb5..7edac58b9 100644 --- a/fast/stages/2-project-factory/variables.tf +++ b/fast/stages/2-project-factory/variables.tf @@ -37,13 +37,13 @@ variable "context" { variable "factories_config" { description = "Path to folder with YAML resource description data files." type = object({ - defaults = optional(string, "data/defaults.yaml") - folders = optional(string, "data/folders") - projects = optional(string, "data/projects") - budgets = optional(object({ - billing_account_id = string - data = string - })) + dataset = optional(string, "datasets/classic") + paths = optional(object({ + defaults = optional(string, "defaults.yaml") + folders = optional(string, "folders") + projects = optional(string, "projects") + budgets = optional(string) + }), {}) }) nullable = false default = {} diff --git a/fast/stages/2-security/README.md b/fast/stages/2-security/README.md index d340828eb..4526bfb54 100644 --- a/fast/stages/2-security/README.md +++ b/fast/stages/2-security/README.md @@ -43,17 +43,19 @@ The default dataset provides two projects, each rooted in a separate folder. I d If a different folder or project configuration is needed copy the full dataset to a different path to avoid accidental changes from upstream, then add or remove files in `data/folders` and `data/projects`. -A simple KMS keyrin is provided in `keyrings` folder, use it as an example if more are needed. For CAS too a sample configuration is already present, but the Certificate Authorities factory is disabled by default. If CAS is needed define the `factories_config.certificate_authorities` variable attribute in your tfvars and set it to the path of the relevant data folder (`data/certificate-authorities` by default) as shown in this snippet. +A simple KMS keyrin is provided in `keyrings` folder, use it as an example if more are needed. For CAS too a sample configuration is already present, but the Certificate Authorities factory is disabled by default. If CAS is needed define the `factories_config.paths.certificate_authorities` variable attribute in your tfvars and set it to the path of the relevant data folder (`data/certificate-authorities` by default) as shown in this snippet. ```hcl factories_config = { - certificate_authorities = "data/certificate-authorities" + paths = { + certificate_authorities = "certificate-authorities" + } } ``` ### Defaults file -Configurations defaults are stored in the `defaults.yaml` file in the selected dataset. Relocating the defaults file is good practice to avoid accindetal changes from upstream, this is done via the `factories_config.default` variable attribute. +Configurations defaults are stored in the `defaults.yaml` file in the selected dataset. Relocating the defaults file is good practice to avoid accindetal changes from upstream, this is done via the `factories_config.paths.default` variable attribute. Once a suitable place has been found for the file, edit it to match the desired configuration. Several pieces of information coming from the previous stage (prefix, billing account, etc.) are pre-populated in the project defaults so they don't need to be explicitly set. If some of them need to be overridden, the attributes in `projects.overrides` take precedence as shown in this annotated sample. @@ -81,11 +83,15 @@ A tfvars file allows you to control paths for the project factories data, and to ```hcl factories_config = { - certificate_authorities = "/some/path/data/certificate-authorities" - defaults = "/some/path/data/defaults.yaml" - folders = "/some/path/data/folders" - keyrings = "/some/path/data/keyrings" - projects = "/some/path/data/projects" + dataset = "datasets/mydataset" + paths = { + certificate_authorities = "certificate-authorities" + # the following default to relative paths in the dataset + # defaults = "defaults.yaml" + # folders = "folders" + # keyrings = "keyrings" + # projects = "projects" + } } outputs_location = "~/fast-config" ``` @@ -185,7 +191,7 @@ A reference Certificate Authority Services (CAS) is also part of this stage, all | [prefix](variables-fast.tf#L57) | Prefix used for resources that need unique names. Use a maximum of 9 chars for organizations, and 11 chars for tenants. | string | ✓ | | 0-org-setup | | [context](variables.tf#L17) | Context-specific interpolations. | object({…}) | | {} | | | [custom_roles](variables-fast.tf#L25) | Custom roles defined at the org level, in key => id format. | map(string) | | {} | 0-org-setup | -| [factories_config](variables.tf#L36) | Configuration for the resource factories or external data. | object({…}) | | {} | | +| [factories_config](variables.tf#L36) | Configuration for the resource factories or external data. | object({…}) | | {} | | | [folder_ids](variables-fast.tf#L33) | Folders created in the bootstrap stage. | map(string) | | {} | 0-org-setup | | [iam_principals](variables-fast.tf#L41) | IAM-format principals. | map(string) | | {} | 0-org-setup | | [perimeters](variables-fast.tf#L49) | Optional VPC-SC perimeter ids. | map(string) | | {} | 1-vpcsc | diff --git a/fast/stages/2-security/data/certificate-authorities/prod-ca-0.yaml b/fast/stages/2-security/datasets/classic/certificate-authorities/prod-ca-0.yaml similarity index 89% rename from fast/stages/2-security/data/certificate-authorities/prod-ca-0.yaml rename to fast/stages/2-security/datasets/classic/certificate-authorities/prod-ca-0.yaml index 724bd8f13..13401e526 100644 --- a/fast/stages/2-security/data/certificate-authorities/prod-ca-0.yaml +++ b/fast/stages/2-security/datasets/classic/certificate-authorities/prod-ca-0.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../schemas/certificate-authority.schema.json +# yaml-language-server: $schema=../../../schemas/certificate-authority.schema.json location: $locations:primary project_id: $project_ids:prod-sec-core-0 diff --git a/fast/stages/2-security/data/defaults.yaml b/fast/stages/2-security/datasets/classic/defaults.yaml similarity index 94% rename from fast/stages/2-security/data/defaults.yaml rename to fast/stages/2-security/datasets/classic/defaults.yaml index 26cc067b2..afa19a36f 100644 --- a/fast/stages/2-security/data/defaults.yaml +++ b/fast/stages/2-security/datasets/classic/defaults.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../schemas/defaults.schema.json +# yaml-language-server: $schema=../../schemas/defaults.schema.json context: locations: diff --git a/fast/stages/2-security/data/keyrings/dev-primary-default.yaml b/fast/stages/2-security/datasets/classic/keyrings/dev-primary-default.yaml similarity index 92% rename from fast/stages/2-security/data/keyrings/dev-primary-default.yaml rename to fast/stages/2-security/datasets/classic/keyrings/dev-primary-default.yaml index 0626fb51e..7ca7761d2 100644 --- a/fast/stages/2-security/data/keyrings/dev-primary-default.yaml +++ b/fast/stages/2-security/datasets/classic/keyrings/dev-primary-default.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../schemas/keyring.schema.json +# yaml-language-server: $schema=../../../schemas/keyring.schema.json location: $locations:primary project_id: $project_ids:dev-sec-core-0 diff --git a/fast/stages/2-security/data/keyrings/prod-primary-default.yaml b/fast/stages/2-security/datasets/classic/keyrings/prod-primary-default.yaml similarity index 91% rename from fast/stages/2-security/data/keyrings/prod-primary-default.yaml rename to fast/stages/2-security/datasets/classic/keyrings/prod-primary-default.yaml index fbcfd5c21..c605e07f6 100644 --- a/fast/stages/2-security/data/keyrings/prod-primary-default.yaml +++ b/fast/stages/2-security/datasets/classic/keyrings/prod-primary-default.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../schemas/keyring.schema.json +# yaml-language-server: $schema=../../../schemas/keyring.schema.json location: $locations:primary project_id: $project_ids:prod-sec-core-0 diff --git a/fast/stages/2-security/data/projects/dev-sec-core-0.yaml b/fast/stages/2-security/datasets/classic/projects/dev-sec-core-0.yaml similarity index 95% rename from fast/stages/2-security/data/projects/dev-sec-core-0.yaml rename to fast/stages/2-security/datasets/classic/projects/dev-sec-core-0.yaml index 4fa0a53c4..82a1d77b1 100644 --- a/fast/stages/2-security/data/projects/dev-sec-core-0.yaml +++ b/fast/stages/2-security/datasets/classic/projects/dev-sec-core-0.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../../0-org-setup/schemas/project.schema.json +# yaml-language-server: $schema=../../../schemas/project.schema.json parent: $folder_ids:security/dev services: diff --git a/fast/stages/2-security/data/projects/prod-sec-core-0.yaml b/fast/stages/2-security/datasets/classic/projects/prod-sec-core-0.yaml similarity index 94% rename from fast/stages/2-security/data/projects/prod-sec-core-0.yaml rename to fast/stages/2-security/datasets/classic/projects/prod-sec-core-0.yaml index e9a9ddd5b..e187e29a9 100644 --- a/fast/stages/2-security/data/projects/prod-sec-core-0.yaml +++ b/fast/stages/2-security/datasets/classic/projects/prod-sec-core-0.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../../0-org-setup/schemas/project.schema.json +# yaml-language-server: $schema=../../../schemas/project.schema.json parent: $folder_ids:security/prod services: diff --git a/fast/stages/2-security/factory-cas.tf b/fast/stages/2-security/factory-cas.tf index 1bb032abe..a238898ee 100644 --- a/fast/stages/2-security/factory-cas.tf +++ b/fast/stages/2-security/factory-cas.tf @@ -15,16 +15,13 @@ */ locals { - _ca_pools_path = try( - pathexpand(var.factories_config.certificate_authorities), null - ) _ca_pools_files = try( - fileset(local._ca_pools_path, "**/*.yaml"), + fileset(local.paths.certificate_authorities, "**/*.yaml"), [] ) _ca_pools = { for f in local._ca_pools_files : trimsuffix(basename(f), ".yaml") => yamldecode(file( - "${coalesce(local._ca_pools_path, "-")}/${f}" + "${coalesce(local.paths.certificate_authorities, "-")}/${f}" )) } ca_pools = { diff --git a/fast/stages/2-security/factory-keyrings.tf b/fast/stages/2-security/factory-keyrings.tf index 97b33569d..872f55064 100644 --- a/fast/stages/2-security/factory-keyrings.tf +++ b/fast/stages/2-security/factory-keyrings.tf @@ -15,16 +15,13 @@ */ locals { - _keyrings_path = try( - pathexpand(var.factories_config.keyrings), null - ) _keyrings_files = try( - fileset(local._keyrings_path, "**/*.yaml"), + fileset(local.paths.keyrings, "**/*.yaml"), [] ) _keyrings = { for f in local._keyrings_files : trimsuffix(basename(f), ".yaml") => yamldecode(file( - "${coalesce(local._keyrings_path, "-")}/${f}" + "${coalesce(local.paths.keyrings, "-")}/${f}" )) } keyrings = { diff --git a/fast/stages/2-security/factory-projects.tf b/fast/stages/2-security/factory-projects.tf index 20eef0fa3..68969ba67 100644 --- a/fast/stages/2-security/factory-projects.tf +++ b/fast/stages/2-security/factory-projects.tf @@ -25,7 +25,10 @@ module "factory" { ) context = local.ctx factories_config = { - folders = var.factories_config.folders - projects = var.factories_config.projects + basepath = var.factories_config.dataset + budgets = { + billing_account = var.billing_account.id + } + paths = var.factories_config.paths } } diff --git a/fast/stages/2-security/main.tf b/fast/stages/2-security/main.tf index 76196b9a2..267b1c72f 100644 --- a/fast/stages/2-security/main.tf +++ b/fast/stages/2-security/main.tf @@ -15,9 +15,6 @@ */ locals { - paths = { - for k, v in var.factories_config : k => try(pathexpand(v), null) - } _ctx = { for k, v in var.context : k => merge( v, try(local._defaults.context[k], {}) @@ -60,6 +57,13 @@ locals { null ) } + paths = { + for k, v in var.factories_config.paths : k => try(pathexpand( + startswith(v, "/") || startswith(v, ".") + ? v : + "${var.factories_config.dataset}/${v}" + ), null) + } project_defaults = { defaults = merge( { diff --git a/fast/stages/2-security/schemas/project.schema.json b/fast/stages/2-security/schemas/project.schema.json index 4881ece7c..8d437b134 100644 --- a/fast/stages/2-security/schemas/project.schema.json +++ b/fast/stages/2-security/schemas/project.schema.json @@ -425,7 +425,7 @@ "type": "object", "additionalProperties": false, "patternProperties": { - "^[a-z0-9-]+$": { + "^[a-zA-Z0-9_-]+$": { "$ref": "#/$defs/log_bucket" } } diff --git a/fast/stages/2-security/variables.tf b/fast/stages/2-security/variables.tf index d3e290109..7792dcbbf 100644 --- a/fast/stages/2-security/variables.tf +++ b/fast/stages/2-security/variables.tf @@ -36,11 +36,14 @@ variable "context" { variable "factories_config" { description = "Configuration for the resource factories or external data." type = object({ - certificate_authorities = optional(string) # "data/certificate-authorities" - defaults = optional(string, "data/defaults.yaml") - folders = optional(string, "data/folders") - keyrings = optional(string, "data/keyrings") - projects = optional(string, "data/projects") + dataset = optional(string, "datasets/classic") + paths = optional(object({ + certificate_authorities = optional(string) # "certificate-authorities" + defaults = optional(string, "defaults.yaml") + folders = optional(string, "folders") + keyrings = optional(string, "keyrings") + projects = optional(string, "projects") + }), {}) }) nullable = false default = {} diff --git a/modules/net-vpc-factory/data/defaults.yaml b/modules/net-vpc-factory/data/defaults.yaml index ec4d438a0..a7a1fd14e 100644 --- a/modules/net-vpc-factory/data/defaults.yaml +++ b/modules/net-vpc-factory/data/defaults.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../schemas/defaults.schema.json +# yaml-language-server: $schema=../schemas/defaults.schema.json context: cidr_ranges_sets: diff --git a/modules/net-vpc-factory/data/example/.config.yaml b/modules/net-vpc-factory/data/example/.config.yaml index 00db87f58..7189a7adc 100644 --- a/modules/net-vpc-factory/data/example/.config.yaml +++ b/modules/net-vpc-factory/data/example/.config.yaml @@ -2,12 +2,8 @@ --- # start of document (---) avoids errors if the file only contains comments -# yaml-language-server: $schema=../../../schemas/vpc-factory.schema.json +# yaml-language-server: $schema=../../schemas/vpc-factory.schema.json name: example project_id: $project_ids:net auto_create_subnetworks: false -subnets: - - name: example-default-primary - region: $locations:primary - ip_cidr_range: 172.16.0.0/24 diff --git a/modules/net-vpc-factory/data/example/firewall-rules/default-ingress.yaml b/modules/net-vpc-factory/data/example/firewall-rules/default-ingress.yaml index b06302ff0..cd99ce114 100644 --- a/modules/net-vpc-factory/data/example/firewall-rules/default-ingress.yaml +++ b/modules/net-vpc-factory/data/example/firewall-rules/default-ingress.yaml @@ -2,7 +2,7 @@ --- # start of document (---) avoids errors if the file only contains comments -# yaml-language-server: $schema=../../../../schemas/firewall-rules.schema.json +# yaml-language-server: $schema=../../../schemas/firewall-rules.schema.json ingress: ingress-default-prod-deny: diff --git a/modules/net-vpc-factory/data/example/subnets/example-default-secondary.yaml b/modules/net-vpc-factory/data/example/subnets/example-default-secondary.yaml index 485d34826..90772cd4a 100644 --- a/modules/net-vpc-factory/data/example/subnets/example-default-secondary.yaml +++ b/modules/net-vpc-factory/data/example/subnets/example-default-secondary.yaml @@ -1,6 +1,6 @@ # skip boilerplate check -# yaml-language-server: $schema=../../../../schemas/subnet.schema.json +# yaml-language-server: $schema=../../../schemas/subnet.schema.json name: example-default-secondary region: $locations:secondary diff --git a/modules/net-vpc-factory/schemas/defaults.schema.json b/modules/net-vpc-factory/schemas/defaults.schema.json new file mode 100644 index 000000000..8fabe8acc --- /dev/null +++ b/modules/net-vpc-factory/schemas/defaults.schema.json @@ -0,0 +1,59 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Net VPC Factory Defaults", + "type": "object", + "additionalProperties": false, + "properties": { + "context": { + "type": "object", + "additionalProperties": false, + "properties": { + "cidr_ranges_sets": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "iam_principals": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "locations": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "project_ids": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, + "vpcs": { + "type": "object", + "additionalProperties": false, + "properties": { + "auto_create_subnetworks": { + "type": "boolean", + "default": false + }, + "delete_default_route_on_create": { + "type": "boolean", + "default": true + }, + "mtu": { + "type": "number", + "default": 1500 + } + } + } + } +} diff --git a/modules/project-factory/README.md b/modules/project-factory/README.md index 591cf7dd9..10cfbc736 100644 --- a/modules/project-factory/README.md +++ b/modules/project-factory/README.md @@ -310,8 +310,8 @@ As an example, the id of the folder defined in `folders/networking/prod/.config. Project ids ise the `$project_ids:` namespace, with ids defined in two different ways: -- projects defined in the `var.factories_config.project` tree use the filename (dirname is stripped) -- projects defined in the `var.factories_config.folders` tree use the full path (dirname is kept) +- projects defined in the `var.factories_config.paths.project` tree use the filename (dirname is stripped) +- projects defined in the `var.factories_config.paths.folders` tree use the full path (dirname is kept) As an example, the id of the project defined in the `projects/team-0/app-0-0.yaml` file will be accessible via `$project_ids:app-0-0`. The id of the project defined in the `folders/shared/iac-core-0.yaml` file will be accessible via `$project_ids:shared/iac-core-0`. @@ -463,13 +463,10 @@ module "project-factory" { } # location where the yaml files are read from factories_config = { + basepath = "data" budgets = { - billing_account_id = var.billing_account_id - data = "data/budgets" + billing_account = var.billing_account_id } - folders = "data/hierarchy" - project_templates = "data/templates" - projects = "data/projects" } notification_channels = { billing-default = { @@ -498,7 +495,7 @@ service_encryption_key_ids: - $kms_keys:compute-prod-ew1 tag_bindings: context: $tag_values:context/gke -# tftest-file id=t0 path=data/templates/container/base.yaml schema=project.schema.json +# tftest-file id=t0 path=data/project-templates/container/base.yaml schema=project.schema.json ``` A simple hierarchy of folders: @@ -515,31 +512,31 @@ data_access_logs: DATA_READ: exempted_members: - $iam_principals:gcp-devops -# tftest-file id=0 path=data/hierarchy/team-a/.config.yaml schema=folder.schema.json +# tftest-file id=0 path=data/folders/team-a/.config.yaml schema=folder.schema.json ``` ```yaml name: Team B # explicit parent definition via key parent: $folder_ids:teams -# tftest-file id=1 path=data/hierarchy/team-b/.config.yaml schema=folder.schema.json +# tftest-file id=1 path=data/folders/team-b/.config.yaml schema=folder.schema.json ``` ```yaml name: Team C # explicit parent definition via folder id parent: folders/5678901234 -# tftest-file id=2 path=data/hierarchy/team-c/.config.yaml schema=folder.schema.json +# tftest-file id=2 path=data/folders/team-c/.config.yaml schema=folder.schema.json ``` ```yaml name: Apps -# tftest-file id=2.1 path=data/hierarchy/team-c/apps/.config.yaml schema=folder.schema.json +# tftest-file id=2.1 path=data/folders/team-c/apps/.config.yaml schema=folder.schema.json ``` ```yaml name: Test -# tftest-file id=2.2 path=data/hierarchy/team-c/apps/test/.config.yaml schema=folder.schema.json +# tftest-file id=2.2 path=data/folders/team-c/apps/test/.config.yaml schema=folder.schema.json ``` ```yaml @@ -553,7 +550,7 @@ asset_feeds: content_type: RESOURCE asset_types: - compute.googleapis.com/Instance -# tftest-file id=2.3 path=data/hierarchy/team-c/apps/test/app-x/.config.yaml schema=folder.schema.json +# tftest-file id=2.3 path=data/folders/team-c/apps/test/app-x/.config.yaml schema=folder.schema.json ``` ```yaml @@ -574,14 +571,14 @@ pam_entitlements: privileged_access: - role: roles/writer -# tftest-file id=3 path=data/hierarchy/team-a/app-0/.config.yaml schema=folder.schema.json +# tftest-file id=3 path=data/folders/team-a/app-0/.config.yaml schema=folder.schema.json ``` ```yaml name: App 0 tag_bindings: drs-allow-all: $tag_values:org-policies/drs-allow-all -# tftest-file id=4 path=data/hierarchy/team-b/app-0/.config.yaml schema=folder.schema.json +# tftest-file id=4 path=data/folders/team-b/app-0/.config.yaml schema=folder.schema.json ``` One project defined within the folder hierarchy, using a lower level factory for org policies: @@ -589,7 +586,7 @@ One project defined within the folder hierarchy, using a lower level factory for ```yaml billing_account: 012345-67890A-BCDEF0 factories_config: - org_policies: data/factories/org-policies + org_policies: factories/org-policies services: - container.googleapis.com - storage.googleapis.com @@ -610,7 +607,7 @@ workload_identity_pools: oidc: template: github -# tftest-file id=5 path=data/hierarchy/teams-iac-0.yaml schema=project.schema.json +# tftest-file id=5 path=data/folders/teams-iac-0.yaml schema=project.schema.json ``` More traditional project definitions via the project factory data: @@ -858,14 +855,14 @@ compute.disableSerialPortAccess: | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [factories_config](variables.tf#L165) | Path to folder with YAML resource description data files. | object({…}) | ✓ | | +| [factories_config](variables.tf#L165) | Path to folder with YAML resource description data files. | object({…}) | ✓ | | | [context](variables.tf#L17) | Context-specific interpolations. | object({…}) | | {} | | [data_defaults](variables.tf#L42) | Optional default values used when corresponding project or folder data from files are missing. | object({…}) | | {} | | [data_merges](variables.tf#L107) | Optional values that will be merged with corresponding data from files. Combines with `data_defaults`, file data, and `data_overrides`. | object({…}) | | {} | | [data_overrides](variables.tf#L126) | Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`. | object({…}) | | {} | | [folders](variables-folders.tf#L17) | Folders data merged with factory data. | map(object({…})) | | {} | | [notification_channels](variables-billing.tf#L17) | Notification channels used by budget alerts. | map(object({…})) | | {} | -| [projects](variables-projects.tf#L17) | Projects data merged with factory data. | map(object({…})) | | {} | +| [projects](variables-projects.tf#L17) | Projects data merged with factory data. | map(object({…})) | | {} | ## Outputs @@ -924,7 +921,7 @@ module "project-factory" { ] } factories_config = { - projects = "data/projects" + basepath = "data" } } # tftest modules=7 resources=31 files=test-0,test-1,test-2 inventory=test-1.yaml diff --git a/modules/project-factory/budgets.tf b/modules/project-factory/budgets.tf index f1648ab12..80d4e9923 100644 --- a/modules/project-factory/budgets.tf +++ b/modules/project-factory/budgets.tf @@ -17,6 +17,10 @@ # tfdoc:file:description Billing budget factory locals. locals { + budgets_enabled = ( + var.factories_config.budgets.billing_account != null && + local.paths.budgets != null + ) budget_folder_sets = flatten([ for k, v in local.folders_input : [ for vv in try(v.billing_budgets, []) : { @@ -37,8 +41,8 @@ locals { module "billing-budgets" { source = "../billing-account" - count = var.factories_config.budgets != null ? 1 : 0 - id = var.factories_config.budgets.billing_account_id + count = local.budgets_enabled ? 1 : 0 + id = var.factories_config.budgets.billing_account context = merge(local.ctx, { folder_ids = local.ctx.folder_ids folder_sets = { @@ -53,7 +57,7 @@ module "billing-budgets" { project_numbers = local.ctx_project_numbers }) factories_config = { - budgets_data_path = var.factories_config.budgets.data + budgets_data_path = local.paths.budgets } budget_notification_channels = ( var.notification_channels diff --git a/modules/project-factory/folders.tf b/modules/project-factory/folders.tf index bea5d77cb..8f2f804ea 100644 --- a/modules/project-factory/folders.tf +++ b/modules/project-factory/folders.tf @@ -19,18 +19,15 @@ # TODO: folder automation locals { - _folders_path = try( - pathexpand(var.factories_config.folders), null - ) _folders_files = try( - fileset(local._folders_path, "**/**/.config.yaml"), + fileset(local.paths.folders, "**/**/.config.yaml"), [] ) _folders_raw = merge( var.folders, { for f in local._folders_files : dirname(f) => yamldecode(file( - "${coalesce(local._folders_path, "-")}/${f}" + "${coalesce(local.paths.folders, "-")}/${f}" )) } ) diff --git a/modules/project-factory/main.tf b/modules/project-factory/main.tf index 8cf1b5999..a77f8e97a 100644 --- a/modules/project-factory/main.tf +++ b/modules/project-factory/main.tf @@ -26,6 +26,13 @@ locals { local.projects_sas_iam_emails, local.automation_sas_iam_emails ) + paths = { + for k, v in var.factories_config.paths : k => try(pathexpand( + var.factories_config.basepath == null || startswith(v, "/") || startswith(v, ".") + ? v : + "${var.factories_config.basepath}/${v}" + ), null) + } } resource "terraform_data" "defaults_preconditions" { @@ -38,8 +45,8 @@ resource "terraform_data" "defaults_preconditions" { error_message = "No default storage location defined in defaults or overrides variables." } # precondition { - # condition = local.projects_input == null - # error_message = jsonencode(local.ctx_tag_values) + # condition = local.paths == null + # error_message = jsonencode(local.paths) # } } } diff --git a/modules/project-factory/projects-defaults.tf b/modules/project-factory/projects-defaults.tf index 64a5ff7cd..07f1182f7 100644 --- a/modules/project-factory/projects-defaults.tf +++ b/modules/project-factory/projects-defaults.tf @@ -50,7 +50,16 @@ locals { try(v.contacts, null), local.data_defaults.defaults.contacts ) - factories_config = try(v.factories_config, {}) + factories_config = { + custom_roles = try(v.factories_config.custom_roles, null) + observability = try(v.factories_config.observability, null) + org_policies = try(v.factories_config.org_policies, null) + pam_entitlements = try(v.factories_config.pam_entitlements, null) + quotas = try(v.factories_config.quotas, null) + scc_mute_configs = try(v.factories_config.scc_mute_configs, null) + scc_sha_custom_modules = try(v.factories_config.scc_sha_custom_modules, null) + tags = try(v.factories_config.tags, null) + } iam = try(v.iam, {}) # type: map(list(string)) iam_bindings = try(v.iam_bindings, {}) # type: map(object({...})) iam_bindings_additive = try(v.iam_bindings_additive, {}) # type: map(object({...})) diff --git a/modules/project-factory/projects.tf b/modules/project-factory/projects.tf index d7dcbc2b8..217f202a4 100644 --- a/modules/project-factory/projects.tf +++ b/modules/project-factory/projects.tf @@ -19,10 +19,10 @@ locals { # project data from folders tree _folder_projects_raw = { - for f in try(fileset(local._folders_path, "**/*.yaml"), []) : + for f in try(fileset(local.paths.folders, "**/*.yaml"), []) : trimsuffix(f, ".yaml") => merge( { parent = dirname(f) == "." ? null : "$folder_ids:${dirname(f)}" }, - yamldecode(file("${local._folders_path}/${f}")) + yamldecode(file("${local.paths.folders}/${f}")) ) if !endswith(f, "/.config.yaml") } _projects_input = { @@ -32,16 +32,13 @@ locals { v ) } - _projects_path = try( - pathexpand(var.factories_config.projects), null - ) _projects_raw = { - for f in try(fileset(local._projects_path, "**/*.yaml"), []) : - trimsuffix(f, ".yaml") => yamldecode(file("${local._projects_path}/${f}")) + for f in try(fileset(local.paths.projects, "**/*.yaml"), []) : + trimsuffix(f, ".yaml") => yamldecode(file("${local.paths.projects}/${f}")) if !endswith(f, ".config.yaml") } _templates_path = try( - pathexpand(var.factories_config.project_templates), null + pathexpand(local.paths.project_templates), null ) _templates_raw = { for f in try(fileset(local._templates_path, "**/*.yaml"), []) : @@ -121,12 +118,11 @@ module "projects" { }) default_service_account = try(each.value.default_service_account, "keep") factories_config = { - custom_roles = try(each.value.factories_config.custom_roles, null) - org_policies = try(each.value.factories_config.org_policies, null) - observability = try(each.value.factories_config.observability, null) - quotas = try(each.value.factories_config.quotas, null) - scc_sha_custom_modules = try(each.value.factories_config.scc_sha_custom_modules, null) - tags = try(each.value.factories_config.tags, null) + for k, v in each.value.factories_config : k => try(pathexpand( + var.factories_config.basepath == null || startswith(v, "/") || startswith(v, ".") + ? v : + "${var.factories_config.basepath}/${v}" + ), null) } kms_autokeys = try(each.value.kms.autokeys, {}) labels = merge( diff --git a/modules/project-factory/schemas/project.schema.json b/modules/project-factory/schemas/project.schema.json index 4881ece7c..8d437b134 100644 --- a/modules/project-factory/schemas/project.schema.json +++ b/modules/project-factory/schemas/project.schema.json @@ -425,7 +425,7 @@ "type": "object", "additionalProperties": false, "patternProperties": { - "^[a-z0-9-]+$": { + "^[a-zA-Z0-9_-]+$": { "$ref": "#/$defs/log_bucket" } } diff --git a/modules/project-factory/variables-projects.tf b/modules/project-factory/variables-projects.tf index 88f69a388..d2e122283 100644 --- a/modules/project-factory/variables-projects.tf +++ b/modules/project-factory/variables-projects.tf @@ -231,6 +231,16 @@ variable "projects" { friendly_name = optional(string) location = optional(string) })), {}) + factories_config = optional(object({ + custom_roles = optional(string) + observability = optional(string) + org_policies = optional(string) + pam_entitlements = optional(string) + quotas = optional(string) + scc_mute_configs = optional(string) + scc_sha_custom_modules = optional(string) + tags = optional(string) + }), {}) iam = optional(map(list(string)), {}) iam_bindings = optional(map(object({ members = list(string) diff --git a/modules/project-factory/variables.tf b/modules/project-factory/variables.tf index c27c8ee25..3c4aa3919 100644 --- a/modules/project-factory/variables.tf +++ b/modules/project-factory/variables.tf @@ -165,13 +165,16 @@ variable "data_overrides" { variable "factories_config" { description = "Path to folder with YAML resource description data files." type = object({ - folders = optional(string) - project_templates = optional(string) - projects = optional(string) + basepath = string budgets = optional(object({ - billing_account_id = string - data = string - })) + billing_account = optional(string) + }), {}) + paths = optional(object({ + budgets = optional(string, "budgets") + folders = optional(string, "folders") + project_templates = optional(string, "project-templates") + projects = optional(string, "projects") + }), {}) }) nullable = false } diff --git a/modules/project/README.md b/modules/project/README.md index ae5ac7b1d..32cbd6faf 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -2041,10 +2041,10 @@ module "project" { comparison = "COMPARISON_GT" threshold_value = 100 duration = "60s" - aggregations = { + aggregations = [{ alignment_period = "60s" per_series_aligner = "ALIGN_RATE" - } + }] } }] } @@ -2153,8 +2153,8 @@ alerts: duration: 60s comparison: COMPARISON_GT aggregations: - alignment_period: 60s - per_series_aligner: ALIGN_RATE + - alignment_period: 60s + per_series_aligner: ALIGN_RATE user_labels: foo: bar ``` @@ -2285,7 +2285,7 @@ module "project" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [name](variables.tf#L248) | Project name and id suffix. | string | ✓ | | -| [alerts](variables-observability.tf#L17) | Monitoring alerts. | map(object({…})) | | {} | +| [alerts](variables-observability.tf#L17) | Monitoring alerts. | map(object({…})) | | {} | | [asset_feeds](variables.tf#L18) | Cloud Asset Inventory feeds. | map(object({…})) | | {} | | [asset_search](variables.tf#L51) | Cloud Asset Inventory search configurations. | map(object({…})) | | {} | | [auto_create_network](variables.tf#L61) | Whether to create the default network for the project. | bool | | false | diff --git a/modules/project/alerts.tf b/modules/project/alerts.tf index a28a67dc4..9795023d6 100644 --- a/modules/project/alerts.tf +++ b/modules/project/alerts.tf @@ -45,12 +45,14 @@ locals { condition_absent = !can(c.condition_absent) ? null : { duration = c.condition_absent.duration filter = try(c.condition_absent.filter, null) - aggregations = !can(c.condition_absent.aggregations) ? null : { - per_series_aligner = try(c.condition_absent.aggregations.per_series_aligner, null) - group_by_fields = try(c.condition_absent.aggregations.group_by_fields, null) - cross_series_reducer = try(c.condition_absent.aggregations.cross_series_reducer, null) - alignment_period = try(c.condition_absent.aggregations.alignment_period, null) - } + aggregations = !can(c.condition_absent.aggregations) ? null : [ + for a in c.condition_absent.aggregations : { + per_series_aligner = try(a.per_series_aligner, null) + group_by_fields = try(a.group_by_fields, null) + cross_series_reducer = try(a.cross_series_reducer, null) + alignment_period = try(a.alignment_period, null) + } + ] trigger = !can(c.condition_absent.trigger) ? null : { count = try(c.condition_absent.trigger.count, null) percent = try(c.condition_absent.trigger.percent, null) @@ -85,18 +87,22 @@ locals { evaluation_missing_data = try(c.condition_threshold.evaluation_missing_data, null) filter = try(c.condition_threshold.filter, null) threshold_value = try(c.condition_threshold.threshold_value, null) - aggregations = !can(c.condition_threshold.aggregations) ? null : { - per_series_aligner = try(c.condition_threshold.aggregations.per_series_aligner, null) - group_by_fields = try(c.condition_threshold.aggregations.group_by_fields, null) - cross_series_reducer = try(c.condition_threshold.aggregations.cross_series_reducer, null) - alignment_period = try(c.condition_threshold.aggregations.alignment_period, null) - } - denominator_aggregations = !can(c.condition_threshold.denominator_aggregations) ? null : { - per_series_aligner = try(c.condition_threshold.denominator_aggregations.per_series_aligner, null) - group_by_fields = try(c.condition_threshold.denominator_aggregations.group_by_fields, null) - cross_series_reducer = try(c.condition_threshold.denominator_aggregations.cross_series_reducer, null) - alignment_period = try(c.condition_threshold.denominator_aggregations.alignment_period, null) - } + aggregations = !can(c.condition_threshold.aggregations) ? null : [ + for a in c.condition_threshold.aggregations : { + per_series_aligner = try(a.per_series_aligner, null) + group_by_fields = try(a.group_by_fields, null) + cross_series_reducer = try(a.cross_series_reducer, null) + alignment_period = try(a.alignment_period, null) + } + ] + denominator_aggregations = !can(c.condition_threshold.denominator_aggregations) ? null : [ + for a in c.condition_threshold.denominator_aggregations : { + per_series_aligner = try(a.per_series_aligner, null) + group_by_fields = try(a.group_by_fields, null) + cross_series_reducer = try(a.cross_series_reducer, null) + alignment_period = try(a.alignment_period, null) + } + ] forecast_options = !can(c.condition_threshold.forecast_options) ? null : { forecast_horizon = c.condition_threshold.forecast_options.forecast_horizon } @@ -173,7 +179,7 @@ resource "google_monitoring_alert_policy" "alerts" { duration = condition_absent.value.duration filter = condition_absent.value.filter dynamic "aggregations" { - for_each = condition_absent.value.aggregations[*] + for_each = coalesce(condition_absent.value.aggregations, []) content { alignment_period = aggregations.value.alignment_period cross_series_reducer = aggregations.value.cross_series_reducer @@ -235,7 +241,7 @@ resource "google_monitoring_alert_policy" "alerts" { filter = condition_threshold.value.filter threshold_value = condition_threshold.value.threshold_value dynamic "aggregations" { - for_each = condition_threshold.value.aggregations[*] + for_each = coalesce(condition_threshold.value.aggregations, []) content { alignment_period = aggregations.value.alignment_period cross_series_reducer = aggregations.value.cross_series_reducer @@ -244,7 +250,7 @@ resource "google_monitoring_alert_policy" "alerts" { } } dynamic "denominator_aggregations" { - for_each = condition_threshold.value.denominator_aggregations[*] + for_each = coalesce(condition_threshold.value.denominator_aggregations, []) content { alignment_period = denominator_aggregations.value.alignment_period cross_series_reducer = denominator_aggregations.value.cross_series_reducer diff --git a/modules/project/schemas/observability.schema.json b/modules/project/schemas/observability.schema.json index cf3eb2f0a..b33bb72b3 100644 --- a/modules/project/schemas/observability.schema.json +++ b/modules/project/schemas/observability.schema.json @@ -478,23 +478,26 @@ ] }, "aggregations": { - "type": "object", - "additionalProperties": false, - "properties": { - "per_series_aligner": { - "type": "string" - }, - "group_by_fields": { - "type": "array", - "items": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "per_series_aligner": { + "type": "string" + }, + "group_by_fields": { + "type": "array", + "items": { + "type": "string" + } + }, + "cross_series_reducer": { + "type": "string" + }, + "alignment_period": { "type": "string" } - }, - "cross_series_reducer": { - "type": "string" - }, - "alignment_period": { - "type": "string" } } }, @@ -511,4 +514,4 @@ } } } -} +} \ No newline at end of file diff --git a/modules/project/variables-observability.tf b/modules/project/variables-observability.tf index 82ea7eeb4..2cc672b5c 100644 --- a/modules/project/variables-observability.tf +++ b/modules/project/variables-observability.tf @@ -39,12 +39,12 @@ variable "alerts" { condition_absent = optional(object({ duration = string filter = optional(string) - aggregations = optional(object({ + aggregations = optional(list(object({ per_series_aligner = optional(string) group_by_fields = optional(list(string)) cross_series_reducer = optional(string) alignment_period = optional(string) - })) + }))) trigger = optional(object({ count = optional(number) percent = optional(number) @@ -79,18 +79,18 @@ variable "alerts" { evaluation_missing_data = optional(string) filter = optional(string) threshold_value = optional(number) - aggregations = optional(object({ + aggregations = optional(list(object({ per_series_aligner = optional(string) group_by_fields = optional(list(string)) cross_series_reducer = optional(string) alignment_period = optional(string) - })) - denominator_aggregations = optional(object({ + }))) + denominator_aggregations = optional(list(object({ per_series_aligner = optional(string) group_by_fields = optional(list(string)) cross_series_reducer = optional(string) alignment_period = optional(string) - })) + }))) forecast_options = optional(object({ forecast_horizon = string })) diff --git a/tests/fast/stages/s0_org_setup/hardened.tfvars b/tests/fast/stages/s0_org_setup/hardened.tfvars index 8210fc435..c65fa5f92 100644 --- a/tests/fast/stages/s0_org_setup/hardened.tfvars +++ b/tests/fast/stages/s0_org_setup/hardened.tfvars @@ -1,8 +1,7 @@ factories_config = { - cicd_workflows = "data-hardened/cicd-workflows.yaml" - defaults = "data-hardened/defaults.yaml" - folders = "datasets/hardened/folders" - observability = "datasets/hardened/observability" - organization = "datasets/hardened/organization" - projects = "datasets/hardened/projects" + dataset = "datasets/hardened" + paths = { + cicd_workflows = "./data-hardened/cicd-workflows.yaml" + defaults = "./data-hardened/defaults.yaml" + } } diff --git a/tests/fast/stages/s0_org_setup/hardened.yaml b/tests/fast/stages/s0_org_setup/hardened.yaml index 7de8af8e5..5d32078f3 100644 --- a/tests/fast/stages/s0_org_setup/hardened.yaml +++ b/tests/fast/stages/s0_org_setup/hardened.yaml @@ -13,6 +13,7 @@ # limitations under the License. # yamllint disable rule:line-length + values: google_storage_bucket_object.providers["0-org-setup"]: bucket: ft0-prod-iac-core-0-iac-outputs @@ -44,6 +45,7 @@ values: name: providers/0-org-setup-providers.tf retention: [] source: null + source_md5hash: 2a0bbb00e4b7f1454a50ac7f26c23c05 temporary_hold: null timeouts: null google_storage_bucket_object.providers["0-org-setup-ro"]: @@ -76,6 +78,7 @@ values: name: providers/0-org-setup-ro-providers.tf retention: [] source: null + source_md5hash: 2a0bbb00e4b7f1454a50ac7f26c23c05 temporary_hold: null timeouts: null google_storage_bucket_object.providers["1-vpcsc"]: @@ -109,6 +112,7 @@ values: name: providers/1-vpcsc-providers.tf retention: [] source: null + source_md5hash: d2df90abc46524d941227a1dec12dd86 temporary_hold: null timeouts: null google_storage_bucket_object.providers["2-networking"]: @@ -142,6 +146,7 @@ values: name: providers/2-networking-providers.tf retention: [] source: null + source_md5hash: a724885c3dcc9850116aca1ef4d4fc5a temporary_hold: null timeouts: null google_storage_bucket_object.providers["2-project-factory"]: @@ -175,6 +180,7 @@ values: name: providers/2-project-factory-providers.tf retention: [] source: null + source_md5hash: 165844578c46bc04c4581139c8b8b8d4 temporary_hold: null timeouts: null google_storage_bucket_object.providers["2-security"]: @@ -208,6 +214,7 @@ values: name: providers/2-security-providers.tf retention: [] source: null + source_md5hash: 5969d3e40a61a42d849a81417a6a84eb temporary_hold: null timeouts: null google_storage_bucket_object.tfvars["globals"]: @@ -227,6 +234,7 @@ values: name: tfvars/0-globals.auto.tfvars.json retention: [] source: null + source_md5hash: cdbf79d3eff8bced040e5deccf39d765 temporary_hold: null timeouts: null google_storage_bucket_object.tfvars["org-setup"]: @@ -263,6 +271,7 @@ values: name: versions/0-org-setup-version.txt retention: [] source: fast_version.txt + source_md5hash: a564c0ab78f4b481f7886f9871376d2c temporary_hold: null timeouts: null google_storage_bucket_object.workflows["org-setup"]: @@ -376,6 +385,7 @@ values: name: workflows/org-setup.yaml retention: [] source: null + source_md5hash: e5dc153b195e936b1c81bc33db1935c7 temporary_hold: null timeouts: null local_file.providers["0-org-setup"]: @@ -8014,6 +8024,7 @@ values: timeouts: null value_extractor: null module.projects-observability[0].google_logging_metric.metrics["storageIamChanges"]: + bucket_name: l$log_buckets:log-0/audit-logs bucket_options: [] description: Cloud Storage IAM Permission Changes disabled: null @@ -8060,10 +8071,15 @@ values: condition_sql: [] condition_threshold: - aggregations: - - alignment_period: null - cross_series_reducer: null - group_by_fields: null - per_series_aligner: null + - alignment_period: 60s + cross_series_reducer: REDUCE_SUM + group_by_fields: + - metric.label.principal + - metric.label.method_name + - metric.label.organization_id + - metric.label.folder_id + - metric.label.project_id + per_series_aligner: ALIGN_SUM comparison: COMPARISON_GT denominator_aggregations: [] denominator_filter: null @@ -8114,10 +8130,13 @@ values: condition_sql: [] condition_threshold: - aggregations: - - alignment_period: null - cross_series_reducer: null - group_by_fields: null - per_series_aligner: null + - alignment_period: 60s + cross_series_reducer: REDUCE_SUM + group_by_fields: + - metric.label.principal + - metric.label.method_name + - metric.label.project_id + per_series_aligner: ALIGN_SUM comparison: COMPARISON_GT denominator_aggregations: [] denominator_filter: null @@ -8172,10 +8191,14 @@ values: condition_sql: [] condition_threshold: - aggregations: - - alignment_period: null - cross_series_reducer: null - group_by_fields: null - per_series_aligner: null + - alignment_period: 60s + cross_series_reducer: REDUCE_SUM + group_by_fields: + - metric.label.principal + - metric.label.method_name + - metric.label.project_id + - metric.label.database_id + per_series_aligner: ALIGN_SUM comparison: COMPARISON_GT denominator_aggregations: [] denominator_filter: null @@ -8222,10 +8245,15 @@ values: condition_sql: [] condition_threshold: - aggregations: - - alignment_period: null - cross_series_reducer: null - group_by_fields: null - per_series_aligner: null + - alignment_period: 60s + cross_series_reducer: REDUCE_SUM + group_by_fields: + - metric.label.principal + - metric.label.method_name + - metric.label.organization_id + - metric.label.project_id + - metric.label.role_name + per_series_aligner: ALIGN_SUM comparison: COMPARISON_GT denominator_aggregations: [] denominator_filter: null @@ -8266,10 +8294,12 @@ values: condition_sql: [] condition_threshold: - aggregations: - - alignment_period: null - cross_series_reducer: null - group_by_fields: null - per_series_aligner: null + - alignment_period: 60s + cross_series_reducer: REDUCE_SUM + group_by_fields: + - metric.label.principal + - metric.label.method_name + per_series_aligner: ALIGN_SUM comparison: COMPARISON_GT denominator_aggregations: [] denominator_filter: null @@ -8311,10 +8341,14 @@ values: condition_sql: [] condition_threshold: - aggregations: - - alignment_period: null - cross_series_reducer: null - group_by_fields: null - per_series_aligner: null + - alignment_period: 60s + cross_series_reducer: REDUCE_SUM + group_by_fields: + - metric.label.principal + - metric.label.method_name + - metric.label.project_id + - metric.label.firewall_rule_id + per_series_aligner: ALIGN_SUM comparison: COMPARISON_GT denominator_aggregations: [] denominator_filter: null @@ -8355,10 +8389,14 @@ values: condition_sql: [] condition_threshold: - aggregations: - - alignment_period: null - cross_series_reducer: null - group_by_fields: null - per_series_aligner: null + - alignment_period: 60s + cross_series_reducer: REDUCE_SUM + group_by_fields: + - metric.label.principal + - metric.label.method_name + - metric.label.project_id + - metric.label.network_id + per_series_aligner: ALIGN_SUM comparison: COMPARISON_GT denominator_aggregations: [] denominator_filter: null @@ -8401,10 +8439,14 @@ values: condition_sql: [] condition_threshold: - aggregations: - - alignment_period: null - cross_series_reducer: null - group_by_fields: null - per_series_aligner: null + - alignment_period: 60s + cross_series_reducer: REDUCE_SUM + group_by_fields: + - metric.label.principal + - metric.label.method_name + - metric.label.project_id + - metric.label.route_id + per_series_aligner: ALIGN_SUM comparison: COMPARISON_GT denominator_aggregations: [] denominator_filter: null @@ -8444,10 +8486,15 @@ values: condition_sql: [] condition_threshold: - aggregations: - - alignment_period: null - cross_series_reducer: null - group_by_fields: null - per_series_aligner: null + - alignment_period: 60s + cross_series_reducer: REDUCE_SUM + group_by_fields: + - metric.label.principal + - metric.label.method_name + - metric.label.organization_id + - metric.label.folder_id + - metric.label.project_id + per_series_aligner: ALIGN_SUM comparison: COMPARISON_GT denominator_aggregations: [] denominator_filter: null @@ -8489,10 +8536,15 @@ values: condition_sql: [] condition_threshold: - aggregations: - - alignment_period: null - cross_series_reducer: null - group_by_fields: null - per_series_aligner: null + - alignment_period: 60s + cross_series_reducer: REDUCE_SUM + group_by_fields: + - metric.label.principal + - metric.label.method_name + - metric.label.project_id + - metric.label.location + - metric.label.bucket_name + per_series_aligner: ALIGN_SUM comparison: COMPARISON_GT denominator_aggregations: [] denominator_filter: null @@ -8581,3 +8633,16 @@ counts: modules: 58 resources: 718 terraform_data: 4 + +outputs: + iam_principals: + domain: domain:example.org + gcp-billing-admins: group:gcp-billing-admins@example.org + gcp-devops: group:gcp-devops@example.org + gcp-network-admins: group:gcp-network-admins@example.org + gcp-organization-admins: group:fabric-fast-owners@google.com + gcp-secops-admins: group:gcp-secops-admins@example.org + gcp-security-admins: group:gcp-security-admins@example.org + gcp-support: group:gcp-support@example.org + projects: __missing__ + tfvars: __missing__ diff --git a/tests/fast/stages/s0_org_setup/simple.tfvars b/tests/fast/stages/s0_org_setup/simple.tfvars index c1ccf98b9..2951df71e 100644 --- a/tests/fast/stages/s0_org_setup/simple.tfvars +++ b/tests/fast/stages/s0_org_setup/simple.tfvars @@ -1,4 +1,6 @@ factories_config = { - cicd_workflows = "data-simple/cicd-workflows.yaml" - defaults = "data-simple/defaults.yaml" + paths = { + cicd_workflows = "./data-simple/cicd-workflows.yaml" + defaults = "./data-simple/defaults.yaml" + } } diff --git a/tests/fast/stages/s1_vpcsc/data/vpc-sc/access-levels/geo_it.yaml b/tests/fast/stages/s1_vpcsc/data-simple/access-levels/geo_it.yaml similarity index 85% rename from tests/fast/stages/s1_vpcsc/data/vpc-sc/access-levels/geo_it.yaml rename to tests/fast/stages/s1_vpcsc/data-simple/access-levels/geo_it.yaml index e26492b09..1ada79a83 100644 --- a/tests/fast/stages/s1_vpcsc/data/vpc-sc/access-levels/geo_it.yaml +++ b/tests/fast/stages/s1_vpcsc/data-simple/access-levels/geo_it.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../../../../../../modules/vpc-sc/schemas/access-level.schema.json +# yaml-language-server: $schema=../../../../../../modules/vpc-sc/schemas/access-level.schema.json conditions: - regions: diff --git a/tests/fast/stages/s1_vpcsc/data/vpc-sc/access-levels/identity_me.yaml b/tests/fast/stages/s1_vpcsc/data-simple/access-levels/identity_me.yaml similarity index 86% rename from tests/fast/stages/s1_vpcsc/data/vpc-sc/access-levels/identity_me.yaml rename to tests/fast/stages/s1_vpcsc/data-simple/access-levels/identity_me.yaml index 69a3ccd04..7b37b64ed 100644 --- a/tests/fast/stages/s1_vpcsc/data/vpc-sc/access-levels/identity_me.yaml +++ b/tests/fast/stages/s1_vpcsc/data-simple/access-levels/identity_me.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../../../../../../modules/vpc-sc/schemas/access-level.schema.json +# yaml-language-server: $schema=../../../../../../modules/vpc-sc/schemas/access-level.schema.json # yamllint disable rule:indentation conditions: diff --git a/tests/fast/stages/s1_vpcsc/data/vpc-sc/egress-policies/test.yaml b/tests/fast/stages/s1_vpcsc/data-simple/egress-policies/test.yaml similarity index 89% rename from tests/fast/stages/s1_vpcsc/data/vpc-sc/egress-policies/test.yaml rename to tests/fast/stages/s1_vpcsc/data-simple/egress-policies/test.yaml index 1e2e812bf..0f44ec6c1 100644 --- a/tests/fast/stages/s1_vpcsc/data/vpc-sc/egress-policies/test.yaml +++ b/tests/fast/stages/s1_vpcsc/data-simple/egress-policies/test.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../../../../../../modules/vpc-sc/schemas/egress-policy.schema.json +# yaml-language-server: $schema=../../../../../../modules/vpc-sc/schemas/egress-policy.schema.json from: identities: diff --git a/tests/fast/stages/s1_vpcsc/data/vpc-sc/ingress-policies/fast-org-log-sinks.yaml b/tests/fast/stages/s1_vpcsc/data-simple/ingress-policies/fast-org-log-sinks.yaml similarity index 88% rename from tests/fast/stages/s1_vpcsc/data/vpc-sc/ingress-policies/fast-org-log-sinks.yaml rename to tests/fast/stages/s1_vpcsc/data-simple/ingress-policies/fast-org-log-sinks.yaml index 5dbf1a032..dd8e806f5 100644 --- a/tests/fast/stages/s1_vpcsc/data/vpc-sc/ingress-policies/fast-org-log-sinks.yaml +++ b/tests/fast/stages/s1_vpcsc/data-simple/ingress-policies/fast-org-log-sinks.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../../../../../../modules/vpc-sc/schemas/ingress-policy.schema.json +# yaml-language-server: $schema=../../../../../../modules/vpc-sc/schemas/ingress-policy.schema.json from: access_levels: diff --git a/tests/fast/stages/s1_vpcsc/data/vpc-sc/ingress-policies/test.yaml b/tests/fast/stages/s1_vpcsc/data-simple/ingress-policies/test.yaml similarity index 89% rename from tests/fast/stages/s1_vpcsc/data/vpc-sc/ingress-policies/test.yaml rename to tests/fast/stages/s1_vpcsc/data-simple/ingress-policies/test.yaml index 3240474ca..c0eeef2fa 100644 --- a/tests/fast/stages/s1_vpcsc/data/vpc-sc/ingress-policies/test.yaml +++ b/tests/fast/stages/s1_vpcsc/data-simple/ingress-policies/test.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../../../../../../modules/vpc-sc/schemas/ingress-policy.schema.json +# yaml-language-server: $schema=../../../../../../modules/vpc-sc/schemas/ingress-policy.schema.json from: access_levels: diff --git a/tests/fast/stages/s1_vpcsc/data/vpc-sc/perimeters/default.yaml b/tests/fast/stages/s1_vpcsc/data-simple/perimeters/default.yaml similarity index 90% rename from tests/fast/stages/s1_vpcsc/data/vpc-sc/perimeters/default.yaml rename to tests/fast/stages/s1_vpcsc/data-simple/perimeters/default.yaml index e9234ab5e..3100d6af7 100644 --- a/tests/fast/stages/s1_vpcsc/data/vpc-sc/perimeters/default.yaml +++ b/tests/fast/stages/s1_vpcsc/data-simple/perimeters/default.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# yaml-language-server: $schema=../../../../../../../modules/vpc-sc/schemas/perimeter.schema.json +# yaml-language-server: $schema=../../../../../../modules/vpc-sc/schemas/perimeter.schema.json use_explicit_dry_run_spec: true spec: diff --git a/tests/fast/stages/s1_vpcsc/data/vpc-sc/restricted-services.yaml b/tests/fast/stages/s1_vpcsc/data-simple/restricted-services.yaml similarity index 100% rename from tests/fast/stages/s1_vpcsc/data/vpc-sc/restricted-services.yaml rename to tests/fast/stages/s1_vpcsc/data-simple/restricted-services.yaml diff --git a/tests/fast/stages/s1_vpcsc/factory.tfvars b/tests/fast/stages/s1_vpcsc/factory.tfvars index e6f7d1698..3b74e8753 100644 --- a/tests/fast/stages/s1_vpcsc/factory.tfvars +++ b/tests/fast/stages/s1_vpcsc/factory.tfvars @@ -2,10 +2,12 @@ automation = { outputs_bucket = "test" } factories_config = { - access_levels = "../../../tests/fast/stages/s1_vpcsc/data/vpc-sc/access-levels" - egress_policies = "../../../tests/fast/stages/s1_vpcsc/data/vpc-sc/egress-policies" - ingress_policies = "../../../tests/fast/stages/s1_vpcsc/data/vpc-sc/ingress-policies" - perimeters = "../../../tests/fast/stages/s1_vpcsc/data/vpc-sc/perimeters" + paths = { + access_levels = "./data-simple/access-levels" + egress_policies = "./data-simple/egress-policies" + ingress_policies = "./data-simple/ingress-policies" + perimeters = "./data-simple/perimeters" + } } logging = { project_number = "1234567890" diff --git a/tests/fast/stages/s1_vpcsc/hardened.tfvars b/tests/fast/stages/s1_vpcsc/hardened.tfvars new file mode 100644 index 000000000..8bd6cc0e6 --- /dev/null +++ b/tests/fast/stages/s1_vpcsc/hardened.tfvars @@ -0,0 +1,27 @@ +automation = { + outputs_bucket = "test" +} +factories_config = { + dataset = "datasets/hardened" +} +logging = { + project_number = "1234567890" + writer_identities = { + audit-logs = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com" + iam = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com" + vpc-sc = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com" + workspace-audit-logs = "serviceAccount:o1234567890-1234567890@gcp-sa-logging.iam.gserviceaccount.com" + } +} +organization = { + domain = "fast.example.com" + id = 123456789012 + customer_id = "C00000000" +} +prefix = "fast" +resource_discovery = { + enabled = false +} +storage_buckets = { + "iac-0/iac-outputs" = "test" +} diff --git a/tests/fast/stages/s1_vpcsc/hardened.yaml b/tests/fast/stages/s1_vpcsc/hardened.yaml new file mode 100644 index 000000000..806427c62 --- /dev/null +++ b/tests/fast/stages/s1_vpcsc/hardened.yaml @@ -0,0 +1,203 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + google_storage_bucket_object.tfvars[0]: + bucket: test + cache_control: null + content_disposition: null + content_encoding: null + content_language: null + contexts: [] + customer_encryption: [] + deletion_policy: null + detect_md5hash: null + event_based_hold: null + force_empty_content_type: null + metadata: null + name: tfvars/1-vpcsc.auto.tfvars.json + retention: [] + source: null + temporary_hold: null + timeouts: null + google_storage_bucket_object.version[0]: + bucket: test + cache_control: null + content_disposition: null + content_encoding: null + content_language: null + contexts: [] + customer_encryption: [] + deletion_policy: null + detect_md5hash: null + event_based_hold: null + force_empty_content_type: null + metadata: null + name: versions/1-vpcsc-version.txt + retention: [] + source: fast_version.txt + temporary_hold: null + timeouts: null + local_file.tfvars["1"]: + content_base64: null + directory_permission: '0777' + file_permission: '0644' + sensitive_content: null + source: null + module.vpc-sc.google_access_context_manager_access_level.basic["geo"]: + basic: + - combining_function: AND + conditions: + - device_policy: [] + ip_subnetworks: [] + members: [] + negate: null + regions: + - ES + - ID + - IT + required_access_levels: [] + vpc_network_sources: [] + custom: [] + description: null + timeouts: null + title: geo + module.vpc-sc.google_access_context_manager_access_policy.default[0]: + parent: organizations/123456789012 + scopes: null + timeouts: null + title: default + module.vpc-sc.google_access_context_manager_service_perimeter.regular["default"]: + description: null + perimeter_type: PERIMETER_TYPE_REGULAR + spec: [] + status: + - egress_policies: [] + ingress_policies: + - ingress_from: + - identities: + - serviceAccount:o1234567890-1234567890@gcp-sa-logging.iam.gserviceaccount.com + - serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com + identity_type: null + sources: + - access_level: '*' + resource: null + ingress_to: + - operations: + - method_selectors: [] + service_name: '*' + resources: + - projects/1234567890 + roles: [] + title: fast-org-log-sinks + resources: null + restricted_services: + - accessapproval.googleapis.com + - adsdatahub.googleapis.com + - aiplatform.googleapis.com + - apigee.googleapis.com + - apigeeconnect.googleapis.com + - artifactregistry.googleapis.com + - assuredworkloads.googleapis.com + - automl.googleapis.com + - bigquery.googleapis.com + - bigquerydatatransfer.googleapis.com + - bigtable.googleapis.com + - binaryauthorization.googleapis.com + - cloudasset.googleapis.com + - cloudbuild.googleapis.com + - cloudfunctions.googleapis.com + - cloudkms.googleapis.com + - cloudprofiler.googleapis.com + - cloudresourcemanager.googleapis.com + - cloudsearch.googleapis.com + - cloudtrace.googleapis.com + - composer.googleapis.com + - compute.googleapis.com + - connectgateway.googleapis.com + - contactcenterinsights.googleapis.com + - container.googleapis.com + - containeranalysis.googleapis.com + - containerregistry.googleapis.com + - containerthreatdetection.googleapis.com + - datacatalog.googleapis.com + - dataflow.googleapis.com + - datafusion.googleapis.com + - dataproc.googleapis.com + - datastream.googleapis.com + - dialogflow.googleapis.com + - dlp.googleapis.com + - dns.googleapis.com + - documentai.googleapis.com + - eventarc.googleapis.com + - file.googleapis.com + - gameservices.googleapis.com + - gkeconnect.googleapis.com + - gkehub.googleapis.com + - healthcare.googleapis.com + - iam.googleapis.com + - iaptunnel.googleapis.com + - language.googleapis.com + - lifesciences.googleapis.com + - logging.googleapis.com + - managedidentities.googleapis.com + - memcache.googleapis.com + - meshca.googleapis.com + - metastore.googleapis.com + - ml.googleapis.com + - monitoring.googleapis.com + - networkconnectivity.googleapis.com + - networkmanagement.googleapis.com + - networksecurity.googleapis.com + - networkservices.googleapis.com + - notebooks.googleapis.com + - opsconfigmonitoring.googleapis.com + - osconfig.googleapis.com + - oslogin.googleapis.com + - privateca.googleapis.com + - pubsub.googleapis.com + - pubsublite.googleapis.com + - recaptchaenterprise.googleapis.com + - recommender.googleapis.com + - redis.googleapis.com + - run.googleapis.com + - secretmanager.googleapis.com + - servicecontrol.googleapis.com + - servicedirectory.googleapis.com + - spanner.googleapis.com + - speakerid.googleapis.com + - speech.googleapis.com + - sqladmin.googleapis.com + - storage.googleapis.com + - storagetransfer.googleapis.com + - texttospeech.googleapis.com + - tpu.googleapis.com + - trafficdirector.googleapis.com + - transcoder.googleapis.com + - translate.googleapis.com + - videointelligence.googleapis.com + - vision.googleapis.com + - vpcaccess.googleapis.com + vpc_accessible_services: [] + timeouts: null + title: default + use_explicit_dry_run_spec: false +counts: + google_access_context_manager_access_level: 1 + google_access_context_manager_access_policy: 1 + google_access_context_manager_service_perimeter: 1 + google_storage_bucket_object: 2 + local_file: 1 + modules: 1 + resources: 6 diff --git a/tests/fast/stages/s1_vpcsc/simple.tfvars b/tests/fast/stages/s1_vpcsc/simple.tfvars index 3c732e543..621afb43e 100644 --- a/tests/fast/stages/s1_vpcsc/simple.tfvars +++ b/tests/fast/stages/s1_vpcsc/simple.tfvars @@ -2,9 +2,11 @@ automation = { outputs_bucket = "test" } factories_config = { - access_levels = "../../../tests/fast/stages/s1_vpcsc/data/vpc-sc/access-levels" - egress_policies = "../../../tests/fast/stages/s1_vpcsc/data/vpc-sc/egress-policies" - ingress_policies = "../../../tests/fast/stages/s1_vpcsc/data/vpc-sc/ingress-policies" + paths = { + access_levels = "./data-simple/access-levels" + egress_policies = "./data-simple/egress-policies" + ingress_policies = "./data-simple/ingress-policies" + } } logging = { project_number = "1234567890" diff --git a/tests/fast/stages/s1_vpcsc/tftest.yaml b/tests/fast/stages/s1_vpcsc/tftest.yaml index 83924ecb4..57ec3aaf6 100644 --- a/tests/fast/stages/s1_vpcsc/tftest.yaml +++ b/tests/fast/stages/s1_vpcsc/tftest.yaml @@ -15,7 +15,15 @@ module: fast/stages/1-vpcsc tests: + hardened: + extra_dirs: + - ../../../tests/fast/stages/s1_vpcsc/data-simple simple: + extra_dirs: + - ../../../tests/fast/stages/s1_vpcsc/data-simple factory: inventory: - simple.yaml + extra_dirs: + - ../../../tests/fast/stages/s1_vpcsc/data-simple + diff --git a/tests/fast/stages/s2_networking/ncc.tfvars b/tests/fast/stages/s2_networking/ncc.tfvars index 06b587ee0..7e2bcbe5b 100644 --- a/tests/fast/stages/s2_networking/ncc.tfvars +++ b/tests/fast/stages/s2_networking/ncc.tfvars @@ -5,18 +5,8 @@ billing_account = { id = "000000-111111-222222" } factories_config = { - defaults = "datasets/hub-and-spokes-ncc/defaults.yaml" - dns = "datasets/hub-and-spokes-ncc/dns/zones" - dns-response-policies = "datasets/hub-and-spokes-ncc/dns/response-policies" - firewall-policies = "datasets/hub-and-spokes-ncc/firewall-policies" - folders = "datasets/hub-and-spokes-ncc/folders" - interconnect = "datasets/hub-and-spokes-ncc/interconnect" - ncc-hubs = "datasets/hub-and-spokes-ncc/ncc-hubs" - nvas = "datasets/hub-and-spokes-ncc/nvas" - projects = "datasets/hub-and-spokes-ncc/projects" - vpcs = "datasets/hub-and-spokes-ncc/vpcs" + dataset = "datasets/hub-and-spokes-ncc" } - folder_ids = { "networking" = "folders/12345678" "networking/prod" = "folders/23456789" diff --git a/tests/fast/stages/s2_networking/nva.tfvars b/tests/fast/stages/s2_networking/nva.tfvars index 45d37181d..e4cb7e9e7 100644 --- a/tests/fast/stages/s2_networking/nva.tfvars +++ b/tests/fast/stages/s2_networking/nva.tfvars @@ -5,18 +5,8 @@ billing_account = { id = "000000-111111-222222" } factories_config = { - defaults = "datasets/hub-and-spokes-nva/defaults.yaml" - dns = "datasets/hub-and-spokes-nva/dns/zones" - dns-response-policies = "datasets/hub-and-spokes-nva/dns/response-policies" - firewall-policies = "datasets/hub-and-spokes-nva/firewall-policies" - folders = "datasets/hub-and-spokes-nva/folders" - interconnect = "datasets/hub-and-spokes-nva/interconnect" - ncc-hubs = "datasets/hub-and-spokes-nva/ncc-hubs" - nvas = "datasets/hub-and-spokes-nva/nvas" - projects = "datasets/hub-and-spokes-nva/projects" - vpcs = "datasets/hub-and-spokes-nva/vpcs" + dataset = "datasets/hub-and-spokes-nva" } - folder_ids = { "networking" = "folders/12345678" "networking/prod" = "folders/23456789" diff --git a/tests/fast/stages/s2_networking/simple.tfvars b/tests/fast/stages/s2_networking/simple.tfvars index 411d0a0f4..f89ba0da0 100644 --- a/tests/fast/stages/s2_networking/simple.tfvars +++ b/tests/fast/stages/s2_networking/simple.tfvars @@ -5,18 +5,8 @@ billing_account = { id = "000000-111111-222222" } factories_config = { - defaults = "datasets/hub-and-spokes-peerings/defaults.yaml" - dns = "datasets/hub-and-spokes-peerings/dns/zones" - dns-response-policies = "datasets/hub-and-spokes-peerings/dns/response-policies" - firewall-policies = "datasets/hub-and-spokes-peerings/firewall-policies" - folders = "datasets/hub-and-spokes-peerings/folders" - interconnect = "datasets/hub-and-spokes-peerings/interconnect" - ncc-hubs = "datasets/hub-and-spokes-peerings/ncc-hubs" - nvas = "datasets/hub-and-spokes-peerings/nvas" - projects = "datasets/hub-and-spokes-peerings/projects" - vpcs = "datasets/hub-and-spokes-peerings/vpcs" + dataset = "datasets/hub-and-spokes-peerings" } - folder_ids = { "networking" = "folders/12345678" "networking/prod" = "folders/23456789" diff --git a/tests/fast/stages/s2_networking/vpns.tfvars b/tests/fast/stages/s2_networking/vpns.tfvars index 243983668..4b03bcce7 100644 --- a/tests/fast/stages/s2_networking/vpns.tfvars +++ b/tests/fast/stages/s2_networking/vpns.tfvars @@ -5,18 +5,8 @@ billing_account = { id = "000000-111111-222222" } factories_config = { - defaults = "datasets/hub-and-spokes-vpns/defaults.yaml" - dns = "datasets/hub-and-spokes-vpns/dns/zones" - dns-response-policies = "datasets/hub-and-spokes-vpns/dns/response-policies" - firewall-policies = "datasets/hub-and-spokes-vpns/firewall-policies" - folders = "datasets/hub-and-spokes-vpns/folders" - interconnect = "datasets/hub-and-spokes-vpns/interconnect" - ncc-hubs = "datasets/hub-and-spokes-vpns/ncc-hubs" - nvas = "datasets/hub-and-spokes-vpns/nvas" - projects = "datasets/hub-and-spokes-vpns/projects" - vpcs = "datasets/hub-and-spokes-vpns/vpcs" + dataset = "datasets/hub-and-spokes-vpns" } - folder_ids = { "networking" = "folders/12345678" "networking/prod" = "folders/23456789" diff --git a/tests/fast/stages/s2_security/simple.tfvars b/tests/fast/stages/s2_security/simple.tfvars index 9acdd7855..19a94c132 100644 --- a/tests/fast/stages/s2_security/simple.tfvars +++ b/tests/fast/stages/s2_security/simple.tfvars @@ -5,8 +5,10 @@ billing_account = { id = "000000-111111-222222" } factories_config = { - certificate_authorities = "data-simple/certificate-authorities" - defaults = "data-simple/defaults.yaml" + paths = { + certificate_authorities = "./data-simple/certificate-authorities" + defaults = "./data-simple/defaults.yaml" + } } folder_ids = { security = "folders/12345678" diff --git a/tools/check_yaml_schema.py b/tools/check_yaml_schema.py new file mode 100755 index 000000000..dd792f436 --- /dev/null +++ b/tools/check_yaml_schema.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +'''Check YAML files against schemas defined in modelines.''' + +import json +import os +import pathlib +import re +import sys +import yaml + +import click +import jsonschema +import requests + +# Regex for the modeline +MODELINE_RE = re.compile(r'^\s*#\s*yaml-language-server:\s*\$schema=(.*)\s*$', + re.MULTILINE) + + +def load_schema(uri, base_path): + """Load a schema from a URI (URL or file path).""" + if uri.startswith('http://') or uri.startswith('https://'): + try: + response = requests.get(uri) + response.raise_for_status() + return response.json() + except Exception as e: + return None + else: + # Local file + schema_path = pathlib.Path(uri) + if not schema_path.is_absolute(): + schema_path = base_path.parent / schema_path + + if not schema_path.exists(): + return None + + try: + with open(schema_path, 'r') as f: + return json.load(f) + except json.JSONDecodeError: + try: + with open(schema_path, 'r') as f: + return yaml.safe_load(f) + except Exception: + return None + except Exception: + return None + + +@click.command() +@click.argument('paths', type=str, nargs=-1) +@click.option('--verbose', is_flag=True, + help='Print files with no schema definition.') +def main(paths, verbose): + """Validate YAML files against schemas defined in modelines.""" + files_to_check = [] + + # Collect files + for path in paths: + path = pathlib.Path(path) + if path.is_file(): + if path.suffix in ('.yaml', '.yml'): + files_to_check.append(path) + elif path.is_dir(): + for p in path.rglob('*'): + if p.is_file() and p.suffix in ('.yaml', '.yml'): + files_to_check.append(p) + + errors = [] + + for file_path in files_to_check: + try: + with open(file_path, 'r') as f: + content = f.read() + + match = MODELINE_RE.search(content) + if match: + schema_uri = match.group(1).strip() + schema = load_schema(schema_uri, file_path) + + if schema: + # Parse YAML content + try: + # Validate all documents in the file + docs = list(yaml.safe_load_all(content)) + for i, doc in enumerate(docs): + if doc is None: + continue # Skip empty docs + try: + jsonschema.validate(instance=doc, schema=schema) + except jsonschema.ValidationError as e: + errors.append( + f"{file_path} (doc {i}): Validation Error: {e.message}") + except jsonschema.SchemaError as e: + errors.append( + f"{file_path} (doc {i}): Schema Error: {e.message}") + except yaml.YAMLError as e: + errors.append(f"{file_path}: Invalid YAML - {e}") + else: + errors.append(f"{file_path}: Could not load schema {schema_uri}") + else: + if verbose: + print(f"Skipping {file_path}: No schema defined.") + + except Exception as e: + errors.append(f"{file_path}: Error processing file - {e}") + + if errors: + print("Validation failed for the following files:") + for error in errors: + print(f" - {error}") + sys.exit(1) + + +if __name__ == "__main__": + main()