From 3e84236345c538028c2fcb100da7b2741f15e46c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Wed, 26 Mar 2025 10:13:43 +0000 Subject: [PATCH] Project object c14n in separte file Create separate file for canonicalization of project factory objects and introduce duplicate-diff lint checker. --- .github/workflows/linting.yml | 5 + .pre-commit-config.yaml | 9 + modules/project-factory/README.md | 1 + .../factory-projects-object.tf | 241 ++++++++++++++++++ modules/project-factory/factory-projects.tf | 138 +--------- tools/duplicate-diff.py | 33 +++ 6 files changed, 302 insertions(+), 125 deletions(-) create mode 100644 modules/project-factory/factory-projects-object.tf create mode 100644 tools/duplicate-diff.py diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 03a2b8f20..5c8d09a69 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -120,3 +120,8 @@ jobs: diff -rub /tmp/versions.tf /tmp/versions.tofu DIFF_EC=$? [[ "${DIFF_EC}" -eq "0" || -z "${OUTPUT_TF}" || -z "${OUTPUT_TOFU}" ]] + + - name: Check for diverging files + id: duplicates + run: | + python3 tools/duplicate-diff.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d12df74c8..7785b8928 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -78,6 +78,15 @@ repos: name: Check links in markdown files language: system entry: tools/check_links.py --no-show-summary --scan-files + - id: duplicate-diff + name: duplicate-diff + entry: python3 tools/duplicate-diff.py + language: python + language_version: python3 + files: '.*(\.sh|\.tf|\.tftpl|\.tpl)$' + pass_filenames: true + require_serial: true + # - repo: https://github.com/adrienverge/yamllint # rev: v1.34.0 diff --git a/modules/project-factory/README.md b/modules/project-factory/README.md index 00f55be66..d3f6de13b 100644 --- a/modules/project-factory/README.md +++ b/modules/project-factory/README.md @@ -473,6 +473,7 @@ service_accounts: | [automation.tf](./automation.tf) | Automation projects locals and resources. | gcs · iam-service-account | | [factory-budgets.tf](./factory-budgets.tf) | Billing budget factory locals. | | | [factory-folders.tf](./factory-folders.tf) | Folder hierarchy factory locals. | | +| [factory-projects-object.tf](./factory-projects-object.tf) | None | | | [factory-projects.tf](./factory-projects.tf) | Projects factory locals. | | | [folders.tf](./folders.tf) | Folder hierarchy factory resources. | folder | | [main.tf](./main.tf) | Projects and billing budgets factory resources. | billing-account · gcs · iam-service-account · project | diff --git a/modules/project-factory/factory-projects-object.tf b/modules/project-factory/factory-projects-object.tf new file mode 100644 index 000000000..4dca20ba2 --- /dev/null +++ b/modules/project-factory/factory-projects-object.tf @@ -0,0 +1,241 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# inputs +# local._projects_input - parsed data from yaml as map +# local._projects_config = object({ +# data_overrides = ... +# data_defaults = ... +# }) +# outputs: +# projects - map +locals { + __projects_config = { + data_defaults = merge({ + billing_account = null + contacts = {} + factories_config = merge({ + custom_roles = null + observability = null + org_policies = null + quotas = null + }, try(local._projects_config.data_defaults.factories_config, { + custom_roles = null + observability = null + org_policies = null + quotas = null + }) + ) + labels = {} + metric_scopes = [] + parent = null + prefix = null + service_encryption_key_ids = {} + services = [] + shared_vpc_service_config = merge({ + host_project = null + network_users = [] + service_agent_iam = {} + service_agent_subnet_iam = {} + service_iam_grants = [] + network_subnet_users = {} + }, try(local._projects_config.data_defaults.shared_vpc_service_config, { + host_project = null + network_users = [] + service_agent_iam = {} + service_agent_subnet_iam = {} + service_iam_grants = [] + network_subnet_users = {} + }) + ) + storage_location = null + tag_bindings = {} + service_accounts = {} + vpc_sc = merge({ + perimeter_name = null + perimeter_bridges = [] + is_dry_run = false + }, try(local._projects_config.data_defaults.vpc_sc, { + perimeter_name = null + perimeter_bridges = [] + is_dry_run = false + }) + ) + logging_data_access = {} + }, + try(local._projects_config.data_defaults, {}) + ) + data_overrides = merge({ + billing_account = null + contacts = {} + factories_config = merge({ + custom_roles = null + observability = null + org_policies = null + quotas = null + }, try(local._projects_config.data_overrides.factories_config, { + custom_roles = null + observability = null + org_policies = null + quotas = null + }) + ) + parent = null + prefix = null + service_encryption_key_ids = {} + storage_location = null + tag_bindings = {} + services = [] + service_accounts = {} + vpc_sc = merge({ + perimeter_name = null + perimeter_bridges = [] + is_dry_run = false + }, try(local._projects_config.data_overrides.vpc_sc, { + perimeter_name = null + perimeter_bridges = [] + is_dry_run = false + }) + ) + logging_data_access = {} + }, + try(local._projects_config.data_overrides, {}) + ) + } + _projects_output = { + for k, v in local._projects_input : lookup(v, "name", k) => merge(v, { + billing_account = try(coalesce( + local.__projects_config.data_overrides.billing_account, + try(v.billing_account, null), + local.__projects_config.data_defaults.billing_account + ), null) + contacts = coalesce( + local.__projects_config.data_overrides.contacts, + try(v.contacts, null), + local.__projects_config.data_defaults.contacts + ) + factories_config = { + custom_roles = try( + coalesce( + local.__projects_config.data_overrides.factories_config.custom_roles, + try(v.factories_config.custom_roles, null), + local.__projects_config.data_defaults.factories_config.custom_roles + ), + null + ) + observability = try( + coalesce( + local.__projects_config.data_overrides.factories_config.observability, + try(v.factories_config.observability, null), + local.__projects_config.data_defaults.factories_config.observability + ), + null) + org_policies = try( + coalesce( + local.__projects_config.data_overrides.factories_config.org_policies, + try(v.factories_config.org_policies, null), + local.__projects_config.data_defaults.factories_config.org_policies + ), + null) + quotas = try( + coalesce( + local.__projects_config.data_overrides.factories_config.quotas, + try(v.factories_config.quotas, null), + local.__projects_config.data_defaults.factories_config.quotas + ), + null) + } + labels = coalesce( + try(v.labels, null), + local.__projects_config.data_defaults.labels + ) + metric_scopes = coalesce( + try(v.metric_scopes, null), + local.__projects_config.data_defaults.metric_scopes + ) + org_policies = try(v.org_policies, {}) + parent = coalesce( + local.__projects_config.data_overrides.parent, + try(v.parent, null), + local.__projects_config.data_defaults.parent + ) + prefix = coalesce( + local.__projects_config.data_overrides.prefix, + try(v.prefix, null), + local.__projects_config.data_defaults.prefix + ) + service_encryption_key_ids = coalesce( + local.__projects_config.data_overrides.service_encryption_key_ids, + try(v.service_encryption_key_ids, null), + local.__projects_config.data_defaults.service_encryption_key_ids + ) + services = coalesce( + local.__projects_config.data_overrides.services, + try(v.services, null), + local.__projects_config.data_defaults.services + ) + shared_vpc_host_config = ( + try(v.shared_vpc_host_config, null) != null + ? merge( + { service_projects = [] }, + v.shared_vpc_host_config + ) + : null + ) + shared_vpc_service_config = ( + try(v.shared_vpc_service_config, null) != null + ? merge( + { + host_project = null + network_users = [] + service_agent_iam = {} + service_agent_subnet_iam = {} + service_iam_grants = [] + network_subnet_users = {} + }, + v.shared_vpc_service_config + ) + : local.__projects_config.data_defaults.shared_vpc_service_config + ) + tag_bindings = coalesce( + local.__projects_config.data_overrides.tag_bindings, + try(v.tag_bindings, null), + local.__projects_config.data_defaults.tag_bindings + ) + vpc_sc = ( + local.__projects_config.data_overrides.vpc_sc != null + ? local.__projects_config.data_overrides.vpc_sc + : ( + try(v.vpc_sc, null) != null + ? merge({ + perimeter_name = null + perimeter_bridges = [] + is_dry_run = false + }, v.vpc_sc) + : local.__projects_config.data_defaults.vpc_sc + ) + ) + logging_data_access = coalesce( + local.__projects_config.data_overrides.logging_data_access, + try(v.logging_data_access, null), + local.__projects_config.data_defaults.logging_data_access + ) + # non-project resources + # buckets = try(v.buckets, {}) + # service_accounts = try(v.service_accounts, {}) + }) + } +} diff --git a/modules/project-factory/factory-projects.tf b/modules/project-factory/factory-projects.tf index dca889ca6..deb619822 100644 --- a/modules/project-factory/factory-projects.tf +++ b/modules/project-factory/factory-projects.tf @@ -28,7 +28,7 @@ locals { } ) _project_path = try(pathexpand(var.factories_config.projects_data_path), null) - _projects = merge( + _projects_input = merge( { for f in try(fileset(local._project_path, "**/*.yaml"), []) : basename(trimsuffix(f, ".yaml")) => yamldecode(file("${local._project_path}/${f}")) @@ -36,139 +36,27 @@ locals { local._hierarchy_projects ) _project_budgets = flatten([ - for k, v in local._projects : [ + for k, v in local._projects_input : [ for b in try(v.billing_budgets, []) : { budget = b project = lookup(v, "name", k) } ] ]) + _projects_config = { + data_overrides = var.data_overrides + data_defaults = var.data_defaults + } + projects = { + for k, v in local._projects_output : k => merge({ + buckets = try(v.buckets, {}) + service_accounts = try(v.service_accounts, {}) + }, v) + } project_budgets = { for v in local._project_budgets : v.budget => v.project... } - projects = { - for k, v in local._projects : lookup(v, "name", k) => merge(v, { - billing_account = try(coalesce( - var.data_overrides.billing_account, - try(v.billing_account, null), - var.data_defaults.billing_account - ), null) - contacts = coalesce( - var.data_overrides.contacts, - try(v.contacts, null), - var.data_defaults.contacts - ) - factories_config = { - custom_roles = try( - coalesce( - var.data_overrides.factories_config.custom_roles, - try(v.factories_config.custom_roles, null), - var.data_defaults.factories_config.custom_roles - ), - null - ) - observability = try( - coalesce( - var.data_overrides.factories_config.observability, - try(v.factories_config.observability, null), - var.data_defaults.factories_config.observability - ), - null) - org_policies = try( - coalesce( - var.data_overrides.factories_config.org_policies, - try(v.factories_config.org_policies, null), - var.data_defaults.factories_config.org_policies - ), - null) - quotas = try( - coalesce( - var.data_overrides.factories_config.quotas, - try(v.factories_config.quotas, null), - var.data_defaults.factories_config.quotas - ), - null) - } - labels = coalesce( - try(v.labels, null), - var.data_defaults.labels - ) - metric_scopes = coalesce( - try(v.metric_scopes, null), - var.data_defaults.metric_scopes - ) - org_policies = try(v.org_policies, {}) - parent = coalesce( - var.data_overrides.parent, - try(v.parent, null), - var.data_defaults.parent - ) - prefix = coalesce( - var.data_overrides.prefix, - try(v.prefix, null), - var.data_defaults.prefix - ) - service_encryption_key_ids = coalesce( - var.data_overrides.service_encryption_key_ids, - try(v.service_encryption_key_ids, null), - var.data_defaults.service_encryption_key_ids - ) - services = coalesce( - var.data_overrides.services, - try(v.services, null), - var.data_defaults.services - ) - shared_vpc_host_config = ( - try(v.shared_vpc_host_config, null) != null - ? merge( - { service_projects = [] }, - v.shared_vpc_host_config - ) - : null - ) - shared_vpc_service_config = ( - try(v.shared_vpc_service_config, null) != null - ? merge( - { - host_project = null - network_users = [] - service_agent_iam = {} - service_agent_subnet_iam = {} - service_iam_grants = [] - network_subnet_users = {} - }, - v.shared_vpc_service_config - ) - : var.data_defaults.shared_vpc_service_config - ) - tag_bindings = coalesce( - var.data_overrides.tag_bindings, - try(v.tag_bindings, null), - var.data_defaults.tag_bindings - ) - vpc_sc = ( - var.data_overrides.vpc_sc != null - ? var.data_overrides.vpc_sc - : ( - try(v.vpc_sc, null) != null - ? merge({ - perimeter_name = null - perimeter_bridges = [] - is_dry_run = false - }, v.vpc_sc) - : var.data_defaults.vpc_sc - ) - ) - logging_data_access = coalesce( - var.data_overrides.logging_data_access, - try(v.logging_data_access, null), - var.data_defaults.logging_data_access - ) - # non-project resources - buckets = try(v.buckets, {}) - service_accounts = try(v.service_accounts, {}) - }) - } + buckets = flatten([ for k, v in local.projects : [ for name, opts in v.buckets : { diff --git a/tools/duplicate-diff.py b/tools/duplicate-diff.py new file mode 100644 index 000000000..895dcc04a --- /dev/null +++ b/tools/duplicate-diff.py @@ -0,0 +1,33 @@ +# +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import filecmp +import sys + +duplicates = [ + [ + # "modules/net-vpc-factory/factory-projects-object.tf", + # data factory + "modules/project-factory/factory-projects-object.tf", + ], +] + +for group in duplicates: + first = group[0] + for second in group[1:]: + if not filecmp.cmp(first, second): # true if files are the same + print(f'found diff between {first} and {second}') + sys.exit(1)