kovagoadi/hunfabric
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Project object c14n in separte file

Browse Source 
Create separate file for canonicalization of project factory objects and
introduce duplicate-diff lint checker.
This commit is contained in:
Wiktor Niesiobędzki
2025-03-26 10:13:43 +00:00
committed by Wiktor Niesiobędzki
parent e4f3f713bb
commit 3e84236345
6 changed files with 302 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/linting.yml vendored
View File

@@ -120,3 +120,8 @@ jobs:
diff -rub /tmp/versions.tf /tmp/versions.tofu
DIFF_EC=$?
[[ "${DIFF_EC}" -eq "0" || -z "${OUTPUT_TF}" || -z "${OUTPUT_TOFU}" ]]
- name: Check for diverging files
id: duplicates
run: |
python3 tools/duplicate-diff.py

9
.pre-commit-config.yaml
View File

@@ -78,6 +78,15 @@ repos:
name: Check links in markdown files
language: system
entry: tools/check_links.py --no-show-summary --scan-files
- id: duplicate-diff
name: duplicate-diff
entry: python3 tools/duplicate-diff.py
language: python
language_version: python3
files: '.*(\.sh|\.tf|\.tftpl|\.tpl)$'
pass_filenames: true
require_serial: true
# - repo: https://github.com/adrienverge/yamllint
# rev: v1.34.0

1
modules/project-factory/README.md
View File

@@ -473,6 +473,7 @@ service_accounts:
| [automation.tf](./automation.tf) | Automation projects locals and resources. | <code>gcs</code> · <code>iam-service-account</code> |
| [factory-budgets.tf](./factory-budgets.tf) | Billing budget factory locals. | |
| [factory-folders.tf](./factory-folders.tf) | Folder hierarchy factory locals. | |
| [factory-projects-object.tf](./factory-projects-object.tf) | None | |
| [factory-projects.tf](./factory-projects.tf) | Projects factory locals. | |
| [folders.tf](./folders.tf) | Folder hierarchy factory resources. | <code>folder</code> |
| [main.tf](./main.tf) | Projects and billing budgets factory resources. | <code>billing-account</code> · <code>gcs</code> · <code>iam-service-account</code> · <code>project</code> |

241
modules/project-factory/factory-projects-object.tf Normal file
View File

@@ -0,0 +1,241 @@
/**
* Copyright 2025 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
# inputs
# local._projects_input - parsed data from yaml as map
# local._projects_config = object({
# data_overrides = ...
# data_defaults = ...
# })
# outputs:
# projects - map
locals {
__projects_config = {
data_defaults = merge({
billing_account = null
contacts = {}
factories_config = merge({
custom_roles = null
observability = null
org_policies = null
quotas = null
}, try(local._projects_config.data_defaults.factories_config, {
custom_roles = null
observability = null
org_policies = null
quotas = null
})
)
labels = {}
metric_scopes = []
parent = null
prefix = null
service_encryption_key_ids = {}
services = []
shared_vpc_service_config = merge({
host_project = null
network_users = []
service_agent_iam = {}
service_agent_subnet_iam = {}
service_iam_grants = []
network_subnet_users = {}
}, try(local._projects_config.data_defaults.shared_vpc_service_config, {
host_project = null
network_users = []
service_agent_iam = {}
service_agent_subnet_iam = {}
service_iam_grants = []
network_subnet_users = {}
})
)
storage_location = null
tag_bindings = {}
service_accounts = {}
vpc_sc = merge({
perimeter_name = null
perimeter_bridges = []
is_dry_run = false
}, try(local._projects_config.data_defaults.vpc_sc, {
perimeter_name = null
perimeter_bridges = []
is_dry_run = false
})
)
logging_data_access = {}
},
try(local._projects_config.data_defaults, {})
)
data_overrides = merge({
billing_account = null
contacts = {}
factories_config = merge({
custom_roles = null
observability = null
org_policies = null
quotas = null
}, try(local._projects_config.data_overrides.factories_config, {
custom_roles = null
observability = null
org_policies = null
quotas = null
})
)
parent = null
prefix = null
service_encryption_key_ids = {}
storage_location = null
tag_bindings = {}
services = []
service_accounts = {}
vpc_sc = merge({
perimeter_name = null
perimeter_bridges = []
is_dry_run = false
}, try(local._projects_config.data_overrides.vpc_sc, {
perimeter_name = null
perimeter_bridges = []
is_dry_run = false
})
)
logging_data_access = {}
},
try(local._projects_config.data_overrides, {})
)
}
_projects_output = {
for k, v in local._projects_input : lookup(v, "name", k) => merge(v, {
billing_account = try(coalesce(
local.__projects_config.data_overrides.billing_account,
try(v.billing_account, null),
local.__projects_config.data_defaults.billing_account
), null)
contacts = coalesce(
local.__projects_config.data_overrides.contacts,
try(v.contacts, null),
local.__projects_config.data_defaults.contacts
)
factories_config = {
custom_roles = try(
coalesce(
local.__projects_config.data_overrides.factories_config.custom_roles,
try(v.factories_config.custom_roles, null),
local.__projects_config.data_defaults.factories_config.custom_roles
),
null
)
observability = try(
coalesce(
local.__projects_config.data_overrides.factories_config.observability,
try(v.factories_config.observability, null),
local.__projects_config.data_defaults.factories_config.observability
),
null)
org_policies = try(
coalesce(
local.__projects_config.data_overrides.factories_config.org_policies,
try(v.factories_config.org_policies, null),
local.__projects_config.data_defaults.factories_config.org_policies
),
null)
quotas = try(
coalesce(
local.__projects_config.data_overrides.factories_config.quotas,
try(v.factories_config.quotas, null),
local.__projects_config.data_defaults.factories_config.quotas
),
null)
}
labels = coalesce(
try(v.labels, null),
local.__projects_config.data_defaults.labels
)
metric_scopes = coalesce(
try(v.metric_scopes, null),
local.__projects_config.data_defaults.metric_scopes
)
org_policies = try(v.org_policies, {})
parent = coalesce(
local.__projects_config.data_overrides.parent,
try(v.parent, null),
local.__projects_config.data_defaults.parent
)
prefix = coalesce(
local.__projects_config.data_overrides.prefix,
try(v.prefix, null),
local.__projects_config.data_defaults.prefix
)
service_encryption_key_ids = coalesce(
local.__projects_config.data_overrides.service_encryption_key_ids,
try(v.service_encryption_key_ids, null),
local.__projects_config.data_defaults.service_encryption_key_ids
)
services = coalesce(
local.__projects_config.data_overrides.services,
try(v.services, null),
local.__projects_config.data_defaults.services
)
shared_vpc_host_config = (
try(v.shared_vpc_host_config, null) != null
? merge(
{ service_projects = [] },
v.shared_vpc_host_config
)
: null
)
shared_vpc_service_config = (
try(v.shared_vpc_service_config, null) != null
? merge(
{
host_project = null
network_users = []
service_agent_iam = {}
service_agent_subnet_iam = {}
service_iam_grants = []
network_subnet_users = {}
},
v.shared_vpc_service_config
)
: local.__projects_config.data_defaults.shared_vpc_service_config
)
tag_bindings = coalesce(
local.__projects_config.data_overrides.tag_bindings,
try(v.tag_bindings, null),
local.__projects_config.data_defaults.tag_bindings
)
vpc_sc = (
local.__projects_config.data_overrides.vpc_sc != null
? local.__projects_config.data_overrides.vpc_sc
: (
try(v.vpc_sc, null) != null
? merge({
perimeter_name = null
perimeter_bridges = []
is_dry_run = false
}, v.vpc_sc)
: local.__projects_config.data_defaults.vpc_sc
)
)
logging_data_access = coalesce(
local.__projects_config.data_overrides.logging_data_access,
try(v.logging_data_access, null),
local.__projects_config.data_defaults.logging_data_access
)
# non-project resources
# buckets = try(v.buckets, {})
# service_accounts = try(v.service_accounts, {})
})
}
}

138
modules/project-factory/factory-projects.tf
View File

@@ -28,7 +28,7 @@ locals {
}
)
_project_path = try(pathexpand(var.factories_config.projects_data_path), null)
_projects = merge(
_projects_input = merge(
{
for f in try(fileset(local._project_path, "**/*.yaml"), []) :
basename(trimsuffix(f, ".yaml")) => yamldecode(file("${local._project_path}/${f}"))
@@ -36,139 +36,27 @@ locals {
local._hierarchy_projects
)
_project_budgets = flatten([
for k, v in local._projects : [
for k, v in local._projects_input : [
for b in try(v.billing_budgets, []) : {
budget = b
project = lookup(v, "name", k)
}
]
])
_projects_config = {
data_overrides = var.data_overrides
data_defaults = var.data_defaults
}
projects = {
for k, v in local._projects_output : k => merge({
buckets = try(v.buckets, {})
service_accounts = try(v.service_accounts, {})
}, v)
}
project_budgets = {
for v in local._project_budgets : v.budget => v.project...
}
projects = {
for k, v in local._projects : lookup(v, "name", k) => merge(v, {
billing_account = try(coalesce(
var.data_overrides.billing_account,
try(v.billing_account, null),
var.data_defaults.billing_account
), null)
contacts = coalesce(
var.data_overrides.contacts,
try(v.contacts, null),
var.data_defaults.contacts
)
factories_config = {
custom_roles = try(
coalesce(
var.data_overrides.factories_config.custom_roles,
try(v.factories_config.custom_roles, null),
var.data_defaults.factories_config.custom_roles
),
null
)
observability = try(
coalesce(
var.data_overrides.factories_config.observability,
try(v.factories_config.observability, null),
var.data_defaults.factories_config.observability
),
null)
org_policies = try(
coalesce(
var.data_overrides.factories_config.org_policies,
try(v.factories_config.org_policies, null),
var.data_defaults.factories_config.org_policies
),
null)
quotas = try(
coalesce(
var.data_overrides.factories_config.quotas,
try(v.factories_config.quotas, null),
var.data_defaults.factories_config.quotas
),
null)
}
labels = coalesce(
try(v.labels, null),
var.data_defaults.labels
)
metric_scopes = coalesce(
try(v.metric_scopes, null),
var.data_defaults.metric_scopes
)
org_policies = try(v.org_policies, {})
parent = coalesce(
var.data_overrides.parent,
try(v.parent, null),
var.data_defaults.parent
)
prefix = coalesce(
var.data_overrides.prefix,
try(v.prefix, null),
var.data_defaults.prefix
)
service_encryption_key_ids = coalesce(
var.data_overrides.service_encryption_key_ids,
try(v.service_encryption_key_ids, null),
var.data_defaults.service_encryption_key_ids
)
services = coalesce(
var.data_overrides.services,
try(v.services, null),
var.data_defaults.services
)
shared_vpc_host_config = (
try(v.shared_vpc_host_config, null) != null
? merge(
{ service_projects = [] },
v.shared_vpc_host_config
)
: null
)
shared_vpc_service_config = (
try(v.shared_vpc_service_config, null) != null
? merge(
{
host_project = null
network_users = []
service_agent_iam = {}
service_agent_subnet_iam = {}
service_iam_grants = []
network_subnet_users = {}
},
v.shared_vpc_service_config
)
: var.data_defaults.shared_vpc_service_config
)
tag_bindings = coalesce(
var.data_overrides.tag_bindings,
try(v.tag_bindings, null),
var.data_defaults.tag_bindings
)
vpc_sc = (
var.data_overrides.vpc_sc != null
? var.data_overrides.vpc_sc
: (
try(v.vpc_sc, null) != null
? merge({
perimeter_name = null
perimeter_bridges = []
is_dry_run = false
}, v.vpc_sc)
: var.data_defaults.vpc_sc
)
)
logging_data_access = coalesce(
var.data_overrides.logging_data_access,
try(v.logging_data_access, null),
var.data_defaults.logging_data_access
)
# non-project resources
buckets = try(v.buckets, {})
service_accounts = try(v.service_accounts, {})
})
}
buckets = flatten([
for k, v in local.projects : [
for name, opts in v.buckets : {

33
tools/duplicate-diff.py Normal file
View File

@@ -0,0 +1,33 @@
#
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import filecmp
import sys
duplicates = [
[
# "modules/net-vpc-factory/factory-projects-object.tf",
# data factory
"modules/project-factory/factory-projects-object.tf",
],
]
for group in duplicates:
first = group[0]
for second in group[1:]:
if not filecmp.cmp(first, second): # true if files are the same
print(f'found diff between {first} and {second}')
sys.exit(1)