kovagoadi/hunfabric
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Decouple logging sinks in FAST stages 0 and 1 (#3858)

Browse Source 
* Decouple logging sinks in FAST stages 0 and 1

* Fix ternary

* Fix tests

* Fix vpcsc tests

* Fix typo

* remove filenames
This commit is contained in:
Julio Castillo
2026-04-11 00:14:15 +02:00
committed by GitHub
parent ebc4669b4d
commit 1cb643b3c1
11 changed files with 287 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
fast/stages/0-org-setup/output-files.tf
View File

@@ -1,5 +1,5 @@
/**
* Copyright 2025 Google LLC
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -45,6 +45,19 @@ locals {
local.org_tag_values
)
})
of_logging_sinks = {
# Include project_id in the destination if supported (omitted for
# "storage" sinks).
for k, v in module.organization-iam[0].logging_sinks :
k => merge(
v,
(
strcontains(v.destination, "projects/")
? { project_id = split("/", v.destination)[2] }
: {}
)
)
}
of_outputs_bucket = (
local.output_files.storage_bucket == null
? null
@@ -98,16 +111,12 @@ locals {
automation = {
outputs_bucket = local.of_outputs_bucket
}
custom_roles = local.of_ctx.custom_roles
folder_ids = local.of_ctx.folder_ids
iam_principals = local.of_ctx.iam_principals
logging = {
writer_identities = module.organization-iam[0].sink_writer_identities
project_number = module.factory.project_numbers["log-0"]
}
project_ids = local.of_ctx.project_ids,
project_numbers = module.factory.project_numbers
# project_numbers = module.factory.project_numbers
custom_roles = local.of_ctx.custom_roles
folder_ids = local.of_ctx.folder_ids
iam_principals = local.of_ctx.iam_principals
logging_sinks = local.of_logging_sinks
project_ids = local.of_ctx.project_ids,
project_numbers = module.factory.project_numbers
service_accounts = module.factory.service_account_emails
storage_buckets = module.factory.storage_buckets
tag_values = local.of_ctx.tag_values

13
fast/stages/1-vpcsc/README.md
View File

@@ -456,7 +456,7 @@ Some references that might be useful in setting up this stage:
| name | description | type | required | default | producer |
|---|---|:---:|:---:|:---:|:---:|
| [organization](variables-fast.tf#L35) | Organization details. | <code>object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | | <code>0-org-setup</code> |
| [organization](variables-fast.tf#L48) | Organization details. | <code>object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | | <code>0-org-setup</code> |
| [access_levels](variables.tf#L17) | Access level definitions. | <code>map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>&#123;&#125;</code> | |
| [access_policy](variables.tf#L67) | Access policy id (used for tenant-level VPC-SC configurations). | <code>number</code> | | <code>null</code> | |
| [context](variables.tf#L73) | External context used in replacements. | <code>object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>&#123;&#125;</code> | |
@@ -464,13 +464,14 @@ Some references that might be useful in setting up this stage:
| [factories_config](variables.tf#L130) | Paths to folders that enable factory functionality. | <code>object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>&#123;&#125;</code> | |
| [iam_principals](variables-fast.tf#L17) | Org-level IAM principals. | <code>map&#40;string&#41;</code> | | <code>&#123;&#125;</code> | <code>0-org-setup</code> |
| [ingress_policies](variables.tf#L147) | Ingress policy definitions that can be referenced in perimeters. | <code>map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>&#123;&#125;</code> | |
| [logging](variables-fast.tf#L25) | Log writer identities for organization / folders. | <code>object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> | <code>0-org-setup</code> |
| [logging_sinks](variables-fast.tf#L25) | Log sinks for the organization. | <code>map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>&#123;&#125;</code> | <code>0-org-setup</code> |
| [perimeters](variables.tf#L189) | Perimeter definitions. | <code>map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>&#123;&#125;</code> | |
| [project_numbers](variables-fast.tf#L46) | Project numbers. | <code>map&#40;number&#41;</code> | | <code>&#123;&#125;</code> | <code>0-org-setup</code> |
| [project_ids](variables-fast.tf#L59) | Project IDs. | <code>map&#40;string&#41;</code> | | <code>&#123;&#125;</code> | <code>0-org-setup</code> |
| [project_numbers](variables-fast.tf#L67) | Project numbers. | <code>map&#40;number&#41;</code> | | <code>&#123;&#125;</code> | <code>0-org-setup</code> |
| [resource_discovery](variables.tf#L223) | Automatic discovery of perimeter projects. | <code>object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>&#123;&#125;</code> | |
| [root_node](variables-fast.tf#L54) | Root node for the hierarchy, if running in tenant mode. | <code>string</code> | | <code>null</code> | <code>0-org-setup</code> |
| [service_accounts](variables-fast.tf#L68) | Org-level service accounts. | <code>map&#40;string&#41;</code> | | <code>&#123;&#125;</code> | <code>0-org-setup</code> |
| [storage_buckets](variables-fast.tf#L76) | Storage buckets created in the bootstrap stage. | <code>map&#40;string&#41;</code> | | <code>&#123;&#125;</code> | <code>0-org-setup</code> |
| [root_node](variables-fast.tf#L75) | Root node for the hierarchy, if running in tenant mode. | <code>string</code> | | <code>null</code> | <code>0-org-setup</code> |
| [service_accounts](variables-fast.tf#L89) | Org-level service accounts. | <code>map&#40;string&#41;</code> | | <code>&#123;&#125;</code> | <code>0-org-setup</code> |
| [storage_buckets](variables-fast.tf#L97) | Storage buckets created in the bootstrap stage. | <code>map&#40;string&#41;</code> | | <code>&#123;&#125;</code> | <code>0-org-setup</code> |
## Outputs

22
fast/stages/1-vpcsc/main.tf
View File

@@ -1,5 +1,5 @@
/**
* Copyright 2025 Google LLC
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -25,7 +25,12 @@ locals {
for k, v in local.ctx.storage_buckets : "$storage_buckets:${k}" => v
}
# fail if we have no valid defaults
_defaults = yamldecode(file(local.paths.defaults))
_defaults = yamldecode(file(local.paths.defaults))
_project_numbers = merge(var.project_numbers, local._ctx.project_numbers)
_project_id_to_num = {
for k, v in var.project_ids :
v => local._project_numbers[k]
}
discovered_projects = var.resource_discovery.enabled != true ? [] : [
for v in module.vpc-sc-discovery[0].project_numbers :
"projects/${v}"
@@ -42,13 +47,20 @@ locals {
local._ctx.iam_principals
)
identity_sets = merge(local._ctx.identity_sets, {
logging_identities = try(distinct(values(var.logging.writer_identities)), [])
logging_identities = distinct([
for _, v in var.logging_sinks : v.writer_identity
])
})
project_numbers = merge(var.project_numbers, local._ctx.project_numbers)
project_numbers = local._project_numbers
resource_sets = merge(
{
discovered_projects = local.discovered_projects
logging_project = try(["projects/${var.logging.project_number}"], [])
logging_project = distinct(compact([
for _, v in var.logging_sinks :
try(v.project_id, null) != null
? "projects/${lookup(local._project_id_to_num, v.project_id, v.project_id)}"
: null
]))
org_setup_projects = [
for k, v in var.project_numbers : "projects/${v}"
]

37
fast/stages/1-vpcsc/variables-fast.tf
View File

@@ -1,5 +1,5 @@
/**
* Copyright 2024 Google LLC
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -22,14 +22,27 @@ variable "iam_principals" {
default = {}
}
variable "logging" {
variable "logging_sinks" {
# tfdoc:variable:source 0-org-setup
description = "Log writer identities for organization / folders."
type = object({
writer_identities = map(string)
project_number = optional(string)
})
default = null
description = "Log sinks for the organization."
type = map(object({
project_id = optional(string)
writer_identity = string
## other available fields:
# bigquery_options = list(string)
# description = string
# disabled = bool
# destination = string
# exclusions = list(string)
# filter = string
# id = string
# include_children = bool
# intercept_children = bool
# name = string
# org_id = string
}))
default = {}
nullable = false
}
variable "organization" {
@@ -43,6 +56,14 @@ variable "organization" {
nullable = false
}
variable "project_ids" {
# tfdoc:variable:source 0-org-setup
description = "Project IDs."
type = map(string)
nullable = false
default = {}
}
variable "project_numbers" {
# tfdoc:variable:source 0-org-setup
description = "Project numbers."

25
modules/organization/README.md
View File

@@ -1045,16 +1045,17 @@ module "org" {
| [custom_roles](outputs.tf#L34) | Map of custom roles resources created in the organization. | |
| [id](outputs.tf#L39) | Fully qualified organization id. | |
| [logging_identities](outputs.tf#L57) | Principals used for logging sinks. | |
| [network_tag_keys](outputs.tf#L69) | Tag key resources. | |
| [network_tag_values](outputs.tf#L78) | Tag value resources. | |
| [organization_id](outputs.tf#L88) | Organization id dependent on module resources. | |
| [organization_policies_ids](outputs.tf#L105) | Map of ORGANIZATION_POLICIES => ID in the organization. | |
| [scc_custom_sha_modules_ids](outputs.tf#L110) | Map of SCC CUSTOM SHA MODULES => ID in the organization. | |
| [scc_mute_configs](outputs.tf#L115) | SCC mute configurations. | |
| [service_agents](outputs.tf#L120) | Identities of all organization-level service agents. | |
| [sink_writer_identities](outputs.tf#L125) | Writer identities created for each sink. | |
| [tag_keys](outputs.tf#L133) | Tag key resources. | |
| [tag_values](outputs.tf#L142) | Tag value resources. | |
| [workforce_identity_provider_names](outputs.tf#L150) | Workforce Identity provider names. | |
| [workforce_identity_providers](outputs.tf#L157) | Workforce Identity provider attributes. | |
| [logging_sinks](outputs.tf#L69) | Logging sink resources. | |
| [network_tag_keys](outputs.tf#L77) | Tag key resources. | |
| [network_tag_values](outputs.tf#L86) | Tag value resources. | |
| [organization_id](outputs.tf#L96) | Organization id dependent on module resources. | |
| [organization_policies_ids](outputs.tf#L113) | Map of ORGANIZATION_POLICIES => ID in the organization. | |
| [scc_custom_sha_modules_ids](outputs.tf#L118) | Map of SCC CUSTOM SHA MODULES => ID in the organization. | |
| [scc_mute_configs](outputs.tf#L123) | SCC mute configurations. | |
| [service_agents](outputs.tf#L128) | Identities of all organization-level service agents. | |
| [sink_writer_identities](outputs.tf#L133) | Writer identities created for each sink. | |
| [tag_keys](outputs.tf#L141) | Tag key resources. | |
| [tag_values](outputs.tf#L150) | Tag value resources. | |
| [workforce_identity_provider_names](outputs.tf#L158) | Workforce Identity provider names. | |
| [workforce_identity_providers](outputs.tf#L165) | Workforce Identity provider attributes. | |
<!-- END TFDOC -->

8
modules/organization/outputs.tf
View File

@@ -66,6 +66,14 @@ output "logging_identities" {
}
}
output "logging_sinks" {
description = "Logging sink resources."
value = {
for name, sink in google_logging_organization_sink.sink :
name => sink
}
}
output "network_tag_keys" {
description = "Tag key resources."
value = {

73
tests/fast/stages/s1_vpcsc/factory.tfvars
View File

@@ -9,13 +9,66 @@ factories_config = {
perimeters = "./data-simple/perimeters"
}
}
logging = {
project_number = "1234567890"
writer_identities = {
audit-logs = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
iam = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
vpc-sc = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
workspace-audit-logs = "serviceAccount:o1234567890-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
logging_sinks = {
audit-logs = {
bigquery_options = []
description = "audit-logs (Terraform-managed)."
destination = "logging.googleapis.com/projects/ft0-prod-audit-logs-0/locations/global/buckets/audit-logs"
disabled = false
exclusions = []
filter = ""
id = "organizations/1234567890/sinks/audit-logs"
include_children = true
intercept_children = false
name = "audit-logs"
org_id = "529325294915"
project_id = "ft0-prod-audit-logs-0"
writer_identity = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
}
iam = {
bigquery_options = []
description = "iam (Terraform-managed)."
destination = "logging.googleapis.com/projects/ft0-prod-audit-logs-0/locations/global/buckets/iam"
disabled = false
exclusions = []
filter = ""
id = "organizations/1234567890/sinks/iam"
include_children = true
intercept_children = false
name = "iam"
org_id = "529325294915"
project_id = "ft0-prod-audit-logs-0"
writer_identity = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
}
vpc-sc = {
bigquery_options = []
description = "vpc-sc (Terraform-managed)."
destination = "logging.googleapis.com/projects/ft0-prod-audit-logs-0/locations/global/buckets/vpc-sc"
disabled = false
exclusions = []
filter = ""
id = "organizations/1234567890/sinks/vpc-sc"
include_children = true
intercept_children = false
name = "vpc-sc"
org_id = "529325294915"
project_id = "ft0-prod-audit-logs-0"
writer_identity = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
}
workspace = {
bigquery_options = []
description = "workspace (Terraform-managed)."
destination = "logging.googleapis.com/projects/ft0-prod-audit-logs-0/locations/global/buckets/workspace"
disabled = false
exclusions = []
filter = ""
id = "organizations/1234567890/sinks/workspace"
include_children = true
intercept_children = false
name = "workspace"
org_id = "529325294915"
project_id = "ft0-prod-audit-logs-0"
writer_identity = "serviceAccount:o1234567890-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
}
}
organization = {
@@ -24,6 +77,12 @@ organization = {
customer_id = "C00000000"
}
prefix = "fast"
project_ids = {
log-0 = "ft0-prod-audit-logs-0"
}
project_numbers = {
log-0 = 1122334455
}
resource_discovery = {
enabled = false
}

28
tests/fast/stages/s1_vpcsc/hardened.tfvars
View File

@@ -9,13 +9,21 @@ iam_principals = {
"service_agents/org/ktd-hpsa" = "serviceAccount:service-org-1234567890@gcp-sa-ktd-hpsa.iam.gserviceaccount.com"
"service_agents/org/security-center-api" = "serviceAccount:service-org-1234567890@security-center-api.iam.gserviceaccount.com"
}
logging = {
project_number = "1234567890"
writer_identities = {
audit-logs = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
iam = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
vpc-sc = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
workspace-audit-logs = "serviceAccount:o1234567890-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
logging_sinks = {
audit-logs = {
bigquery_options = []
description = "audit-logs (Terraform-managed)."
destination = "logging.googleapis.com/projects/ft0-prod-audit-logs-0/locations/global/buckets/audit-logs"
disabled = false
exclusions = []
filter = ""
id = "organizations/1234567890/sinks/audit-logs"
include_children = true
intercept_children = false
name = "audit-logs"
org_id = "529325294915"
project_id = "ft0-prod-audit-logs-0"
writer_identity = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
}
}
organization = {
@@ -24,6 +32,12 @@ organization = {
customer_id = "C00000000"
}
prefix = "fast"
project_ids = {
log-0 = "ft0-prod-audit-logs-0"
}
project_numbers = {
log-0 = 1122334455
}
resource_discovery = {
enabled = false
}

5
tests/fast/stages/s1_vpcsc/hardened.yaml
View File

@@ -47,6 +47,7 @@ values:
name: versions/1-vpcsc-version.txt
retention: []
source: fast_version.txt
source_md5hash: f20ea48e4666fcbce129cff9fa658d16
temporary_hold: null
timeouts: null
local_file.tfvars["1"]:
@@ -106,7 +107,6 @@ values:
ingress_policies:
- ingress_from:
- identities:
- serviceAccount:o1234567890-1234567890@gcp-sa-logging.iam.gserviceaccount.com
- serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com
identity_type: null
sources:
@@ -117,7 +117,7 @@ values:
- method_selectors: []
service_name: '*'
resources:
- projects/1234567890
- projects/1122334455
roles: []
title: fast-org-log-sinks
- ingress_from:
@@ -229,6 +229,7 @@ values:
timeouts: null
title: default
use_explicit_dry_run_spec: false
counts:
google_access_context_manager_access_level: 1
google_access_context_manager_access_policy: 1

73
tests/fast/stages/s1_vpcsc/simple.tfvars
View File

@@ -8,13 +8,66 @@ factories_config = {
ingress_policies = "./data-simple/ingress-policies"
}
}
logging = {
project_number = "1234567890"
writer_identities = {
audit-logs = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
iam = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
vpc-sc = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
workspace-audit-logs = "serviceAccount:o1234567890-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
logging_sinks = {
audit-logs = {
bigquery_options = []
description = "audit-logs (Terraform-managed)."
destination = "logging.googleapis.com/projects/ft0-prod-audit-logs-0/locations/global/buckets/audit-logs"
disabled = false
exclusions = []
filter = ""
id = "organizations/1234567890/sinks/audit-logs"
include_children = true
intercept_children = false
name = "audit-logs"
org_id = "529325294915"
project_id = "ft0-prod-audit-logs-0"
writer_identity = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
}
iam = {
bigquery_options = []
description = "iam (Terraform-managed)."
destination = "logging.googleapis.com/projects/ft0-prod-audit-logs-0/locations/global/buckets/iam"
disabled = false
exclusions = []
filter = ""
id = "organizations/1234567890/sinks/iam"
include_children = true
intercept_children = false
name = "iam"
org_id = "529325294915"
project_id = "ft0-prod-audit-logs-0"
writer_identity = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
}
vpc-sc = {
bigquery_options = []
description = "vpc-sc (Terraform-managed)."
destination = "logging.googleapis.com/projects/ft0-prod-audit-logs-0/locations/global/buckets/vpc-sc"
disabled = false
exclusions = []
filter = ""
id = "organizations/1234567890/sinks/vpc-sc"
include_children = true
intercept_children = false
name = "vpc-sc"
org_id = "529325294915"
project_id = "ft0-prod-audit-logs-0"
writer_identity = "serviceAccount:service-org-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
}
workspace = {
bigquery_options = []
description = "workspace (Terraform-managed)."
destination = "logging.googleapis.com/projects/ft0-prod-audit-logs-0/locations/global/buckets/workspace"
disabled = false
exclusions = []
filter = ""
id = "organizations/1234567890/sinks/workspace"
include_children = true
intercept_children = false
name = "workspace"
org_id = "529325294915"
project_id = "ft0-prod-audit-logs-0"
writer_identity = "serviceAccount:o1234567890-1234567890@gcp-sa-logging.iam.gserviceaccount.com"
}
}
organization = {
@@ -37,6 +90,12 @@ perimeters = {
}
}
prefix = "fast"
project_ids = {
log-0 = "ft0-prod-audit-logs-0"
}
project_numbers = {
log-0 = 1122334455
}
resource_discovery = {
enabled = false
}

39
tests/fast/stages/s1_vpcsc/simple.yaml
View File

@@ -1,4 +1,4 @@
# Copyright 2025 Google LLC
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -19,15 +19,43 @@ values:
content_disposition: null
content_encoding: null
content_language: null
contexts: []
customer_encryption: []
deletion_policy: null
detect_md5hash: null
event_based_hold: null
force_empty_content_type: null
metadata: null
name: tfvars/1-vpcsc.auto.tfvars.json
retention: []
source: null
temporary_hold: null
timeouts: null
google_storage_bucket_object.version[0]:
bucket: test
cache_control: null
content_disposition: null
content_encoding: null
content_language: null
contexts: []
customer_encryption: []
deletion_policy: null
detect_md5hash: null
event_based_hold: null
force_empty_content_type: null
metadata: null
name: versions/1-vpcsc-version.txt
retention: []
source: fast_version.txt
source_md5hash: f20ea48e4666fcbce129cff9fa658d16
temporary_hold: null
timeouts: null
local_file.tfvars["1"]:
content_base64: null
directory_permission: '0777'
file_permission: '0644'
sensitive_content: null
source: null
module.vpc-sc.google_access_context_manager_access_level.basic["geo_it"]:
basic:
- combining_function: AND
@@ -89,6 +117,8 @@ values:
service_name: monitoring.googleapis.com
resources:
- '*'
roles: []
title: test
ingress_policies:
- ingress_from:
- identities:
@@ -103,7 +133,9 @@ values:
- method_selectors: []
service_name: '*'
resources:
- projects/1234567890
- projects/1122334455
roles: []
title: fast-org-log-sinks
- ingress_from:
- identities:
- user:user@fast.example.com
@@ -123,6 +155,8 @@ values:
service_name: monitoring.googleapis.com
resources:
- '*'
roles: []
title: test
resources:
- projects/1234567890
restricted_services:
@@ -217,6 +251,7 @@ values:
timeouts: null
title: default
use_explicit_dry_run_spec: true
counts:
google_access_context_manager_access_level: 2
google_access_context_manager_access_policy: 1