diff --git a/CHANGELOG.md b/CHANGELOG.md index e0e0181f4..48eeae806 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +- **incompatible change** Add support for partitioned tables on Organization sinks - new `cloud-run` module - added gVNIC support to `compute-vm` module - added a rule factory to `net-vpc-firewall` module diff --git a/modules/organization/README.md b/modules/organization/README.md index 5e60be8ca..5b9108c3d 100644 --- a/modules/organization/README.md +++ b/modules/organization/README.md @@ -110,36 +110,40 @@ module "org" { logging_sinks = { warnings = { - type = "gcs" - destination = module.gcs.name - filter = "severity=WARNING" - iam = false - include_children = true - exclusions = {} + type = "gcs" + destination = module.gcs.name + filter = "severity=WARNING" + iam = false + include_children = true + bq_partitioned_table = null + exclusions = {} } info = { - type = "bigquery" - destination = module.dataset.id - filter = "severity=INFO" - iam = false - include_children = true - exclusions = {} + type = "bigquery" + destination = module.dataset.id + filter = "severity=INFO" + iam = false + include_children = true + bq_partitioned_table = true + exclusions = {} } notice = { - type = "pubsub" - destination = module.pubsub.id - filter = "severity=NOTICE" - iam = true - include_children = true - exclusions = {} + type = "pubsub" + destination = module.pubsub.id + filter = "severity=NOTICE" + iam = true + include_children = true + bq_partitioned_table = null + exclusions = {} } debug = { - type = "logging" - destination = module.bucket.id - filter = "severity=DEBUG" - iam = true - include_children = false - exclusions = { + type = "logging" + destination = module.bucket.id + filter = "severity=DEBUG" + iam = true + include_children = false + bq_partitioned_table = null + exclusions = { no-compute = "logName:compute" } } @@ -186,7 +190,7 @@ module "org" { | *iam_audit_config_authoritative* | IAM Authoritative service audit logging configuration. Service as key, map of log permission (eg DATA_READ) and excluded members as value for each service. Audit config should also be authoritative when using authoritative bindings. Use with caution. | map(map(list(string))) | | null | | *iam_bindings_authoritative* | IAM authoritative bindings, in {ROLE => [MEMBERS]} format. Roles and members not explicitly listed will be cleared. Bindings should also be authoritative when using authoritative audit config. Use with caution. | map(list(string)) | | null | | *logging_exclusions* | Logging exclusions for this organization in the form {NAME -> FILTER}. | map(string) | | {} | -| *logging_sinks* | Logging sinks to create for this organization. | map(object({...})) | | {} | +| *logging_sinks* | Logging sinks to create for this organization. | map(object({...})) | | {} | | *policy_boolean* | Map of boolean org policies and enforcement value, set value to null for policy restore. | map(bool) | | {} | | *policy_list* | Map of list org policies, status is true for allow, false for deny, null for restore. Values can only be used for allow or deny. | map(object({...})) | | {} | diff --git a/modules/organization/main.tf b/modules/organization/main.tf index e845c5494..5f6737d03 100644 --- a/modules/organization/main.tf +++ b/modules/organization/main.tf @@ -289,6 +289,13 @@ resource "google_logging_organization_sink" "sink" { filter = each.value.filter include_children = each.value.include_children + dynamic "bigquery_options" { + for_each = each.value.bq_partitioned_table == true ? [""] : [] + content { + use_partitioned_tables = each.value.bq_partitioned_table + } + } + dynamic "exclusions" { for_each = each.value.exclusions iterator = exclusion diff --git a/modules/organization/variables.tf b/modules/organization/variables.tf index a98492d3e..e01c1e467 100644 --- a/modules/organization/variables.tf +++ b/modules/organization/variables.tf @@ -111,11 +111,12 @@ variable "logging_exclusions" { variable "logging_sinks" { description = "Logging sinks to create for this organization." type = map(object({ - destination = string - type = string - filter = string - iam = bool - include_children = bool + destination = string + type = string + filter = string + iam = bool + include_children = bool + bq_partitioned_table = bool # TODO exclusions also support description and disabled exclusions = map(string) })) diff --git a/tests/modules/organization/fixture/variables.tf b/tests/modules/organization/fixture/variables.tf index eb4e807c1..b94633463 100644 --- a/tests/modules/organization/fixture/variables.tf +++ b/tests/modules/organization/fixture/variables.tf @@ -81,12 +81,13 @@ variable "firewall_policy_attachments" { variable "logging_sinks" { type = map(object({ - destination = string - type = string - filter = string - iam = bool - include_children = bool - exclusions = map(string) + destination = string + type = string + filter = string + iam = bool + include_children = bool + bq_partitioned_table = bool + exclusions = map(string) })) default = {} } diff --git a/tests/modules/organization/test_plan_logging.py b/tests/modules/organization/test_plan_logging.py index 1ad6aec48..98163afce 100644 --- a/tests/modules/organization/test_plan_logging.py +++ b/tests/modules/organization/test_plan_logging.py @@ -22,148 +22,154 @@ FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixture") def test_sinks(plan_runner): - "Test folder-level sinks." - logging_sinks = """ { + "Test folder-level sinks." + logging_sinks = """ { warning = { - type = "gcs" - destination = "mybucket" - filter = "severity=WARNING" - iam = true - include_children = true - exclusions = {} + type = "gcs" + destination = "mybucket" + filter = "severity=WARNING" + iam = true + include_children = true + bq_partitioned_table = null + exclusions = {} } info = { - type = "bigquery" - destination = "projects/myproject/datasets/mydataset" - filter = "severity=INFO" - iam = true - include_children = true - exclusions = {} + type = "bigquery" + destination = "projects/myproject/datasets/mydataset" + filter = "severity=INFO" + iam = true + include_children = true + bq_partitioned_table = false + exclusions = {} } notice = { - type = "pubsub" - destination = "projects/myproject/topics/mytopic" - filter = "severity=NOTICE" - iam = true - include_children = false - exclusions = {} + type = "pubsub" + destination = "projects/myproject/topics/mytopic" + filter = "severity=NOTICE" + iam = true + include_children = false + bq_partitioned_table = null + exclusions = {} } debug = { - type = "logging" - destination = "projects/myproject/locations/global/buckets/mybucket" - filter = "severity=DEBUG" - iam = true - include_children = false - exclusions = { + type = "logging" + destination = "projects/myproject/locations/global/buckets/mybucket" + filter = "severity=DEBUG" + iam = true + include_children = false + bq_partitioned_table = null + exclusions = { no-compute = "logName:compute" no-container = "logName:container" } } } """ - _, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks) - assert len(resources) == 8 + _, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks) + assert len(resources) == 8 - resource_types = Counter([r["type"] for r in resources]) - assert resource_types == { - "google_logging_organization_sink": 4, - "google_bigquery_dataset_iam_member": 1, - "google_project_iam_member": 1, - "google_pubsub_topic_iam_member": 1, - "google_storage_bucket_iam_member": 1, - } + resource_types = Counter([r["type"] for r in resources]) + assert resource_types == { + "google_logging_organization_sink": 4, + "google_bigquery_dataset_iam_member": 1, + "google_project_iam_member": 1, + "google_pubsub_topic_iam_member": 1, + "google_storage_bucket_iam_member": 1, + } - sinks = [r for r in resources if r["type"] == "google_logging_organization_sink"] - assert sorted([r["index"] for r in sinks]) == [ - "debug", - "info", - "notice", - "warning", - ] - values = [ - ( - r["index"], - r["values"]["filter"], - r["values"]["destination"], - r["values"]["include_children"], - ) - for r in sinks - ] - assert sorted(values) == [ - ( - "debug", - "severity=DEBUG", - "logging.googleapis.com/projects/myproject/locations/global/buckets/mybucket", - False, - ), - ( - "info", - "severity=INFO", - "bigquery.googleapis.com/projects/myproject/datasets/mydataset", - True, - ), - ( - "notice", - "severity=NOTICE", - "pubsub.googleapis.com/projects/myproject/topics/mytopic", - False, - ), - ("warning", "severity=WARNING", "storage.googleapis.com/mybucket", True), - ] + sinks = [r for r in resources if r["type"] + == "google_logging_organization_sink"] + assert sorted([r["index"] for r in sinks]) == [ + "debug", + "info", + "notice", + "warning", + ] + values = [ + ( + r["index"], + r["values"]["filter"], + r["values"]["destination"], + r["values"]["include_children"], + ) + for r in sinks + ] + assert sorted(values) == [ + ( + "debug", + "severity=DEBUG", + "logging.googleapis.com/projects/myproject/locations/global/buckets/mybucket", + False, + ), + ( + "info", + "severity=INFO", + "bigquery.googleapis.com/projects/myproject/datasets/mydataset", + True, + ), + ( + "notice", + "severity=NOTICE", + "pubsub.googleapis.com/projects/myproject/topics/mytopic", + False, + ), + ("warning", "severity=WARNING", "storage.googleapis.com/mybucket", True), + ] - bindings = [r for r in resources if "member" in r["type"]] - values = [(r["index"], r["type"], r["values"]["role"]) for r in bindings] - assert sorted(values) == [ - ("debug", "google_project_iam_member", "roles/logging.bucketWriter"), - ("info", "google_bigquery_dataset_iam_member", "roles/bigquery.dataEditor"), - ("notice", "google_pubsub_topic_iam_member", "roles/pubsub.publisher"), - ("warning", "google_storage_bucket_iam_member", "roles/storage.objectCreator"), - ] + bindings = [r for r in resources if "member" in r["type"]] + values = [(r["index"], r["type"], r["values"]["role"]) for r in bindings] + assert sorted(values) == [ + ("debug", "google_project_iam_member", "roles/logging.bucketWriter"), + ("info", "google_bigquery_dataset_iam_member", "roles/bigquery.dataEditor"), + ("notice", "google_pubsub_topic_iam_member", "roles/pubsub.publisher"), + ("warning", "google_storage_bucket_iam_member", "roles/storage.objectCreator"), + ] - exclusions = [(r["index"], r["values"]["exclusions"]) for r in sinks] - assert sorted(exclusions) == [ - ( - "debug", - [ - { - "description": None, - "disabled": False, - "filter": "logName:compute", - "name": "no-compute", - }, - { - "description": None, - "disabled": False, - "filter": "logName:container", - "name": "no-container", - }, - ], - ), - ("info", []), - ("notice", []), - ("warning", []), - ] + exclusions = [(r["index"], r["values"]["exclusions"]) for r in sinks] + assert sorted(exclusions) == [ + ( + "debug", + [ + { + "description": None, + "disabled": False, + "filter": "logName:compute", + "name": "no-compute", + }, + { + "description": None, + "disabled": False, + "filter": "logName:container", + "name": "no-container", + }, + ], + ), + ("info", []), + ("notice", []), + ("warning", []), + ] def test_exclusions(plan_runner): - "Test folder-level logging exclusions." - logging_exclusions = ( - "{" - 'exclusion1 = "resource.type=gce_instance", ' - 'exclusion2 = "severity=NOTICE", ' - "}" - ) - _, resources = plan_runner(FIXTURES_DIR, logging_exclusions=logging_exclusions) - assert len(resources) == 2 - exclusions = [ - r for r in resources if r["type"] == "google_logging_organization_exclusion" - ] - assert sorted([r["index"] for r in exclusions]) == [ - "exclusion1", - "exclusion2", - ] - values = [(r["index"], r["values"]["filter"]) for r in exclusions] - assert sorted(values) == [ - ("exclusion1", "resource.type=gce_instance"), - ("exclusion2", "severity=NOTICE"), - ] + "Test folder-level logging exclusions." + logging_exclusions = ( + "{" + 'exclusion1 = "resource.type=gce_instance", ' + 'exclusion2 = "severity=NOTICE", ' + "}" + ) + _, resources = plan_runner( + FIXTURES_DIR, logging_exclusions=logging_exclusions) + assert len(resources) == 2 + exclusions = [ + r for r in resources if r["type"] == "google_logging_organization_exclusion" + ] + assert sorted([r["index"] for r in exclusions]) == [ + "exclusion1", + "exclusion2", + ] + values = [(r["index"], r["values"]["filter"]) for r in exclusions] + assert sorted(values) == [ + ("exclusion1", "resource.type=gce_instance"), + ("exclusion2", "severity=NOTICE"), + ]