Logging updates (#2139)

* Add support for logging settings to the organization module * Enable log analytics on FAST-managed logging buckets * Fix README * Fix tests
2024-03-08 10:07:12 +01:00
parent 993bef71aa
commit 759e85d6af
8 changed files with 72 additions and 36 deletions
--- a/fast/stages/0-bootstrap/log-export.tf
+++ b/fast/stages/0-bootstrap/log-export.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 Google LLC
+ * Copyright 2024 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -69,7 +69,7 @@ module "log-export-dataset" {
  source        = "../../../modules/bigquery-dataset"
  count         = contains(local.log_types, "bigquery") ? 1 : 0
  project_id    = module.log-export-project.project_id
-  id            = "audit_export"
+  id            = "logs"
  friendly_name = "Audit logs export."
  location      = local.locations.bq
 }
@@ -78,25 +78,29 @@ module "log-export-gcs" {
  source        = "../../../modules/gcs"
  count         = contains(local.log_types, "storage") ? 1 : 0
  project_id    = module.log-export-project.project_id
-  name          = "audit-logs-0"
+  name          = "logs"
  prefix        = local.prefix
  location      = local.locations.gcs
  storage_class = local.gcs_storage_class
 }

 module "log-export-logbucket" {
-  source      = "../../../modules/logging-bucket"
-  for_each    = toset([for k, v in var.log_sinks : k if v.type == "logging"])
-  parent_type = "project"
-  parent      = module.log-export-project.project_id
-  id          = "audit-logs-${each.key}"
-  location    = local.locations.logging
+  source        = "../../../modules/logging-bucket"
+  for_each      = toset([for k, v in var.log_sinks : k if v.type == "logging"])
+  parent_type   = "project"
+  parent        = module.log-export-project.project_id
+  id            = each.key
+  location      = local.locations.logging
+  log_analytics = { enable = true }
+
+  # org-level logging settings ready before we create any logging buckets
+  depends_on = [module.organization-logging]
 }

 module "log-export-pubsub" {
  source     = "../../../modules/pubsub"
  for_each   = toset([for k, v in var.log_sinks : k if v.type == "pubsub"])
  project_id = module.log-export-project.project_id
-  name       = "audit-logs-${each.key}"
+  name       = each.key
  regions    = local.locations.pubsub
 }
--- a/fast/stages/0-bootstrap/organization.tf
+++ b/fast/stages/0-bootstrap/organization.tf
@@ -130,9 +130,22 @@ import {
  to = module.organization.google_org_policy_policy.default[each.key]
 }

-module "organization" {
+module "organization-logging" {
+  # Preconfigure organization-wide logging settings to ensure project
+  # log buckets (_Default, _Required) are created in the location
+  # specified by `var.locations.logging`. This separate
+  # organization-block prevents circular dependencies with later
+  # project creation.
  source          = "../../../modules/organization"
  organization_id = "organizations/${var.organization.id}"
+  logging_settings = {
+    storage_location = var.locations.logging
+  }
+}
+
+module "organization" {
+  source          = "../../../modules/organization"
+  organization_id = module.organization-logging.id
  # human (groups) IAM bindings
  iam_by_principals = {
    for k, v in local.iam_principals :
--- a/modules/organization/README.md
+++ b/modules/organization/README.md
@@ -493,7 +493,7 @@ module "org" {
 | name | description | resources |
 |---|---|---|
 | [iam.tf](./iam.tf) | IAM bindings. | <code>google_organization_iam_binding</code> · <code>google_organization_iam_custom_role</code> · <code>google_organization_iam_member</code> |
-| [logging.tf](./logging.tf) | Log sinks and data access logs. | <code>google_bigquery_dataset_iam_member</code> · <code>google_logging_organization_exclusion</code> · <code>google_logging_organization_sink</code> · <code>google_organization_iam_audit_config</code> · <code>google_project_iam_member</code> · <code>google_pubsub_topic_iam_member</code> · <code>google_storage_bucket_iam_member</code> |
+| [logging.tf](./logging.tf) | Log sinks and data access logs. | <code>google_bigquery_dataset_iam_member</code> · <code>google_logging_organization_exclusion</code> · <code>google_logging_organization_settings</code> · <code>google_logging_organization_sink</code> · <code>google_organization_iam_audit_config</code> · <code>google_project_iam_member</code> · <code>google_pubsub_topic_iam_member</code> · <code>google_storage_bucket_iam_member</code> |
 | [main.tf](./main.tf) | Module-level locals and resources. | <code>google_compute_firewall_policy_association</code> · <code>google_essential_contacts_contact</code> |
 | [org-policy-custom-constraints.tf](./org-policy-custom-constraints.tf) | None | <code>google_org_policy_custom_constraint</code> |
 | [organization-policies.tf](./organization-policies.tf) | Organization-level organization policies. | <code>google_org_policy_policy</code> |
@@ -508,7 +508,7 @@ module "org" {

 | name | description | type | required | default |
 |---|---|:---:|:---:|:---:|
-| [organization_id](variables.tf#L145) | Organization id in organizations/nnnnnn format. | <code>string</code> | ✓ |  |
+| [organization_id](variables.tf#L155) | Organization id in organizations/nnnnnn format. | <code>string</code> | ✓ |  |
 | [contacts](variables.tf#L17) | List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [custom_roles](variables.tf#L24) | Map of role name => list of permissions to create in this project. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [factories_config](variables.tf#L31) | Paths to data files and folders that enable factory functionality. | <code title="object&#40;&#123;&#10;  custom_roles                  &#61; optional&#40;string&#41;&#10;  org_policies                  &#61; optional&#40;string&#41;&#10;  org_policy_custom_constraints &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123;&#125;</code> |
@@ -519,10 +519,11 @@ module "org" {
 | [iam_by_principals](variables-iam.tf#L54) | Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid cycle errors. Merged internally with the `iam` variable. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [logging_data_access](variables.tf#L51) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | <code>map&#40;map&#40;list&#40;string&#41;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [logging_exclusions](variables.tf#L66) | Logging exclusions for this organization in the form {NAME -> FILTER}. | <code>map&#40;string&#41;</code> |  | <code>&#123;&#125;</code> |
-| [logging_sinks](variables.tf#L73) | Logging sinks to create for the organization. | <code title="map&#40;object&#40;&#123;&#10;  bq_partitioned_table &#61; optional&#40;bool, false&#41;&#10;  description          &#61; optional&#40;string&#41;&#10;  destination          &#61; string&#10;  disabled             &#61; optional&#40;bool, false&#41;&#10;  exclusions           &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10;  filter               &#61; optional&#40;string&#41;&#10;  iam                  &#61; optional&#40;bool, true&#41;&#10;  include_children     &#61; optional&#40;bool, true&#41;&#10;  type                 &#61; string&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [logging_settings](variables.tf#L73) | Default settings for logging resources. | <code title="object&#40;&#123;&#10;  disable_default_sink &#61; optional&#40;bool&#41;&#10;  storage_location     &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [logging_sinks](variables.tf#L83) | Logging sinks to create for the organization. | <code title="map&#40;object&#40;&#123;&#10;  bq_partitioned_table &#61; optional&#40;bool, false&#41;&#10;  description          &#61; optional&#40;string&#41;&#10;  destination          &#61; string&#10;  disabled             &#61; optional&#40;bool, false&#41;&#10;  exclusions           &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10;  filter               &#61; optional&#40;string&#41;&#10;  iam                  &#61; optional&#40;bool, true&#41;&#10;  include_children     &#61; optional&#40;bool, true&#41;&#10;  type                 &#61; string&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [network_tags](variables-tags.tf#L17) | Network tags by key name. If `id` is provided, key creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | <code title="map&#40;object&#40;&#123;&#10;  description &#61; optional&#40;string, &#34;Managed by the Terraform organization module.&#34;&#41;&#10;  iam         &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;  id          &#61; optional&#40;string&#41;&#10;  network     &#61; string &#35; project_id&#47;vpc_name&#10;  values &#61; optional&#40;map&#40;object&#40;&#123;&#10;    description &#61; optional&#40;string, &#34;Managed by the Terraform organization module.&#34;&#41;&#10;    iam         &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;  &#125;&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
-| [org_policies](variables.tf#L104) | Organization policies applied to this organization keyed by policy name. | <code title="map&#40;object&#40;&#123;&#10;  inherit_from_parent &#61; optional&#40;bool&#41; &#35; for list policies only.&#10;  reset               &#61; optional&#40;bool&#41;&#10;  rules &#61; optional&#40;list&#40;object&#40;&#123;&#10;    allow &#61; optional&#40;object&#40;&#123;&#10;      all    &#61; optional&#40;bool&#41;&#10;      values &#61; optional&#40;list&#40;string&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    deny &#61; optional&#40;object&#40;&#123;&#10;      all    &#61; optional&#40;bool&#41;&#10;      values &#61; optional&#40;list&#40;string&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    enforce &#61; optional&#40;bool&#41; &#35; for boolean policies only.&#10;    condition &#61; optional&#40;object&#40;&#123;&#10;      description &#61; optional&#40;string&#41;&#10;      expression  &#61; optional&#40;string&#41;&#10;      location    &#61; optional&#40;string&#41;&#10;      title       &#61; optional&#40;string&#41;&#10;    &#125;&#41;, &#123;&#125;&#41;&#10;  &#125;&#41;&#41;, &#91;&#93;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
-| [org_policy_custom_constraints](variables.tf#L131) | Organization policy custom constraints keyed by constraint name. | <code title="map&#40;object&#40;&#123;&#10;  display_name   &#61; optional&#40;string&#41;&#10;  description    &#61; optional&#40;string&#41;&#10;  action_type    &#61; string&#10;  condition      &#61; string&#10;  method_types   &#61; list&#40;string&#41;&#10;  resource_types &#61; list&#40;string&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [org_policies](variables.tf#L114) | Organization policies applied to this organization keyed by policy name. | <code title="map&#40;object&#40;&#123;&#10;  inherit_from_parent &#61; optional&#40;bool&#41; &#35; for list policies only.&#10;  reset               &#61; optional&#40;bool&#41;&#10;  rules &#61; optional&#40;list&#40;object&#40;&#123;&#10;    allow &#61; optional&#40;object&#40;&#123;&#10;      all    &#61; optional&#40;bool&#41;&#10;      values &#61; optional&#40;list&#40;string&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    deny &#61; optional&#40;object&#40;&#123;&#10;      all    &#61; optional&#40;bool&#41;&#10;      values &#61; optional&#40;list&#40;string&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    enforce &#61; optional&#40;bool&#41; &#35; for boolean policies only.&#10;    condition &#61; optional&#40;object&#40;&#123;&#10;      description &#61; optional&#40;string&#41;&#10;      expression  &#61; optional&#40;string&#41;&#10;      location    &#61; optional&#40;string&#41;&#10;      title       &#61; optional&#40;string&#41;&#10;    &#125;&#41;, &#123;&#125;&#41;&#10;  &#125;&#41;&#41;, &#91;&#93;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [org_policy_custom_constraints](variables.tf#L141) | Organization policy custom constraints keyed by constraint name. | <code title="map&#40;object&#40;&#123;&#10;  display_name   &#61; optional&#40;string&#41;&#10;  description    &#61; optional&#40;string&#41;&#10;  action_type    &#61; string&#10;  condition      &#61; string&#10;  method_types   &#61; list&#40;string&#41;&#10;  resource_types &#61; list&#40;string&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [tag_bindings](variables-tags.tf#L45) | Tag bindings for this organization, in key => tag value id format. | <code>map&#40;string&#41;</code> |  | <code>&#123;&#125;</code> |
 | [tags](variables-tags.tf#L52) | Tags by key name. If `id` is provided, key or value creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | <code title="map&#40;object&#40;&#123;&#10;  description &#61; optional&#40;string, &#34;Managed by the Terraform organization module.&#34;&#41;&#10;  iam         &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;  id          &#61; optional&#40;string&#41;&#10;  values &#61; optional&#40;map&#40;object&#40;&#123;&#10;    description &#61; optional&#40;string, &#34;Managed by the Terraform organization module.&#34;&#41;&#10;    iam         &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;    id          &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |

@@ -534,10 +535,10 @@ module "org" {
 | [custom_role_id](outputs.tf#L22) | Map of custom role IDs created in the organization. |  |
 | [custom_roles](outputs.tf#L32) | Map of custom roles resources created in the organization. |  |
 | [id](outputs.tf#L37) | Fully qualified organization id. |  |
-| [network_tag_keys](outputs.tf#L54) | Tag key resources. |  |
-| [network_tag_values](outputs.tf#L63) | Tag value resources. |  |
-| [organization_id](outputs.tf#L73) | Organization id dependent on module resources. |  |
-| [sink_writer_identities](outputs.tf#L90) | Writer identities created for each sink. |  |
-| [tag_keys](outputs.tf#L98) | Tag key resources. |  |
-| [tag_values](outputs.tf#L107) | Tag value resources. |  |
+| [network_tag_keys](outputs.tf#L55) | Tag key resources. |  |
+| [network_tag_values](outputs.tf#L64) | Tag value resources. |  |
+| [organization_id](outputs.tf#L74) | Organization id dependent on module resources. |  |
+| [sink_writer_identities](outputs.tf#L91) | Writer identities created for each sink. |  |
+| [tag_keys](outputs.tf#L99) | Tag key resources. |  |
+| [tag_values](outputs.tf#L108) | Tag value resources. |  |
 <!-- END TFDOC -->
--- a/modules/organization/logging.tf
+++ b/modules/organization/logging.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 Google LLC
+ * Copyright 2024 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -34,6 +34,13 @@ locals {
  }
 }

+resource "google_logging_organization_settings" "default" {
+  count                = var.logging_settings != null ? 1 : 0
+  organization         = local.organization_id_numeric
+  disable_default_sink = var.logging_settings.disable_default_sink
+  storage_location     = var.logging_settings.storage_location
+}
+
 resource "google_organization_iam_audit_config" "default" {
  for_each = var.logging_data_access
  org_id   = local.organization_id_numeric
--- a/modules/organization/outputs.tf
+++ b/modules/organization/outputs.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2023 Google LLC
+ * Copyright 2024 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -38,12 +38,13 @@ output "id" {
  description = "Fully qualified organization id."
  value       = var.organization_id
  depends_on = [
+    google_logging_organization_settings.default,
    google_org_policy_custom_constraint.constraint,
    google_org_policy_policy.default,
    google_organization_iam_binding.authoritative,
    google_organization_iam_binding.bindings,
-    google_organization_iam_member.bindings,
    google_organization_iam_custom_role.roles,
+    google_organization_iam_member.bindings,
    google_tags_tag_key.default,
    google_tags_tag_key_iam_binding.default,
    google_tags_tag_value.default,
--- a/modules/organization/variables.tf
+++ b/modules/organization/variables.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2023 Google LLC
+ * Copyright 2024 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -70,6 +70,16 @@ variable "logging_exclusions" {
  nullable    = false
 }

+variable "logging_settings" {
+  description = "Default settings for logging resources."
+  type = object({
+    # TODO: add support for CMEK
+    disable_default_sink = optional(bool)
+    storage_location     = optional(string)
+  })
+  default = null
+}
+
 variable "logging_sinks" {
  description = "Logging sinks to create for the organization."
  type = map(object({
--- a/tests/fast/stages/s0_bootstrap/checklist.yaml
+++ b/tests/fast/stages/s0_bootstrap/checklist.yaml
@@ -14,27 +14,27 @@

 values:
  module.log-export-logbucket["audit-logs"].google_logging_project_bucket_config.bucket[0]:
-    bucket_id: audit-logs-audit-logs
+    bucket_id: audit-logs
    cmek_settings: []
-    enable_analytics: false
+    enable_analytics: true
    index_configs: []
    location: europe-west1
    locked: null
    project: fast-prod-audit-logs-0
    retention_days: 30
  module.log-export-logbucket["vpc-sc"].google_logging_project_bucket_config.bucket[0]:
-    bucket_id: audit-logs-vpc-sc
+    bucket_id: vpc-sc
    cmek_settings: []
-    enable_analytics: false
+    enable_analytics: true
    index_configs: []
    location: europe-west1
    locked: null
    project: fast-prod-audit-logs-0
    retention_days: 30
  module.log-export-logbucket["workspace-audit-logs"].google_logging_project_bucket_config.bucket[0]:
-    bucket_id: audit-logs-workspace-audit-logs
+    bucket_id: workspace-audit-logs
    cmek_settings: []
-    enable_analytics: false
+    enable_analytics: true
    index_configs: []
    location: europe-west1
    locked: null
@@ -380,5 +380,5 @@ counts:
  google_storage_project_service_account: 3
  google_tags_tag_key: 1
  google_tags_tag_value: 1
-  modules: 16
-  resources: 191
+  modules: 17
+  resources: 192
--- a/tests/fast/stages/s0_bootstrap/simple.yaml
+++ b/tests/fast/stages/s0_bootstrap/simple.yaml
@@ -60,8 +60,8 @@ counts:
  google_tags_tag_key: 1
  google_tags_tag_value: 1
  local_file: 7
-  modules: 15
-  resources: 182
+  modules: 16
+  resources: 183

 outputs:
  custom_roles: