diff --git a/adrs/fast/addon-stages.md b/adrs/fast/addon-stages.md new file mode 100644 index 000000000..325e1c71d --- /dev/null +++ b/adrs/fast/addon-stages.md @@ -0,0 +1,76 @@ +# Add-on stages + +**authors:** [Ludo](https://github.com/ludoo), [Julio](https://github.com/juliocc) +**date:** Jan 5, 2025 + +## Status + +Under implementation + +## Context + +Some optional features are too complex to directly embed in stages, as they would complicate the variable scope, need to be replicated across parallel stages, and introduce a lot of nested code for the benefit of a small subset of users. + +This need has surfaced with the network security stage, which has taken the approach of spreading its resources across different stages (security, networking, and its own netsec) and resulted in very layered, complicated code which is not easy to deploy or maintain. + +This is how the current netsec stage looks like from a resource perspective: + +![image](https://github.com/user-attachments/assets/c9778cd8-8dd4-4f7c-b74b-c5d8ad7e7d30) + +Furthermore, the stage also tries to do "too much", by behaving as a full stage and adopting a design that statically maps its resources onto all FAST environments and networks. This results in code that is really hard to adapt for real life use cases and impossible to keep forward compatible, as changes are extensive and spread out across three stages. + +## Proposal + +The proposal is to adopt a completely different approach, where large optional featuresets that we don't want to embed in our default stages should become "addon stages" that: + +- reuse the IaC service account and bucket of the stage they interact with (e.g. networking for network security) to eliminate the need for custom IAM +- encapsulate all their resources in a single root module (the add-on stage) +- don't implement a static design but deal with the smallest possible unit of work, so that they can be cloned to implement different designs via tfvars +- provide optional FAST output variables for the main stages + +This is what the network security stage looks like, once refactored according this proposal: + +![image](https://github.com/user-attachments/assets/748b8b53-8df7-444e-9c71-f74e462a96f1) + +With this approach + +- there are no dependencies in resman except for a providers file that adds a prefix to the state backend and reuses networking service accounts and bucket +- the stage design does not deal with environments, but simply implements one complete set of NGFW resources in a given project (typically the net landing or shared environment project) and allows free configuration of zones and VPC attachments +- any relevant resource already defined in the "main" stages can be referred to via interpolation, by using the stages outputs as contexts + +The code then becomes really simple to use, read and evolve since it's essentially decoupled from the main stages except for a handful of FAST interface variables. + +Add-on stages should live in a separate folder from stages, and once we finally manage to reafctor networking into a simple stage, we go back to having a clear progression for main stages that should make it easier for users to get to grips with FAST's complexity. We might also want to scrap the plugins folder, and replace with a short document explaining the pattern. + +```bash +fast +├── addons + ├── 1-tenant-factory + └── 2-network-security +├── assets +│   └── templates +├── extras +│   ├── 0-cicd-github +│   └── 0-cicd-gitlab +├── plugins +│   └── 2-networking-serverless-connector +└── stages + ├── 0-bootstrap + ├── 1-resman + ├── 1-vpcsc + ├── 2-networking-a-simple + ├── 2-networking-b-nva + ├── 2-networking-c-separate-envs + ├── 2-project-factory + ├── 2-security + ├── 3-gcve-dev + └── 3-gke-dev +``` + +## Decision + +TBD + +## Consequences + +This approach also maps well to the current tenant factory stage, which essentially acts as a parallel resman stage reusing the same set of IaC resources. diff --git a/blueprints/data-solutions/bq-ml/README.md b/blueprints/data-solutions/bq-ml/README.md index eabfe9d69..39a9c0dec 100644 --- a/blueprints/data-solutions/bq-ml/README.md +++ b/blueprints/data-solutions/bq-ml/README.md @@ -97,5 +97,5 @@ module "test" { prefix = "prefix" } -# tftest modules=9 resources=68 +# tftest modules=9 resources=69 ``` diff --git a/blueprints/data-solutions/data-playground/README.md b/blueprints/data-solutions/data-playground/README.md index 34824bf69..9890619d3 100644 --- a/blueprints/data-solutions/data-playground/README.md +++ b/blueprints/data-solutions/data-playground/README.md @@ -84,5 +84,5 @@ module "test" { parent = "folders/467898377" } } -# tftest modules=8 resources=67 +# tftest modules=8 resources=68 ``` diff --git a/blueprints/data-solutions/vertex-mlops/README.md b/blueprints/data-solutions/vertex-mlops/README.md index 6277c4f52..6c4e5913d 100644 --- a/blueprints/data-solutions/vertex-mlops/README.md +++ b/blueprints/data-solutions/vertex-mlops/README.md @@ -72,7 +72,7 @@ module "test" { project_id = "test-dev" } } -# tftest modules=11 resources=89 +# tftest modules=11 resources=90 ``` ## Variables @@ -128,5 +128,5 @@ module "test" { project_id = "test-dev" } } -# tftest modules=13 resources=94 e2e +# tftest modules=13 resources=95 e2e ``` diff --git a/blueprints/data-solutions/vertex-mlops/vertex.tf b/blueprints/data-solutions/vertex-mlops/vertex.tf index 19e5ca7c5..90e57f391 100644 --- a/blueprints/data-solutions/vertex-mlops/vertex.tf +++ b/blueprints/data-solutions/vertex-mlops/vertex.tf @@ -124,4 +124,7 @@ resource "google_workbench_instance" "playground" { depends_on = [ google_project_iam_member.shared_vpc, ] + lifecycle { + ignore_changes = [gce_setup[0].metadata["resource-url"]] + } } diff --git a/blueprints/secops/bindplane-gke/README.md b/blueprints/secops/bindplane-gke/README.md index 85de48b36..a89beeb99 100644 --- a/blueprints/secops/bindplane-gke/README.md +++ b/blueprints/secops/bindplane-gke/README.md @@ -108,16 +108,16 @@ Access the management console leveraging credentials bootstrapped via terraform | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [bindplane_secrets](variables.tf#L26) | Bindplane secrets. | object({…}) | ✓ | | -| [network_config](variables.tf#L57) | Shared VPC network configurations to use for GKE cluster. | object({…}) | ✓ | | -| [prefix](variables.tf#L79) | Prefix used for resource names. | string | ✓ | | -| [project_id](variables.tf#L98) | Project id, references existing project if `project_create` is null. | string | ✓ | | -| [region](variables.tf#L103) | GCP region. | string | ✓ | | -| [bindplane_config](variables.tf#L17) | Bindplane config. | object({…}) | | {} | -| [cluster_config](variables.tf#L36) | GKE cluster configuration. | object({…}) | | {} | -| [dns_config](variables.tf#L47) | DNS config. | object({…}) | | {} | -| [postgresql_config](variables.tf#L69) | Cloud SQL postgresql config. | object({…}) | | {} | -| [project_create](variables.tf#L89) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | +| [bindplane_secrets](variables.tf#L27) | Bindplane secrets. | object({…}) | ✓ | | +| [network_config](variables.tf#L58) | Shared VPC network configurations to use for GKE cluster. | object({…}) | ✓ | | +| [prefix](variables.tf#L80) | Prefix used for resource names. | string | ✓ | | +| [project_id](variables.tf#L99) | Project id, references existing project if `project_create` is null. | string | ✓ | | +| [region](variables.tf#L104) | GCP region. | string | ✓ | | +| [bindplane_config](variables.tf#L17) | Bindplane config. | object({…}) | | {} | +| [cluster_config](variables.tf#L37) | GKE cluster configuration. | object({…}) | | {} | +| [dns_config](variables.tf#L48) | DNS config. | object({…}) | | {} | +| [postgresql_config](variables.tf#L70) | Cloud SQL postgresql config. | object({…}) | | {} | +| [project_create](variables.tf#L90) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | ## Outputs diff --git a/blueprints/secops/bindplane-gke/config/values.yaml.tpl b/blueprints/secops/bindplane-gke/config/values.yaml.tpl index a6098a935..496f1e160 100644 --- a/blueprints/secops/bindplane-gke/config/values.yaml.tpl +++ b/blueprints/secops/bindplane-gke/config/values.yaml.tpl @@ -34,6 +34,13 @@ config: # of pods is recommended. replicas: 2 +image: + # -- Image name to be used. Defaults to `ghcr.io/observiq/bindplane-ee`. + name: "" + # Overrides the image tag whose default is {{ .Chart.AppVersion }} + # -- Image tag to use. Defaults to the version defined in the Chart's release. + tag: ${tag} + resources: # Allow cpu bursting by leaving limits.cpu unset requests: diff --git a/blueprints/secops/bindplane-gke/main.tf b/blueprints/secops/bindplane-gke/main.tf index 4e0b12c34..6f84c781d 100644 --- a/blueprints/secops/bindplane-gke/main.tf +++ b/blueprints/secops/bindplane-gke/main.tf @@ -215,6 +215,7 @@ resource "helm_release" "bindplane" { gcp_project_id = module.project.project_id hostname = "${var.dns_config.hostname}.${var.dns_config.domain}" address = "ingress" + tag = var.bindplane_config.image_tag })] depends_on = [ diff --git a/blueprints/secops/bindplane-gke/variables.tf b/blueprints/secops/bindplane-gke/variables.tf index 973c5db18..2b2ca5e11 100644 --- a/blueprints/secops/bindplane-gke/variables.tf +++ b/blueprints/secops/bindplane-gke/variables.tf @@ -17,6 +17,7 @@ variable "bindplane_config" { description = "Bindplane config." type = object({ + image_tag = optional(string, "") tls_certificate_cer = optional(string, null) tls_certificate_key = optional(string, null) }) diff --git a/modules/project-factory/README.md b/modules/project-factory/README.md index 03d47043d..7b705612c 100644 --- a/modules/project-factory/README.md +++ b/modules/project-factory/README.md @@ -440,10 +440,10 @@ update_rules: | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [factories_config](variables.tf#L100) | Path to folder with YAML resource description data files. | object({…}) | ✓ | | -| [data_defaults](variables.tf#L17) | Optional default values used when corresponding project data from files are missing. | object({…}) | | {} | -| [data_merges](variables.tf#L54) | Optional values that will be merged with corresponding data from files. Combines with `data_defaults`, file data, and `data_overrides`. | object({…}) | | {} | -| [data_overrides](variables.tf#L73) | Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`. | object({…}) | | {} | +| [factories_config](variables.tf#L112) | Path to folder with YAML resource description data files. | object({…}) | ✓ | | +| [data_defaults](variables.tf#L17) | Optional default values used when corresponding project data from files are missing. | object({…}) | | {} | +| [data_merges](variables.tf#L60) | Optional values that will be merged with corresponding data from files. Combines with `data_defaults`, file data, and `data_overrides`. | object({…}) | | {} | +| [data_overrides](variables.tf#L79) | Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`. | object({…}) | | {} | ## Outputs diff --git a/modules/project-factory/factory-projects.tf b/modules/project-factory/factory-projects.tf index c29be097d..e33a5193a 100644 --- a/modules/project-factory/factory-projects.tf +++ b/modules/project-factory/factory-projects.tf @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -58,6 +58,37 @@ locals { try(v.contacts, null), var.data_defaults.contacts ) + factories_config = { + custom_roles = try( + coalesce( + var.data_overrides.factories_config.custom_roles, + try(v.factories_config.custom_roles, null), + var.data_defaults.factories_config.custom_roles + ), + null + ) + observability = try( + coalesce( + var.data_overrides.factories_config.observability, + try(v.factories_config.observability, null), + var.data_defaults.factories_config.observability + ), + null) + org_policies = try( + coalesce( + var.data_overrides.factories_config.org_policies, + try(v.factories_config.org_policies, null), + var.data_defaults.factories_config.org_policies + ), + null) + quotas = try( + coalesce( + var.data_overrides.factories_config.quotas, + try(v.factories_config.quotas, null), + var.data_defaults.factories_config.quotas + ), + null) + } labels = coalesce( try(v.labels, null), var.data_defaults.labels diff --git a/modules/project-factory/main.tf b/modules/project-factory/main.tf index c806a7593..f350503f3 100644 --- a/modules/project-factory/main.tf +++ b/modules/project-factory/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,6 +41,7 @@ module "projects" { local.context.folder_ids, each.value.parent, each.value.parent ) prefix = each.value.prefix + alerts = try(each.value.alerts, null) auto_create_network = try(each.value.auto_create_network, false) compute_metadata = try(each.value.compute_metadata, {}) # TODO: concat lists for each key @@ -49,6 +50,15 @@ module "projects" { ) default_service_account = try(each.value.default_service_account, "keep") descriptive_name = try(each.value.descriptive_name, null) + factories_config = { + custom_roles = each.value.factories_config.custom_roles + observability = each.value.factories_config.observability + org_policies = each.value.factories_config.org_policies + quotas = each.value.factories_config.quotas + context = { + notification_channels = var.factories_config.context.notification_channels + } + } iam = { for k, v in lookup(each.value, "iam", {}) : k => [ for vv in v : try( @@ -93,13 +103,16 @@ module "projects" { each.value.labels, var.data_merges.labels ) lien_reason = try(each.value.lien_reason, null) + log_scopes = try(each.value.log_scopes, null) logging_data_access = try(each.value.logging_data_access, {}) logging_exclusions = try(each.value.logging_exclusions, {}) + logging_metrics = try(each.value.logging_metrics, null) logging_sinks = try(each.value.logging_sinks, {}) metric_scopes = distinct(concat( each.value.metric_scopes, var.data_merges.metric_scopes )) - org_policies = each.value.org_policies + notification_channels = try(each.value.notification_channels, null) + org_policies = each.value.org_policies service_encryption_key_ids = merge( each.value.service_encryption_key_ids, var.data_merges.service_encryption_key_ids diff --git a/modules/project-factory/variables.tf b/modules/project-factory/variables.tf index bd364e1b4..bf21ae277 100644 --- a/modules/project-factory/variables.tf +++ b/modules/project-factory/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,8 +17,14 @@ variable "data_defaults" { description = "Optional default values used when corresponding project data from files are missing." type = object({ - billing_account = optional(string) - contacts = optional(map(list(string)), {}) + billing_account = optional(string) + contacts = optional(map(list(string)), {}) + factories_config = optional(object({ + custom_roles = optional(string) + observability = optional(string) + org_policies = optional(string) + quotas = optional(string) + }), {}) labels = optional(map(string), {}) metric_scopes = optional(list(string), []) parent = optional(string) @@ -73,8 +79,14 @@ variable "data_merges" { variable "data_overrides" { description = "Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`." type = object({ - billing_account = optional(string) - contacts = optional(map(list(string))) + billing_account = optional(string) + contacts = optional(map(list(string))) + factories_config = optional(object({ + custom_roles = optional(string) + observability = optional(string) + org_policies = optional(string) + quotas = optional(string) + }), {}) parent = optional(string) prefix = optional(string) service_encryption_key_ids = optional(map(list(string))) @@ -100,8 +112,6 @@ variable "data_overrides" { variable "factories_config" { description = "Path to folder with YAML resource description data files." type = object({ - folders_data_path = optional(string) - projects_data_path = optional(string) budgets = optional(object({ billing_account = string budgets_data_path = string @@ -110,11 +120,14 @@ variable "factories_config" { })) context = optional(object({ # TODO: add KMS keys - folder_ids = optional(map(string), {}) - iam_principals = optional(map(string), {}) - tag_values = optional(map(string), {}) - vpc_host_projects = optional(map(string), {}) + folder_ids = optional(map(string), {}) + iam_principals = optional(map(string), {}) + tag_values = optional(map(string), {}) + vpc_host_projects = optional(map(string), {}) + notification_channels = optional(map(string), {}) }), {}) + folders_data_path = optional(string) + projects_data_path = optional(string) }) nullable = false } diff --git a/modules/project/README.md b/modules/project/README.md index 44e45a096..2d8c9961a 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -30,6 +30,8 @@ This module implements the creation and management of one GCP project including - [VPC Service Controls](#vpc-service-controls) - [Project Related Outputs](#project-related-outputs) - [Managing project related configuration without creating it](#managing-project-related-configuration-without-creating-it) +- [Observability](#observability) +- [Observability factory](#observability-factory) - [Files](#files) - [Variables](#variables) - [Outputs](#outputs) @@ -1401,16 +1403,164 @@ module "bucket" { # tftest inventory=data.yaml e2e ``` +## Observability + +Alerting policies, log-based metrics, and notification channels are managed by the `alerts`, `logging_metrics`, and `notification_channels` variables, respectively. + +```hcl +module "project" { + source = "./fabric/modules/project" + name = "project" + billing_account = var.billing_account_id + parent = var.folder_id + prefix = var.prefix + alerts = { + alert-1 = { + display_name = "alert-1" + combiner = "OR" + notification_channels = [ + "my-channel", + "projects/other-project/notificationChannels/1234567890" + ] + conditions = [{ + display_name = "test condition" + condition_threshold = { + filter = "metric.type=\"compute.googleapis.com/instance/disk/write_bytes_count\"" + comparison = "COMPARISON_GT" + threshold_value = 100 + duration = "60s" + aggregations = { + alignment_period = "60s" + per_series_aligner = "ALIGN_RATE" + } + } + }] + } + } + logging_metrics = { + metric-1 = { + name = "metric-1" + filter = "resource.type=\"gce_instance\"" + description = "This is a metric" + metric_descriptor = { + metric_kind = "GAUGE" + value_type = "DOUBLE" + unit = "ms" + } + } + } + notification_channels = { + my-channel = { + display_name = "My Channel" + type = "email" + labels = { + email_address = "hello@example.com" + } + } + } +} +# tftest modules=1 resources=4 +``` + +# Observability factory + +Observability variables are exposed through a factory enabled by setting `var.factories_config.observability`. YAML files configure observability resources using top-level keys: `alerts`, `logging_metrics`, and `notification_channels`, which correspond to the respective variables. All top-level keys are optional, and their structure mirrors their corresponding variable's structure. + +```hcl +module "project" { + source = "./fabric/modules/project" + name = "project" + billing_account = var.billing_account_id + parent = var.folder_id + prefix = var.prefix + factories_config = { + observability = "data/observability" + context = { + notification_channels = { + common-channel = "projects/other-project/notificationChannels/1234567890" + } + } + } +} +# tftest modules=1 resources=5 files=observability +``` + +```yaml +# tftest-file id=observability path=data/observability/observability.yaml schema=observability.schema.json +logging_metrics: + factory-metric-1: + filter: "resource.type=gae_app AND severity>=ERROR" + metric_descriptor: + metric_kind: DELTA + value_type: INT64 + disabled: true + + factory-metric-2: + filter: resource.type=gae_app AND severity>=ERROR + metric_descriptor: + metric_kind: DELTA + value_type: DISTRIBUTION + unit: "1" + labels: + - key: mass + value_type: STRING + description: amount of matter + - key: sku + value_type: INT64 + description: Identifying number for item + display_name: My metric + value_extractor: EXTRACT(jsonPayload.request) + label_extractors: + mass: EXTRACT(jsonPayload.request) + sku: EXTRACT(jsonPayload.id) + bucket_options: + linear_buckets: + num_finite_buckets: 3 + width: 1 + offset: 1 + +notification_channels: + channel-1: + display_name: channel-1 + type: email + labels: + email_address: hello2@example.com + +alerts: + alert-1: + display_name: My Alert Policy + combiner: OR + notification_channels: + - channel-1 + - common-channel + conditions: + - display_name: test condition + condition_threshold: + filter: | + metric.type="compute.googleapis.com/instance/disk/write_bytes_count" AND resource.type="gce_instance" + duration: 60s + comparison: COMPARISON_GT + aggregations: + alignment_period: 60s + per_series_aligner: ALIGN_RATE + user_labels: + foo: bar +``` + + ## Files | name | description | resources | |---|---|---| +| [alerts.tf](./alerts.tf) | None | google_monitoring_alert_policy | | [cmek.tf](./cmek.tf) | Service Agent IAM Bindings for CMEK | google_kms_crypto_key_iam_member | | [iam.tf](./iam.tf) | IAM bindings. | google_project_iam_binding · google_project_iam_custom_role · google_project_iam_member | +| [logging-metrics.tf](./logging-metrics.tf) | None | google_logging_metric | | [logging.tf](./logging.tf) | Log sinks and supporting resources. | google_bigquery_dataset_iam_member · google_logging_log_scope · google_logging_project_exclusion · google_logging_project_sink · google_project_iam_audit_config · google_project_iam_member · google_pubsub_topic_iam_member · google_storage_bucket_iam_member | | [main.tf](./main.tf) | Module-level locals and resources. | google_compute_project_metadata_item · google_essential_contacts_contact · google_monitoring_monitored_project · google_project · google_project_service · google_resource_manager_lien | +| [notification-channels.tf](./notification-channels.tf) | None | google_monitoring_notification_channel | | [organization-policies.tf](./organization-policies.tf) | Project-level organization policies. | google_org_policy_policy | | [outputs.tf](./outputs.tf) | Module outputs. | | | [quotas.tf](./quotas.tf) | None | google_cloud_quotas_quota_preference | @@ -1429,7 +1579,8 @@ module "bucket" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L105) | Project name and id suffix. | string | ✓ | | +| [name](variables.tf#L109) | Project name and id suffix. | string | ✓ | | +| [alerts](variables-observability.tf#L17) | Monitoring alerts. | map(object({…})) | | {} | | [auto_create_network](variables.tf#L17) | Whether to create the default network for the project. | bool | | false | | [billing_account](variables.tf#L23) | Billing account id. | string | | null | | [compute_metadata](variables.tf#L29) | Optional compute metadata key/values. Only usable if compute API has been enabled. | map(string) | | {} | @@ -1438,53 +1589,58 @@ module "bucket" { | [default_service_account](variables.tf#L50) | Project default service account setting: can be one of `delete`, `deprivilege`, `disable`, or `keep`. | string | | "keep" | | [deletion_policy](variables.tf#L64) | Deletion policy setting for this project. | string | | "DELETE" | | [descriptive_name](variables.tf#L75) | Name of the project name. Used for project name instead of `name` variable. | string | | null | -| [factories_config](variables.tf#L81) | Paths to data files and folders that enable factory functionality. | object({…}) | | {} | +| [factories_config](variables.tf#L81) | Paths to data files and folders that enable factory functionality. | object({…}) | | {} | | [iam](variables-iam.tf#L17) | Authoritative IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_bindings](variables-iam.tf#L24) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | | [iam_bindings_additive](variables-iam.tf#L39) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | | [iam_by_principals](variables-iam.tf#L54) | Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid cycle errors. Merged internally with the `iam` variable. | map(list(string)) | | {} | -| [labels](variables.tf#L92) | Resource labels. | map(string) | | {} | -| [lien_reason](variables.tf#L99) | If non-empty, creates a project lien with this description. | string | | null | -| [log_scopes](variables-observability.tf#L39) | Log scopes under this project. | map(object({…})) | | {} | -| [logging_data_access](variables-observability.tf#L17) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | -| [logging_exclusions](variables-observability.tf#L32) | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string) | | {} | -| [logging_sinks](variables-observability.tf#L49) | Logging sinks to create for this project. | map(object({…})) | | {} | -| [metric_scopes](variables-observability.tf#L80) | List of projects that will act as metric scopes for this project. | list(string) | | [] | +| [labels](variables.tf#L96) | Resource labels. | map(string) | | {} | +| [lien_reason](variables.tf#L103) | If non-empty, creates a project lien with this description. | string | | null | +| [log_scopes](variables-observability.tf#L117) | Log scopes under this project. | map(object({…})) | | {} | +| [logging_data_access](variables-observability.tf#L127) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | +| [logging_exclusions](variables-observability.tf#L142) | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string) | | {} | +| [logging_metrics](variables-observability.tf#L149) | Log-based metrics. | map(object({…})) | | {} | +| [logging_sinks](variables-observability.tf#L189) | Logging sinks to create for this project. | map(object({…})) | | {} | +| [metric_scopes](variables-observability.tf#L220) | List of projects that will act as metric scopes for this project. | list(string) | | [] | | [network_tags](variables-tags.tf#L17) | Network tags by key name. If `id` is provided, key creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | -| [org_policies](variables.tf#L110) | Organization policies applied to this project keyed by policy name. | map(object({…})) | | {} | -| [parent](variables.tf#L137) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | | null | -| [prefix](variables.tf#L147) | Optional prefix used to generate project id and name. | string | | null | -| [project_create](variables.tf#L157) | Create project. When set to false, uses a data source to reference existing project. | bool | | true | +| [notification_channels](variables-observability.tf#L227) | Monitoring notification channels. | map(object({…})) | | {} | +| [org_policies](variables.tf#L114) | Organization policies applied to this project keyed by policy name. | map(object({…})) | | {} | +| [parent](variables.tf#L141) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | | null | +| [prefix](variables.tf#L151) | Optional prefix used to generate project id and name. | string | | null | +| [project_create](variables.tf#L161) | Create project. When set to false, uses a data source to reference existing project. | bool | | true | | [quotas](variables-quotas.tf#L17) | Service quota configuration. | map(object({…})) | | {} | -| [service_agents_config](variables.tf#L163) | Automatic service agent configuration options. | object({…}) | | {} | -| [service_config](variables.tf#L174) | Configure service API activation. | object({…}) | | {…} | -| [service_encryption_key_ids](variables.tf#L186) | Service Agents to be granted encryption/decryption permissions over Cloud KMS encryption keys. Format {SERVICE_AGENT => [KEY_ID]}. | map(list(string)) | | {} | -| [services](variables.tf#L193) | Service APIs to enable. | list(string) | | [] | -| [shared_vpc_host_config](variables.tf#L199) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | object({…}) | | null | -| [shared_vpc_service_config](variables.tf#L208) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | object({…}) | | {…} | -| [skip_delete](variables.tf#L236) | Deprecated. Use deletion_policy. | bool | | null | +| [service_agents_config](variables.tf#L167) | Automatic service agent configuration options. | object({…}) | | {} | +| [service_config](variables.tf#L178) | Configure service API activation. | object({…}) | | {…} | +| [service_encryption_key_ids](variables.tf#L190) | Service Agents to be granted encryption/decryption permissions over Cloud KMS encryption keys. Format {SERVICE_AGENT => [KEY_ID]}. | map(list(string)) | | {} | +| [services](variables.tf#L197) | Service APIs to enable. | list(string) | | [] | +| [shared_vpc_host_config](variables.tf#L203) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | object({…}) | | null | +| [shared_vpc_service_config](variables.tf#L212) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | object({…}) | | {…} | +| [skip_delete](variables.tf#L240) | Deprecated. Use deletion_policy. | bool | | null | | [tag_bindings](variables-tags.tf#L81) | Tag bindings for this project, in key => tag value id format. | map(string) | | null | | [tags](variables-tags.tf#L88) | Tags by key name. If `id` is provided, key or value creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | -| [vpc_sc](variables.tf#L248) | VPC-SC configuration for the project, use when `ignore_changes` for resources is set in the VPC-SC module. | object({…}) | | null | +| [vpc_sc](variables.tf#L252) | VPC-SC configuration for the project, use when `ignore_changes` for resources is set in the VPC-SC module. | object({…}) | | null | ## Outputs | name | description | sensitive | |---|---|:---:| -| [custom_role_id](outputs.tf#L17) | Map of custom role IDs created in the project. | | -| [custom_roles](outputs.tf#L27) | Map of custom roles resources created in the project. | | -| [default_service_accounts](outputs.tf#L33) | Emails of the default service accounts for this project. | | -| [id](outputs.tf#L41) | Project id. | | -| [name](outputs.tf#L59) | Project name. | | -| [network_tag_keys](outputs.tf#L71) | Tag key resources. | | -| [network_tag_values](outputs.tf#L80) | Tag value resources. | | -| [number](outputs.tf#L88) | Project number. | | -| [project_id](outputs.tf#L106) | Project id. | | -| [quota_configs](outputs.tf#L124) | Quota configurations. | | -| [quotas](outputs.tf#L135) | Quota resources. | | -| [service_agents](outputs.tf#L140) | List of all (active) service agents for this project. | | -| [services](outputs.tf#L149) | Service APIs to enabled in the project. | | -| [sink_writer_identities](outputs.tf#L158) | Writer identities created for each sink. | | -| [tag_keys](outputs.tf#L165) | Tag key resources. | | -| [tag_values](outputs.tf#L174) | Tag value resources. | | +| [alert_ids](outputs.tf#L17) | Monitoring alert IDs. | | +| [custom_role_id](outputs.tf#L25) | Map of custom role IDs created in the project. | | +| [custom_roles](outputs.tf#L35) | Map of custom roles resources created in the project. | | +| [default_service_accounts](outputs.tf#L40) | Emails of the default service accounts for this project. | | +| [id](outputs.tf#L48) | Project id. | | +| [name](outputs.tf#L66) | Project name. | | +| [network_tag_keys](outputs.tf#L78) | Tag key resources. | | +| [network_tag_values](outputs.tf#L87) | Tag value resources. | | +| [notification_channel_names](outputs.tf#L95) | Notification channel names. | | +| [notification_channels](outputs.tf#L103) | Full notification channel objects. | | +| [number](outputs.tf#L108) | Project number. | | +| [project_id](outputs.tf#L126) | Project id. | | +| [quota_configs](outputs.tf#L144) | Quota configurations. | | +| [quotas](outputs.tf#L155) | Quota resources. | | +| [service_agents](outputs.tf#L160) | List of all (active) service agents for this project. | | +| [services](outputs.tf#L169) | Service APIs to enabled in the project. | | +| [sink_writer_identities](outputs.tf#L178) | Writer identities created for each sink. | | +| [tag_keys](outputs.tf#L185) | Tag key resources. | | +| [tag_values](outputs.tf#L194) | Tag value resources. | | diff --git a/modules/project/alerts.tf b/modules/project/alerts.tf new file mode 100644 index 000000000..625cb189e --- /dev/null +++ b/modules/project/alerts.tf @@ -0,0 +1,285 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _alerts_factory_data_raw = merge([ + for k in local.observability_factory_data_raw : + lookup(k, "alerts", {}) + ]...) + _alerts_factory_data = { + for k, v in local._alerts_factory_data_raw : + k => { + combiner = v.combiner + display_name = try(v.display_name, null) + enabled = try(v.enabled, null) + notification_channels = try(v.notification_channels, []) + severity = try(v.severity, null) + user_labels = try(v.user_labels, null) + alert_strategy = !can(v.alert_strategy) ? null : { + auto_close = try(v.alert_strategy.auto_close, null) + notification_prompts = try(v.alert_strategy.notification_prompts, null) + notification_rate_limit = !can(v.alert_strategy.notification_rate_limit) ? null : { + period = try(v.alert_strategy.notification_rate_limit.period, null) + } + notification_channel_strategy = !can(v.alert_strategy.notification_channel_strategy) ? null : { + notification_channel_names = try(v.alert_strategy.notification_channel_strategy.notification_channel_names, null) + renotify_interval = try(v.alert_strategy.notification_channel_strategy.renotify_interval, null) + } + } + conditions = !can(v.conditions) ? null : [ + for c in v.conditions : { + display_name = c.display_name + condition_absent = !can(c.condition_absent) ? null : { + duration = c.condition_absent.duration + filter = try(c.condition_absent.filter, null) + aggregations = !can(c.condition_absent.aggregations) ? null : { + per_series_aligner = try(c.condition_absent.aggregations.per_series_aligner, null) + group_by_fields = try(c.condition_absent.aggregations.group_by_fields, null) + cross_series_reducer = try(c.condition_absent.aggregations.cross_series_reducer, null) + alignment_period = try(c.condition_absent.aggregations.alignment_period, null) + } + trigger = !can(c.condition_absent.trigger) ? null : { + count = try(c.condition_absent.trigger.count, null) + percent = try(c.condition_absent.trigger.percent, null) + } + } + condition_matched_log = !can(c.condition_matched_log) ? null : { + filter = c.condition_matched_log.filter + label_extractors = try(c.condition_matched_log.label_extractors, null) + } + condition_monitoring_query_language = !can(c.condition_monitoring_query_language) ? null : { + duration = c.condition_monitoring_query_language.duration + query = c.condition_monitoring_query_language.query + evaluation_missing_data = try(c.condition_monitoring_query_language.evaluation_missing_data, null) + trigger = !can(c.condition_monitoring_query_language.trigger) ? null : { + count = try(c.condition_monitoring_query_language.trigger.count, null) + percent = try(c.condition_monitoring_query_language.trigger.percent, null) + } + } + condition_prometheus_query_language = !can(c.condition_prometheus_query_language) ? null : { + query = c.condition_prometheus_query_language.query + alert_rule = try(c.condition_prometheus_query_language.alert_rule, null) + disable_metric_validation = try(c.condition_prometheus_query_language.disable_metric_validation, null) + duration = try(c.condition_prometheus_query_language.duration, null) + evaluation_interval = try(c.condition_prometheus_query_language.evaluation_interval, null) + labels = try(c.condition_prometheus_query_language.labels, null) + rule_group = try(c.condition_prometheus_query_language.rule_group, null) + } + condition_threshold = !can(c.condition_threshold) ? null : { + comparison = c.condition_threshold.comparison + duration = c.condition_threshold.duration + denominator_filter = try(c.condition_threshold.denominator_filter, null) + + + evaluation_missing_data = try(c.condition_threshold.evaluation_missing_data, null) + filter = try(c.condition_threshold.filter, null) + threshold_value = try(c.condition_threshold.threshold_value, null) + aggregations = !can(c.condition_threshold.aggregations) ? null : { + per_series_aligner = try(c.condition_threshold.aggregations.per_series_aligner, null) + group_by_fields = try(c.condition_threshold.aggregations.group_by_fields, null) + cross_series_reducer = try(c.condition_threshold.aggregations.cross_series_reducer, null) + alignment_period = try(c.condition_threshold.aggregations.alignment_period, null) + } + denominator_aggregations = !can(c.condition_threshold.denominator_aggregations) ? null : { + per_series_aligner = try(c.condition_threshold.denominator_aggregations.per_series_aligner, null) + group_by_fields = try(c.condition_threshold.denominator_aggregations.group_by_fields, null) + cross_series_reducer = try(c.condition_threshold.denominator_aggregations.cross_series_reducer, null) + alignment_period = try(c.condition_threshold.denominator_aggregations.alignment_period, null) + } + forecast_options = !can(c.condition_threshold.forecast_options) ? null : { + forecast_horizon = c.condition_threshold.forecast_options.forecast_horizon + } + trigger = !can(c.condition_threshold.trigger) ? null : { + count = try(c.condition_threshold.trigger.count, null) + percent = try(c.condition_threshold.trigger.percent, null) + } + } + } + ] + documentation = !can(v.documentation) ? null : { + content = try(v.documentation.content, null) + mime_type = try(v.documentation.mime_type, null) + subject = try(v.documentation.subject, null) + links = !can(v.documentation.links) ? null : [ + for l in v.documentation.link : { + display_name = try(l.display_name, null) + url = try(l.url, null) + }] + } + } + } + alerts = merge(local._alerts_factory_data, var.alerts) +} + +resource "google_monitoring_alert_policy" "alerts" { + for_each = local.alerts + project = local.project.project_id + + combiner = each.value.combiner + display_name = each.value.display_name + enabled = each.value.enabled + notification_channels = [ + for x in each.value.notification_channels : + try( + # first try to get a channel created by this module + google_monitoring_notification_channel.channels[x].name, + # otherwise check the context + var.factories_config.context.notification_channels[x], + # if nothing else, use the provided channel as is + x + ) + ] + severity = each.value.severity + user_labels = each.value.user_labels + + dynamic "alert_strategy" { + for_each = each.value.alert_strategy[*] + content { + auto_close = alert_strategy.value.auto_close + notification_prompts = alert_strategy.value.notification_prompts + dynamic "notification_channel_strategy" { + for_each = alert_strategy.value.notification_channel_strategy[*] + content { + notification_channel_names = notification_channel_strategy.value.notification_channel_names + renotify_interval = notification_channel_strategy.value.renotify_interval + } + } + dynamic "notification_rate_limit" { + for_each = alert_strategy.value.notification_rate_limit[*] + content { + period = notification_rate_limit.value.period + } + } + } + } + dynamic "conditions" { + for_each = each.value.conditions + content { + display_name = conditions.value.display_name + dynamic "condition_absent" { + for_each = conditions.value.condition_absent[*] + content { + duration = condition_absent.value.duration + filter = condition_absent.value.filter + dynamic "aggregations" { + for_each = condition_absent.value.aggregations[*] + content { + alignment_period = aggregations.value.alignment_period + cross_series_reducer = aggregations.value.cross_series_reducer + group_by_fields = aggregations.value.group_by_fields + per_series_aligner = aggregations.value.per_series_aligner + } + } + dynamic "trigger" { + for_each = condition_absent.value.trigger[*] + content { + count = trigger.value.count + percent = trigger.value.percent + } + } + } + } + dynamic "condition_matched_log" { + for_each = conditions.value.condition_matched_log[*] + content { + filter = condition_matched_log.value.filter + label_extractors = condition_matched_log.value.label_extractors + } + } + dynamic "condition_monitoring_query_language" { + for_each = conditions.value.condition_monitoring_query_language[*] + content { + duration = condition_monitoring_query_language.value.duration + query = condition_monitoring_query_language.value.query + evaluation_missing_data = condition_monitoring_query_language.value.evaluation_missing_data + trigger { + count = condition_monitoring_query_language.value.trigger.count + percent = condition_monitoring_query_language.value.trigger.percent + } + } + } + + dynamic "condition_prometheus_query_language" { + for_each = conditions.value.condition_prometheus_query_language[*] + content { + query = condition_prometheus_query_language.value.query + disable_metric_validation = condition_prometheus_query_language.value.disable_metric_validation + duration = condition_prometheus_query_language.value.duration + evaluation_interval = condition_prometheus_query_language.value.evaluation_interval + labels = condition_prometheus_query_language.value.labels + rule_group = condition_prometheus_query_language.value.rule_group + alert_rule = condition_prometheus_query_language.value.alert_rule + } + } + dynamic "condition_threshold" { + for_each = conditions.value.condition_threshold[*] + content { + comparison = condition_threshold.value.comparison + duration = condition_threshold.value.duration + denominator_filter = condition_threshold.value.denominator_filter + evaluation_missing_data = condition_threshold.value.evaluation_missing_data + filter = condition_threshold.value.filter + threshold_value = condition_threshold.value.threshold_value + dynamic "aggregations" { + for_each = condition_threshold.value.aggregations[*] + content { + alignment_period = aggregations.value.alignment_period + cross_series_reducer = aggregations.value.cross_series_reducer + group_by_fields = aggregations.value.group_by_fields + per_series_aligner = aggregations.value.per_series_aligner + } + } + dynamic "denominator_aggregations" { + for_each = condition_threshold.value.denominator_aggregations[*] + content { + alignment_period = denominator_aggregations.value.alignment_period + group_by_fields = denominator_aggregations.value.group_by_fields + per_series_aligner = denominator_aggregations.value.per_series_aligner + } + } + dynamic "forecast_options" { + for_each = condition_threshold.value.forecast_options[*] + content { + forecast_horizon = forecast_options.value.forecast_horizon + } + } + dynamic "trigger" { + for_each = condition_threshold.value.trigger[*] + content { + count = trigger.value.count + percent = trigger.value.percent + } + } + } + } + } + } + dynamic "documentation" { + for_each = each.value.documentation[*] + content { + content = documentation.value.content + mime_type = documentation.value.mime_type + subject = documentation.value.subject + dynamic "links" { + for_each = documentation.value.links[*] + content { + display_name = links.value.display_name + url = links.value.url + } + } + } + } +} diff --git a/modules/project/logging-metrics.tf b/modules/project/logging-metrics.tf new file mode 100644 index 000000000..f46558690 --- /dev/null +++ b/modules/project/logging-metrics.tf @@ -0,0 +1,120 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _logging_metrics_factory_data_raw = merge([ + for k in local.observability_factory_data_raw : + lookup(k, "logging_metrics", {}) + ]...) + _logging_metrics_factory_data = { + for k, v in local._logging_metrics_factory_data_raw : + k => { + filter = v.filter + bucket_name = try(v.bucket_name, null) + description = try(v.description, null) + disabled = try(v.disabled, null) + label_extractors = try(v.label_extractors, null) + value_extractor = try(v.value_extractor, null) + bucket_options = !can(v.bucket_options) ? null : { + explicit_buckets = !can(v.bucket_options.explicit_buckets) ? null : { + bounds = v.bucket_options.explicit_buckets.bounds + } + exponential_buckets = !can(v.bucket_options.exponential_buckets) ? null : { + num_finite_buckets = v.bucket_options.exponential_buckets.num_finite_buckets + growth_factor = v.bucket_options.exponential_buckets.growth_factor + scale = v.bucket_options.exponential_buckets.scale + } + linear_buckets = !can(v.bucket_options.linear_buckets) ? null : { + num_finite_buckets = v.bucket_options.linear_buckets.num_finite_buckets + width = v.bucket_options.linear_buckets.width + offset = v.bucket_options.linear_buckets.offset + } + } + metric_descriptor = !can(v.metric_descriptor) ? null : { + metric_kind = v.metric_descriptor.metric_kind + value_type = v.metric_descriptor.value_type + display_name = try(v.metric_descriptor.display_name, null) + unit = try(v.metric_descriptor.unit, null) + labels = !can(v.metric_descriptor.labels) ? [] : [ + for vv in v.metric_descriptor.labels : + { + key = vv.key + description = try(vv.description, null) + value_type = try(vv.value_type, null) + } + ] + } + } + } + metrics = merge(local._logging_metrics_factory_data, var.logging_metrics) +} + +resource "google_logging_metric" "metrics" { + for_each = local.metrics + project = local.project.project_id + name = each.key + filter = each.value.filter + description = each.value.description + disabled = each.value.disabled + bucket_name = each.value.bucket_name + value_extractor = each.value.value_extractor + label_extractors = each.value.label_extractors + + dynamic "bucket_options" { + for_each = each.value.bucket_options[*] + content { + dynamic "explicit_buckets" { + for_each = bucket_options.value.explicit_buckets[*] + content { + bounds = explicit_buckets.value.bounds + } + } + dynamic "exponential_buckets" { + for_each = bucket_options.value.exponential_buckets[*] + content { + num_finite_buckets = exponential_buckets.value.num_finite_buckets + growth_factor = exponential_buckets.value.growth_factor + scale = exponential_buckets.value.scale + } + } + dynamic "linear_buckets" { + for_each = bucket_options.value.linear_buckets[*] + content { + num_finite_buckets = linear_buckets.value.num_finite_buckets + width = linear_buckets.value.width + offset = linear_buckets.value.offset + } + } + } + } + dynamic "metric_descriptor" { + for_each = each.value.metric_descriptor[*] + content { + metric_kind = metric_descriptor.value.metric_kind + value_type = metric_descriptor.value.value_type + display_name = metric_descriptor.value.display_name + unit = metric_descriptor.value.unit + dynamic "labels" { + for_each = metric_descriptor.value.labels + content { + key = labels.value.key + description = labels.value.description + value_type = labels.value.value_type + } + } + } + } +} diff --git a/modules/project/main.tf b/modules/project/main.tf index a90361d1e..25d29fb0d 100644 --- a/modules/project/main.tf +++ b/modules/project/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,10 @@ locals { descriptive_name = ( var.descriptive_name != null ? var.descriptive_name : "${local.prefix}${var.name}" ) + observability_factory_data_raw = [ + for f in try(fileset(var.factories_config.observability, "*.yaml"), []) : + yamldecode(file("${var.factories_config.observability}/${f}")) + ] parent_type = var.parent == null ? null : split("/", var.parent)[0] parent_id = var.parent == null ? null : split("/", var.parent)[1] prefix = var.prefix == null ? "" : "${var.prefix}-" diff --git a/modules/project/notification-channels.tf b/modules/project/notification-channels.tf new file mode 100644 index 000000000..45d1678c6 --- /dev/null +++ b/modules/project/notification-channels.tf @@ -0,0 +1,60 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _channels_factory_data_raw = merge([ + for k in local.observability_factory_data_raw : + lookup(k, "notification_channels", {}) + ]...) + _channels_factory_data = { + for k, v in local._channels_factory_data_raw : + k => { + type = v.type + description = try(v.description, null) + display_name = try(v.display_name, null) + enabled = try(v.enabled, null) + labels = try(v.labels, null) + user_labels = try(v.user_labels, null) + sensitive_labels = !can(v.sensitive_labels) ? null : { + auth_token = try(v.sensitive_labels.auth_token, null) + password = try(v.sensitive_labels.password, null) + service_key = try(v.sensitive_labels.service_key, null) + } + } + } + channels = merge(local._channels_factory_data, var.notification_channels) +} + +resource "google_monitoring_notification_channel" "channels" { + for_each = local.channels + project = local.project.project_id + type = each.value.type + description = each.value.description + display_name = each.value.display_name + enabled = each.value.enabled + labels = each.value.labels + user_labels = each.value.user_labels + dynamic "sensitive_labels" { + for_each = each.value.sensitive_labels[*] + content { + auth_token = sensitive_labels.value.auth_token + password = sensitive_labels.value.password + service_key = sensitive_labels.value.service_key + } + } +} + + diff --git a/modules/project/outputs.tf b/modules/project/outputs.tf index 8a296ede5..60134a446 100644 --- a/modules/project/outputs.tf +++ b/modules/project/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,14 @@ * limitations under the License. */ +output "alert_ids" { + description = "Monitoring alert IDs." + value = { + for k, v in google_monitoring_alert_policy.alerts : + k => v.id + } +} + output "custom_role_id" { description = "Map of custom role IDs created in the project." value = { @@ -29,7 +37,6 @@ output "custom_roles" { value = google_project_iam_custom_role.roles } - output "default_service_accounts" { description = "Emails of the default service accounts for this project." value = { @@ -85,6 +92,19 @@ output "network_tag_values" { } } +output "notification_channel_names" { + description = "Notification channel names." + value = { + for k, v in google_monitoring_notification_channel.channels : + k => v.name + } +} + +output "notification_channels" { + description = "Full notification channel objects." + value = google_monitoring_notification_channel.channels +} + output "number" { description = "Project number." value = local.project.number diff --git a/modules/project/schemas/observability.schema.json b/modules/project/schemas/observability.schema.json new file mode 100644 index 000000000..cf3eb2f0a --- /dev/null +++ b/modules/project/schemas/observability.schema.json @@ -0,0 +1,514 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Observability Schema", + "type": "object", + "additionalProperties": false, + "properties": { + "alerts": { + "$ref": "#/$defs/alerts" + }, + "logging_metrics": { + "$ref": "#/$defs/logging_metrics" + }, + "notification_channels": { + "$ref": "#/$defs/notification_channels" + } + }, + "$defs": { + "alerts": { + "title": "Alerts", + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-zA-Z0-9-]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "combiner": { + "type": "string" + }, + "display_name": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "notification_channels": { + "type": "array", + "items": { + "type": "string" + } + }, + "severity": { + "type": "string" + }, + "user_labels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "alert_strategy": { + "type": "object", + "additionalProperties": false, + "properties": { + "auto_close": { + "type": "string" + }, + "notification_prompts": { + "type": "string" + }, + "notification_rate_limit": { + "type": "object", + "additionalProperties": false, + "properties": { + "period": { + "type": "string" + } + } + }, + "notification_channel_strategy": { + "type": "object", + "additionalProperties": false, + "properties": { + "notification_channel_names": { + "type": "array", + "items": { + "type": "string" + } + }, + "renotify_interval": { + "type": "string" + } + } + } + } + }, + "conditions": { + "type": "array", + "items": { + "$ref": "#/$defs/condition" + } + }, + "documentation": { + "type": "object", + "additionalProperties": false, + "properties": { + "content": { + "type": "string" + }, + "mime_type": { + "type": "string" + }, + "subject": { + "type": "string" + }, + "links": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "display_name": { + "type": "string" + }, + "url": { + "type": "string" + } + } + } + } + } + } + }, + "required": [ + "combiner" + ] + } + } + }, + "logging_metrics": { + "title": "Logging Metrics", + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-zA-Z0-9-]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "filter": { + "type": "string" + }, + "bucket_name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "disabled": { + "type": "boolean" + }, + "label_extractors": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "value_extractor": { + "type": "string" + }, + "bucket_options": { + "type": "object", + "additionalProperties": false, + "properties": { + "explicit_buckets": { + "type": "object", + "additionalProperties": false, + "properties": { + "bounds": { + "type": "array", + "items": { + "type": "number" + } + } + } + }, + "exponential_buckets": { + "type": "object", + "additionalProperties": false, + "properties": { + "num_finite_buckets": { + "type": "number" + }, + "growth_factor": { + "type": "number" + }, + "scale": { + "type": "number" + } + } + }, + "linear_buckets": { + "type": "object", + "additionalProperties": false, + "properties": { + "num_finite_buckets": { + "type": "number" + }, + "width": { + "type": "number" + }, + "offset": { + "type": "number" + } + } + } + } + }, + "metric_descriptor": { + "type": "object", + "additionalProperties": false, + "properties": { + "metric_kind": { + "type": "string" + }, + "value_type": { + "type": "string" + }, + "display_name": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "labels": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "key": { + "type": "string" + }, + "description": { + "type": "string" + }, + "value_type": { + "type": "string" + } + }, + "required": [ + "key" + ] + } + } + }, + "required": [ + "metric_kind", + "value_type" + ] + } + }, + "required": [ + "filter" + ] + } + } + }, + "notification_channels": { + "title": "Notification Channels", + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-zA-Z0-9-]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "display_name": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "labels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "user_labels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "sensitive_labels": { + "type": "object", + "additionalProperties": false, + "properties": { + "auth_token": { + "type": "string" + }, + "password": { + "type": "string" + }, + "service_key": { + "type": "string" + } + } + } + }, + "required": [ + "type" + ] + } + } + }, + "condition": { + "type": "object", + "additionalProperties": false, + "properties": { + "display_name": { + "type": "string" + }, + "condition_absent": { + "$ref": "#/$defs/absent_condition" + }, + "condition_matched_log": { + "$ref": "#/$defs/matched_log_condition" + }, + "condition_monitoring_query_language": { + "$ref": "#/$defs/monitoring_query_condition" + }, + "condition_prometheus_query_language": { + "$ref": "#/$defs/prometheus_query_condition" + }, + "condition_threshold": { + "$ref": "#/$defs/threshold_condition" + } + }, + "required": [ + "display_name" + ] + }, + "absent_condition": { + "type": "object", + "additionalProperties": false, + "properties": { + "duration": { + "type": "string" + }, + "filter": { + "type": "string" + }, + "aggregations": { + "$ref": "#/$defs/aggregations" + }, + "trigger": { + "$ref": "#/$defs/trigger" + } + }, + "required": [ + "duration" + ] + }, + "matched_log_condition": { + "type": "object", + "additionalProperties": false, + "properties": { + "filter": { + "type": "string" + }, + "label_extractors": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": [ + "filter" + ] + }, + "monitoring_query_condition": { + "type": "object", + "additionalProperties": false, + "properties": { + "duration": { + "type": "string" + }, + "query": { + "type": "string" + }, + "evaluation_missing_data": { + "type": "string" + }, + "trigger": { + "$ref": "#/$defs/trigger" + } + }, + "required": [ + "duration", + "query" + ] + }, + "prometheus_query_condition": { + "type": "object", + "additionalProperties": false, + "properties": { + "query": { + "type": "string" + }, + "alert_rule": { + "type": "string" + }, + "disable_metric_validation": { + "type": "boolean" + }, + "duration": { + "type": "string" + }, + "evaluation_interval": { + "type": "string" + }, + "labels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "rule_group": { + "type": "string" + } + }, + "required": [ + "query" + ] + }, + "threshold_condition": { + "type": "object", + "additionalProperties": false, + "properties": { + "comparison": { + "type": "string" + }, + "duration": { + "type": "string" + }, + "denominator_filter": { + "type": "string" + }, + "evaluation_missing_data": { + "type": "string" + }, + "filter": { + "type": "string" + }, + "threshold_value": { + "type": "number" + }, + "aggregations": { + "$ref": "#/$defs/aggregations" + }, + "denominator_aggregations": { + "$ref": "#/$defs/aggregations" + }, + "forecast_options": { + "type": "object", + "additionalProperties": false, + "properties": { + "forecast_horizon": { + "type": "string" + } + } + }, + "trigger": { + "$ref": "#/$defs/trigger" + } + }, + "required": [ + "comparison", + "duration" + ] + }, + "aggregations": { + "type": "object", + "additionalProperties": false, + "properties": { + "per_series_aligner": { + "type": "string" + }, + "group_by_fields": { + "type": "array", + "items": { + "type": "string" + } + }, + "cross_series_reducer": { + "type": "string" + }, + "alignment_period": { + "type": "string" + } + } + }, + "trigger": { + "type": "object", + "additionalProperties": false, + "properties": { + "count": { + "type": "number" + }, + "percent": { + "type": "number" + } + } + } + } +} diff --git a/modules/project/service-agents.yaml b/modules/project/service-agents.yaml index 8590aafb9..e7815218c 100644 --- a/modules/project/service-agents.yaml +++ b/modules/project/service-agents.yaml @@ -787,6 +787,13 @@ role: null is_primary: false aliases: [] +- name: gcp-ri-contactcenterinsights + display_name: Contact Center Insights Resource Identity (prod) + api: contactcenterinsights.googleapis.com + identity: service-%s@gcp-ri-contactcenterinsights.iam.gserviceaccount.com + role: null + is_primary: false + aliases: [] - name: container-analysis display_name: Container Analysis Service Agent api: containeranalysis.googleapis.com @@ -969,6 +976,13 @@ role: roles/firebaseapphosting.serviceAgent is_primary: true aliases: [] +- name: crashlytics + display_name: Firebase Crashlytics Service Agent + api: firebasecrashlytics.googleapis.com + identity: service-%s@gcp-sa-crashlytics.iam.gserviceaccount.com + role: roles/firebasecrashlytics.serviceAgent + is_primary: true + aliases: [] - name: firebasedataconnect display_name: Firebase Data Connect Service Account api: firebasedataconnect.googleapis.com @@ -1173,13 +1187,6 @@ role: null is_primary: false aliases: [] -- name: chronicle-spanner - display_name: Internal Chronicle Spanner Service Account - api: chronicle.googleapis.com - identity: service-%s@gcp-sa-chronicle-spanner.iam.gserviceaccount.com - role: null - is_primary: false - aliases: [] - name: fs-spanner display_name: Internal Cloud Firestore Spanner Service Agent api: firestore.googleapis.com @@ -1351,6 +1358,13 @@ role: null is_primary: true aliases: [] +- name: progrollout + display_name: Progressive Rollout Service Agent + api: progressiverollout.googleapis.com + identity: service-%s@gcp-sa-progrollout.iam.gserviceaccount.com + role: roles/progressiverollout.serviceAgent + is_primary: true + aliases: [] - name: pubsublite display_name: Pub/Sub Lite Service Account api: pubsublite.googleapis.com @@ -1572,7 +1586,7 @@ display_name: Vertex AI Online Prediction Service Agent api: aiplatform.googleapis.com identity: service-%s@gcp-sa-vertex-op.iam.gserviceaccount.com - role: null + role: roles/aiplatform.onlinePredictionServiceAgent is_primary: false aliases: [] - name: vertex-tune diff --git a/modules/project/variables-observability.tf b/modules/project/variables-observability.tf index 4b45b6ef4..56020d70a 100644 --- a/modules/project/variables-observability.tf +++ b/modules/project/variables-observability.tf @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,116 @@ * limitations under the License. */ +variable "alerts" { + description = "Monitoring alerts." + type = map(object({ + combiner = string + display_name = optional(string) + enabled = optional(bool) + notification_channels = optional(list(string), []) + severity = optional(string) + user_labels = optional(map(string)) + alert_strategy = optional(object({ + auto_close = optional(string) + notification_prompts = optional(string) + notification_rate_limit = optional(object({ + period = optional(string) + })) + notification_channel_strategy = optional(object({ + notification_channel_names = optional(list(string)) + renotify_interval = optional(string) + })) + })) + conditions = optional(list(object({ + display_name = string + condition_absent = optional(object({ + duration = string + filter = optional(string) + aggregations = optional(object({ + per_series_aligner = optional(string) + group_by_fields = optional(list(string)) + cross_series_reducer = optional(string) + alignment_period = optional(string) + })) + trigger = optional(object({ + count = optional(number) + percent = optional(number) + })) + })) + condition_matched_log = optional(object({ + filter = string + label_extractors = optional(map(string)) + })) + condition_monitoring_query_language = optional(object({ + duration = string + query = string + evaluation_missing_data = optional(string) + trigger = optional(object({ + count = optional(number) + percent = optional(number) + })) + })) + condition_prometheus_query_language = optional(object({ + query = string + alert_rule = optional(string) + disable_metric_validation = optional(bool) + duration = optional(string) + evaluation_interval = optional(string) + labels = optional(map(string)) + rule_group = optional(string) + })) + condition_threshold = optional(object({ + comparison = string + duration = string + denominator_filter = optional(string) + evaluation_missing_data = optional(string) + filter = optional(string) + threshold_value = optional(number) + aggregations = optional(object({ + per_series_aligner = optional(string) + group_by_fields = optional(list(string)) + cross_series_reducer = optional(string) + alignment_period = optional(string) + })) + denominator_aggregations = optional(object({ + per_series_aligner = optional(string) + group_by_fields = optional(list(string)) + cross_series_reducer = optional(string) + alignment_period = optional(string) + })) + forecast_options = optional(object({ + forecast_horizon = string + })) + trigger = optional(object({ + count = optional(number) + percent = optional(number) + })) + })) + })), []) + documentation = optional(object({ + content = optional(string) + mime_type = optional(string) + subject = optional(string) + links = optional(list(object({ + display_name = optional(string) + url = optional(string) + }))) + })) + })) + nullable = false + default = {} +} + +variable "log_scopes" { + description = "Log scopes under this project." + type = map(object({ + description = optional(string) + resource_names = list(string) + })) + nullable = false + default = {} +} + variable "logging_data_access" { description = "Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services." type = map(map(list(string))) @@ -36,11 +146,41 @@ variable "logging_exclusions" { nullable = false } -variable "log_scopes" { - description = "Log scopes under this project." +variable "logging_metrics" { + description = "Log-based metrics." type = map(object({ - description = optional(string) - resource_names = list(string) + filter = string + bucket_name = optional(string) + description = optional(string) + disabled = optional(bool) + label_extractors = optional(map(string)) + value_extractor = optional(string) + bucket_options = optional(object({ + explicit_buckets = optional(object({ + bounds = list(number) + })) + exponential_buckets = optional(object({ + num_finite_buckets = number + growth_factor = number + scale = number + })) + linear_buckets = optional(object({ + num_finite_buckets = number + width = number + offset = number + })) + })) + metric_descriptor = optional(object({ + metric_kind = string + value_type = string + display_name = optional(string) + unit = optional(string) + labels = optional(list(object({ + key = string + description = optional(string) + value_type = optional(string) + })), []) + })) })) nullable = false default = {} @@ -83,3 +223,22 @@ variable "metric_scopes" { default = [] nullable = false } + +variable "notification_channels" { + description = "Monitoring notification channels." + type = map(object({ + type = string + description = optional(string) + display_name = optional(string) + enabled = optional(bool) + labels = optional(map(string)) + user_labels = optional(map(string)) + sensitive_labels = optional(object({ + auth_token = optional(string) + password = optional(string) + service_key = optional(string) + })) + })) + nullable = false + default = {} +} diff --git a/modules/project/variables.tf b/modules/project/variables.tf index 245c24c55..31ee76ff9 100644 --- a/modules/project/variables.tf +++ b/modules/project/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -81,9 +81,13 @@ variable "descriptive_name" { variable "factories_config" { description = "Paths to data files and folders that enable factory functionality." type = object({ - custom_roles = optional(string) - org_policies = optional(string) - quotas = optional(string) + custom_roles = optional(string) + observability = optional(string) + org_policies = optional(string) + quotas = optional(string) + context = optional(object({ + notification_channels = optional(map(string), {}) + }), {}) }) nullable = false default = {} diff --git a/tests/examples_e2e/setup_module/jit.tf.json b/tests/examples_e2e/setup_module/jit.tf.json new file mode 100644 index 000000000..d2a9c8ced --- /dev/null +++ b/tests/examples_e2e/setup_module/jit.tf.json @@ -0,0 +1,28 @@ +{ + "locals": { + "jit_services": { + "alloydb.googleapis.com": "roles/alloydb.serviceAgent", + "apigee.googleapis.com": "roles/apigee.serviceAgent", + "artifactregistry.googleapis.com": "roles/artifactregistry.serviceAgent", + "assuredworkloads.googleapis.com": "roles/assuredworkloads.serviceAgent", + "dns.googleapis.com": "roles/dns.serviceAgent", + "dataplex.googleapis.com": "roles/dataplex.serviceAgent", + "pubsub.googleapis.com": "roles/pubsub.serviceAgent", + "sqladmin.googleapis.com": "roles/cloudsql.serviceAgent", + "dataform.googleapis.com": "roles/dataform.serviceAgent", + "eventarc.googleapis.com": "roles/eventarc.serviceAgent", + "cloudkms.googleapis.com": null, + "dataproc.googleapis.com": "roles/dataproc.serviceAgent", + "cloudfunctions.googleapis.com": "roles/cloudfunctions.serviceAgent", + "run.googleapis.com": "roles/run.serviceAgent", + "iap.googleapis.com": null, + "container.googleapis.com": "roles/container.serviceAgent", + "looker.googleapis.com": "roles/looker.serviceAgent", + "monitoring.googleapis.com": "roles/monitoring.notificationServiceAgent", + "networkconnectivity.googleapis.com": "roles/networkconnectivity.serviceAgent", + "secretmanager.googleapis.com": null, + "vpcaccess.googleapis.com": "roles/vpcaccess.serviceAgent", + "servicenetworking.googleapis.com": "roles/servicenetworking.serviceAgent" + } + } +} diff --git a/tests/examples_e2e/setup_module/main.tf b/tests/examples_e2e/setup_module/main.tf index 82b59b840..47db8dfb3 100644 --- a/tests/examples_e2e/setup_module/main.tf +++ b/tests/examples_e2e/setup_module/main.tf @@ -14,15 +14,13 @@ locals { prefix = "${var.prefix}-${var.timestamp}${var.suffix}" - jit_services = [ - "alloydb.googleapis.com", # no permissions granted by default - "artifactregistry.googleapis.com", # roles/artifactregistry.serviceAgent - "pubsub.googleapis.com", # roles/pubsub.serviceAgent - "storage.googleapis.com", # no permissions granted by default - "sqladmin.googleapis.com", # roles/cloudsql.serviceAgent - ] services = [ - # trimmed down list of services, to be extended as needed + # trimmed down list of services, to be extended as needed. If you + # update this list, make sure to update E2E_SERVICES in + # tools/built_service_agents.py and run: + # + # python tools/build_service_agents.py --e2e > tests/examples_e2e/setup_module/jit.tf.json + # "alloydb.googleapis.com", "analyticshub.googleapis.com", "apigee.googleapis.com", @@ -212,6 +210,7 @@ resource "google_service_networking_connection" "psa_connection" { service = "servicenetworking.googleapis.com" reserved_peering_ranges = [google_compute_global_address.psa_ranges.name] deletion_policy = "ABANDON" + depends_on = [google_project_iam_binding.agents] } ### END OF PSA @@ -223,33 +222,24 @@ resource "google_service_account" "service_account" { } resource "google_project_service_identity" "jit_si" { - for_each = toset(local.jit_services) + for_each = local.jit_services provider = google-beta project = google_project.project.project_id - service = each.value + service = each.key depends_on = [google_project_service.project_service] } -resource "google_project_iam_binding" "cloudsql_agent" { - members = ["serviceAccount:service-${google_project.project.number}@gcp-sa-cloud-sql.iam.gserviceaccount.com"] - project = google_project.project.project_id - role = "roles/cloudsql.serviceAgent" - depends_on = [google_project_service_identity.jit_si] +resource "google_project_iam_binding" "agents" { + for_each = { + for k, v in local.jit_services : k => v if v != null + } + members = [ + google_project_service_identity.jit_si[each.key].member + ] + project = google_project.project.project_id + role = each.value } -resource "google_project_iam_binding" "artifactregistry_agent" { - members = ["serviceAccount:service-${google_project.project.number}@gcp-sa-artifactregistry.iam.gserviceaccount.com"] - project = google_project.project.project_id - role = "roles/artifactregistry.serviceAgent" - depends_on = [google_project_service_identity.jit_si] -} - -resource "google_project_iam_binding" "pubsub_agent" { - members = ["serviceAccount:service-${google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com"] - project = google_project.project.project_id - role = "roles/pubsub.serviceAgent" - depends_on = [google_project_service_identity.jit_si] -} resource "local_file" "terraform_tfvars" { filename = "e2e_tests.tfvars" diff --git a/tools/build_service_agents.py b/tools/build_service_agents.py index c8dc0690d..65e22d28b 100755 --- a/tools/build_service_agents.py +++ b/tools/build_service_agents.py @@ -17,6 +17,8 @@ from dataclasses import asdict, dataclass from itertools import chain +import click +import json import requests import yaml from bs4 import BeautifulSoup @@ -41,6 +43,42 @@ ALIASES = { 'serverless-robot-prod': ['cloudrun', 'run'], } +E2E_SERVICES = [ + "alloydb.googleapis.com", + "analyticshub.googleapis.com", + "apigee.googleapis.com", + "artifactregistry.googleapis.com", + "assuredworkloads.googleapis.com", + "bigquery.googleapis.com", + "cloudbuild.googleapis.com", + "cloudfunctions.googleapis.com", + "cloudkms.googleapis.com", + "cloudresourcemanager.googleapis.com", + "compute.googleapis.com", + "container.googleapis.com", + "dataform.googleapis.com", + "dataplex.googleapis.com", + "dataproc.googleapis.com", + "dns.googleapis.com", + "eventarc.googleapis.com", + "iam.googleapis.com", + "iap.googleapis.com", + "logging.googleapis.com", + "looker.googleapis.com", + "monitoring.googleapis.com", + "networkconnectivity.googleapis.com", + "pubsub.googleapis.com", + "run.googleapis.com", + "secretmanager.googleapis.com", + "servicenetworking.googleapis.com", + "serviceusage.googleapis.com", + "sqladmin.googleapis.com", + "stackdriver.googleapis.com", + "storage-component.googleapis.com", + "storage.googleapis.com", + "vpcaccess.googleapis.com", +] + PRIMARY_OVERRIDE = { 'storage-transfer-service': True, } @@ -57,7 +95,9 @@ class Agent: aliases: list[str] -def main(): +@click.command() +@click.option('--e2e', is_flag=True, default=False) +def main(e2e=False): page = requests.get(SERVICE_AGENTS_URL).content soup = BeautifulSoup(page, 'html.parser') agents = [] @@ -115,11 +155,19 @@ def main(): aliases = set(chain.from_iterable(agent.aliases for agent in agents)) assert aliases.isdisjoint(names) - # take the header from the first lines of this file - header = open(__file__).readlines()[2:15] - print("".join(header)) - # and print all the agents - print(yaml.safe_dump([asdict(a) for a in agents], sort_keys=False)) + if not e2e: + # take the header from the first lines of this file + header = open(__file__).readlines()[2:15] + print("".join(header)) + # and print all the agents + print(yaml.safe_dump([asdict(a) for a in agents], sort_keys=False)) + else: + jit_services = {} + result = {"locals": {"jit_services": jit_services}} + for a in agents: + if a.is_primary and a.api in E2E_SERVICES: + jit_services[a.api] = a.role + print(json.dumps(result, indent=2)) if __name__ == '__main__':