From 2390124877584509076e7a316f10b98d301cc6c5 Mon Sep 17 00:00:00 2001 From: jacklever-hub24 Date: Tue, 8 Apr 2025 22:31:33 +1000 Subject: [PATCH 1/6] =?UTF-8?q?Added=20variable=20for=20activating=20nat?= =?UTF-8?q?=20and=20implementation=20in=20google=5Fapigee=E2=80=A6=20(#299?= =?UTF-8?q?9)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added variable for activating nat and implementation in google_apigee_net_address resource * Regenerated readme * Added tests for nat & activation * Removed unnecessary coalesce * Fixed test params * Added test to confirm activate is working --- modules/apigee/README.md | 6 +-- modules/apigee/main.tf | 1 + modules/apigee/variables.tf | 1 + .../apigee/test_apigee_nat_activate.tfvars | 24 ++++++++++ .../apigee/test_apigee_nat_activate.yaml | 45 +++++++++++++++++++ tests/modules/apigee/tftest.yaml | 1 + 6 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 tests/modules/apigee/test_apigee_nat_activate.tfvars create mode 100644 tests/modules/apigee/test_apigee_nat_activate.yaml diff --git a/modules/apigee/README.md b/modules/apigee/README.md index a4be05599..d209a6a81 100644 --- a/modules/apigee/README.md +++ b/modules/apigee/README.md @@ -359,13 +359,13 @@ module "apigee" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [project_id](variables.tf#L131) | Project ID. | string | ✓ | | +| [project_id](variables.tf#L132) | Project ID. | string | ✓ | | | [addons_config](variables.tf#L17) | Addons configuration. | object({…}) | | null | | [endpoint_attachments](variables.tf#L29) | Endpoint attachments. | map(object({…})) | | {} | | [envgroups](variables.tf#L39) | Environment groups (NAME => [HOSTNAMES]). | map(list(string)) | | {} | | [environments](variables.tf#L46) | Environments. | map(object({…})) | | {} | -| [instances](variables.tf#L74) | Instances ([REGION] => [INSTANCE]). | map(object({…})) | | {} | -| [organization](variables.tf#L99) | Apigee organization. If set to null the organization must already exist. | object({…}) | | null | +| [instances](variables.tf#L74) | Instances ([REGION] => [INSTANCE]). | map(object({…})) | | {} | +| [organization](variables.tf#L100) | Apigee organization. If set to null the organization must already exist. | object({…}) | | null | ## Outputs diff --git a/modules/apigee/main.tf b/modules/apigee/main.tf index 480d19feb..0c8f6bcfa 100644 --- a/modules/apigee/main.tf +++ b/modules/apigee/main.tf @@ -113,6 +113,7 @@ resource "google_apigee_nat_address" "apigee_nat" { } name = each.key instance_id = each.value + activate = var.instances[each.key].activate_nat } resource "google_apigee_instance_attachment" "instance_attachments" { diff --git a/modules/apigee/variables.tf b/modules/apigee/variables.tf index e8d1ffb66..b7f9aa6d8 100644 --- a/modules/apigee/variables.tf +++ b/modules/apigee/variables.tf @@ -79,6 +79,7 @@ variable "instances" { disk_encryption_key = optional(string) display_name = optional(string) enable_nat = optional(bool, false) + activate_nat = optional(bool, false) environments = optional(list(string), []) name = optional(string) runtime_ip_cidr_range = optional(string) diff --git a/tests/modules/apigee/test_apigee_nat_activate.tfvars b/tests/modules/apigee/test_apigee_nat_activate.tfvars new file mode 100644 index 000000000..d313605c3 --- /dev/null +++ b/tests/modules/apigee/test_apigee_nat_activate.tfvars @@ -0,0 +1,24 @@ +project_id = "my-project" +organization = { + display_name = "My Organization" + description = "My Organization" + runtime_type = "CLOUD" + billing_type = "Pay-as-you-go" + database_encryption_key = "123456789" + analytics_region = "europe-west1" + disable_vpc_peering = true +} +environments = { + apis-test = { + display_name = "APIs test" + description = "APIs Test" + envgroups = ["test"] + } +} +instances = { + europe-west1 = { + environments = ["europe-west1"] + enable_nat = true + activate_nat = true + } +} \ No newline at end of file diff --git a/tests/modules/apigee/test_apigee_nat_activate.yaml b/tests/modules/apigee/test_apigee_nat_activate.yaml new file mode 100644 index 000000000..3bf65d5b7 --- /dev/null +++ b/tests/modules/apigee/test_apigee_nat_activate.yaml @@ -0,0 +1,45 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + google_apigee_environment.environments["apis-test"]: + description: APIs Test + display_name: APIs test + name: apis-test + google_apigee_instance.instances["europe-west1"]: + description: Terraform-managed + disk_encryption_key_name: null + display_name: null + location: europe-west1 + name: instance-europe-west1 + google_apigee_organization.organization[0]: + analytics_region: europe-west1 + authorized_network: null + billing_type: Pay-as-you-go + description: null + display_name: null + project_id: my-project + retention: DELETION_RETENTION_UNSPECIFIED + runtime_database_encryption_key_name: '123456789' + runtime_type: CLOUD + disable_vpc_peering: true + google_apigee_nat_address.apigee_nat["europe-west1"]: + activate: true + +counts: + google_apigee_environment: 1 + google_apigee_instance: 1 + google_apigee_instance_attachment: 1 + google_apigee_organization: 1 + google_apigee_nat_address: 1 \ No newline at end of file diff --git a/tests/modules/apigee/tftest.yaml b/tests/modules/apigee/tftest.yaml index 6449de757..8452a0b89 100644 --- a/tests/modules/apigee/tftest.yaml +++ b/tests/modules/apigee/tftest.yaml @@ -28,3 +28,4 @@ tests: organization_only_psc_mode: organization_only_vpc_mode: organization_retention: + test_apigee_nat_activate: From f7a0958e17d187f61e781ac77b96585b6f7492d7 Mon Sep 17 00:00:00 2001 From: Luca Prete Date: Wed, 9 Apr 2025 16:39:20 +0200 Subject: [PATCH 2/6] [FAST] Remove object creator permission from storage viewer custom role (#3020) --- fast/stages/0-bootstrap/data/custom-roles/storage_viewer.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/fast/stages/0-bootstrap/data/custom-roles/storage_viewer.yaml b/fast/stages/0-bootstrap/data/custom-roles/storage_viewer.yaml index a80786e69..59522313d 100644 --- a/fast/stages/0-bootstrap/data/custom-roles/storage_viewer.yaml +++ b/fast/stages/0-bootstrap/data/custom-roles/storage_viewer.yaml @@ -28,7 +28,6 @@ includedPermissions: - storage.managedFolders.list - storage.multipartUploads.list - storage.multipartUploads.listParts - - storage.objects.create - storage.objects.get - storage.objects.getIamPolicy - storage.objects.list From fc84c4f60de339b8d22d597f14ec68ed8105b475 Mon Sep 17 00:00:00 2001 From: Simone Ruffilli Date: Thu, 10 Apr 2025 11:44:39 +0200 Subject: [PATCH 3/6] Module: net-vpc-factory (#2982) This pull request introduces the `net-vpc-factory` module. This new factory handles: * Project setup (most of what's supported by the `project` module) * VPC setup * Routing * Subnets * Connectivity options * NCC (hub, VPC spokes and VPN hybrid spokes) * Peerings * VPN (GCP-to-onprem and GCP-to-GCP) * NAT (everything supported by the `net-cloudnat` module) * DNS (everything supported by the `dns` module) * Firewall (everything supported by the `net-vpc-firewall` module) --- README.md | 2 +- blueprints/factories/README.md | 2 + modules/README.md | 1 + modules/net-vpc-factory/README.md | 831 +++++++++ .../firewall/dev-spoke/default-ingress.yaml | 28 + .../data/firewall/hub/default-ingress.yaml | 28 + .../firewall/prod-spoke/default-ingress.yaml | 28 + .../data/subnets/dev-spoke/dev-default.yaml | 8 + .../data/subnets/hub/hub-default.yaml | 8 + .../data/subnets/prod-spoke/prod-default.yaml | 8 + modules/net-vpc-factory/factory-dns.tf | 87 + modules/net-vpc-factory/factory-nat.tf | 62 + modules/net-vpc-factory/factory-ncc.tf | 139 ++ modules/net-vpc-factory/factory-peering.tf | 51 + modules/net-vpc-factory/factory-project.tf | 53 + modules/net-vpc-factory/factory-vpc.tf | 80 + modules/net-vpc-factory/factory-vpn.tf | 111 ++ modules/net-vpc-factory/main.tf | 38 + modules/net-vpc-factory/outputs.tf | 64 + .../recipes/hub-and-spoke-ncc/net-dev-01.yaml | 58 + .../hub-and-spoke-ncc/net-land-01.yaml | 109 ++ .../hub-and-spoke-ncc/net-prod-01.yaml | 58 + .../hub-and-spoke-peering/net-dev-01.yaml | 58 + .../hub-and-spoke-peering/net-land-01.yaml | 113 ++ .../hub-and-spoke-peering/net-prod-01.yaml | 58 + .../recipes/hub-and-spoke-vpn/net-dev-01.yaml | 83 + .../hub-and-spoke-vpn/net-land-01.yaml | 153 ++ .../hub-and-spoke-vpn/net-prod-01.yaml | 83 + .../recipes/only-projects/net-dev-01.yaml | 28 + .../recipes/only-projects/net-land-01.yaml | 30 + .../recipes/only-projects/net-prod-01.yaml | 28 + .../recipes/separate-envs/net-dev-01.yaml | 72 + .../recipes/separate-envs/net-land-01.yaml | 30 + .../recipes/separate-envs/net-prod-01.yaml | 72 + .../schemas/network-project.schema.json | 1479 +++++++++++++++++ modules/net-vpc-factory/variables.tf | 382 +++++ tests/modules/net_vpc_factory/common.tfvars | 3 + tests/modules/net_vpc_factory/ncc.tfvars | 2 + tests/modules/net_vpc_factory/ncc.yaml | 40 + .../net_vpc_factory/only_projects.tfvars | 2 + .../net_vpc_factory/only_projects.yaml | 21 + tests/modules/net_vpc_factory/peering.tfvars | 2 + tests/modules/net_vpc_factory/peering.yaml | 38 + .../net_vpc_factory/separate_envs.tfvars | 2 + .../net_vpc_factory/separate_envs.yaml | 35 + tests/modules/net_vpc_factory/tftest.yaml | 23 + tests/modules/net_vpc_factory/vpn.tfvars | 1 + tests/modules/net_vpc_factory/vpn.yaml | 37 + 48 files changed, 4728 insertions(+), 1 deletion(-) create mode 100644 modules/net-vpc-factory/README.md create mode 100644 modules/net-vpc-factory/data/firewall/dev-spoke/default-ingress.yaml create mode 100644 modules/net-vpc-factory/data/firewall/hub/default-ingress.yaml create mode 100644 modules/net-vpc-factory/data/firewall/prod-spoke/default-ingress.yaml create mode 100644 modules/net-vpc-factory/data/subnets/dev-spoke/dev-default.yaml create mode 100644 modules/net-vpc-factory/data/subnets/hub/hub-default.yaml create mode 100644 modules/net-vpc-factory/data/subnets/prod-spoke/prod-default.yaml create mode 100644 modules/net-vpc-factory/factory-dns.tf create mode 100644 modules/net-vpc-factory/factory-nat.tf create mode 100644 modules/net-vpc-factory/factory-ncc.tf create mode 100644 modules/net-vpc-factory/factory-peering.tf create mode 100644 modules/net-vpc-factory/factory-project.tf create mode 100644 modules/net-vpc-factory/factory-vpc.tf create mode 100644 modules/net-vpc-factory/factory-vpn.tf create mode 100644 modules/net-vpc-factory/main.tf create mode 100644 modules/net-vpc-factory/outputs.tf create mode 100644 modules/net-vpc-factory/recipes/hub-and-spoke-ncc/net-dev-01.yaml create mode 100644 modules/net-vpc-factory/recipes/hub-and-spoke-ncc/net-land-01.yaml create mode 100644 modules/net-vpc-factory/recipes/hub-and-spoke-ncc/net-prod-01.yaml create mode 100644 modules/net-vpc-factory/recipes/hub-and-spoke-peering/net-dev-01.yaml create mode 100644 modules/net-vpc-factory/recipes/hub-and-spoke-peering/net-land-01.yaml create mode 100644 modules/net-vpc-factory/recipes/hub-and-spoke-peering/net-prod-01.yaml create mode 100644 modules/net-vpc-factory/recipes/hub-and-spoke-vpn/net-dev-01.yaml create mode 100644 modules/net-vpc-factory/recipes/hub-and-spoke-vpn/net-land-01.yaml create mode 100644 modules/net-vpc-factory/recipes/hub-and-spoke-vpn/net-prod-01.yaml create mode 100644 modules/net-vpc-factory/recipes/only-projects/net-dev-01.yaml create mode 100644 modules/net-vpc-factory/recipes/only-projects/net-land-01.yaml create mode 100644 modules/net-vpc-factory/recipes/only-projects/net-prod-01.yaml create mode 100644 modules/net-vpc-factory/recipes/separate-envs/net-dev-01.yaml create mode 100644 modules/net-vpc-factory/recipes/separate-envs/net-land-01.yaml create mode 100644 modules/net-vpc-factory/recipes/separate-envs/net-prod-01.yaml create mode 100644 modules/net-vpc-factory/schemas/network-project.schema.json create mode 100644 modules/net-vpc-factory/variables.tf create mode 100644 tests/modules/net_vpc_factory/common.tfvars create mode 100644 tests/modules/net_vpc_factory/ncc.tfvars create mode 100644 tests/modules/net_vpc_factory/ncc.yaml create mode 100644 tests/modules/net_vpc_factory/only_projects.tfvars create mode 100644 tests/modules/net_vpc_factory/only_projects.yaml create mode 100644 tests/modules/net_vpc_factory/peering.tfvars create mode 100644 tests/modules/net_vpc_factory/peering.yaml create mode 100644 tests/modules/net_vpc_factory/separate_envs.tfvars create mode 100644 tests/modules/net_vpc_factory/separate_envs.yaml create mode 100644 tests/modules/net_vpc_factory/tftest.yaml create mode 100644 tests/modules/net_vpc_factory/vpn.tfvars create mode 100644 tests/modules/net_vpc_factory/vpn.yaml diff --git a/README.md b/README.md index aa537ca80..58d4907fd 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Currently available modules: - **foundational** - [billing account](./modules/billing-account), [Cloud Identity group](./modules/cloud-identity-group/), [folder](./modules/folder), [service accounts](./modules/iam-service-account), [logging bucket](./modules/logging-bucket), [organization](./modules/organization), [project](./modules/project), [projects-data-source](./modules/projects-data-source) - **process factories** - [project factory](./modules/project-factory/README.md) -- **networking** - [DNS](./modules/dns), [DNS Response Policy](./modules/dns-response-policy/), [Cloud Endpoints](./modules/endpoints), [address reservation](./modules/net-address), [NAT](./modules/net-cloudnat), [VLAN Attachment](./modules/net-vlan-attachment/), [External Application LB](./modules/net-lb-app-ext/), [External Passthrough Network LB](./modules/net-lb-ext), [External Regional Application Load Balancer](./modules/net-lb-app-ext-regional/), [Firewall policy](./modules/net-firewall-policy), [Internal Application LB](./modules/net-lb-app-int), [Cross-region Internal Application LB](./modules/net-lb-app-int-cross-region), [Internal Passthrough Network LB](./modules/net-lb-int), [Internal Proxy Network LB](./modules/net-lb-proxy-int), [IPSec over Interconnect](./modules/net-ipsec-over-interconnect), [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC peering](./modules/net-vpc-peering), [VPN dynamic](./modules/net-vpn-dynamic), [HA VPN](./modules/net-vpn-ha), [VPN static](./modules/net-vpn-static), [Service Directory](./modules/service-directory), [Secure Web Proxy](./modules/net-swp) +- **networking** - [DNS](./modules/dns), [DNS Response Policy](./modules/dns-response-policy/), [Cloud Endpoints](./modules/endpoints), [address reservation](./modules/net-address), [NAT](./modules/net-cloudnat), [VLAN Attachment](./modules/net-vlan-attachment/), [External Application LB](./modules/net-lb-app-ext/), [External Passthrough Network LB](./modules/net-lb-ext), [External Regional Application Load Balancer](./modules/net-lb-app-ext-regional/), [Firewall policy](./modules/net-firewall-policy), [Internal Application LB](./modules/net-lb-app-int), [Cross-region Internal Application LB](./modules/net-lb-app-int-cross-region), [Internal Passthrough Network LB](./modules/net-lb-int), [Internal Proxy Network LB](./modules/net-lb-proxy-int), [IPSec over Interconnect](./modules/net-ipsec-over-interconnect), [VPC](./modules/net-vpc), [VPC factory](./modules/net-vpc-factory/README.md), [VPC firewall](./modules/net-vpc-firewall), [VPC peering](./modules/net-vpc-peering), [VPN dynamic](./modules/net-vpn-dynamic), [HA VPN](./modules/net-vpn-ha), [VPN static](./modules/net-vpn-static), [Service Directory](./modules/service-directory), [Secure Web Proxy](./modules/net-swp) - **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [COS container](./modules/cloud-config-container/cos-generic-metadata/) (coredns, mysql, onprem, squid), [GKE cluster](./modules/gke-cluster-standard), [GKE hub](./modules/gke-hub), [GKE nodepool](./modules/gke-nodepool), [GCVE private cloud](./modules/gcve-private-cloud) - **data** - [AlloyDB instance](./modules/alloydb), [Analytics Hub](./modules/analytics-hub), [BigQuery dataset](./modules/bigquery-dataset), [Biglake Catalog](./modules/biglake-catalog), [Bigtable instance](./modules/bigtable-instance), [Dataplex](./modules/dataplex), [Dataplex DataScan](./modules/dataplex-datascan), [Cloud SQL instance](./modules/cloudsql-instance), [Spanner instance](./modules/spanner-instance), [Firestore](./modules/firestore), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag), [Data Catalog Tag](./modules/data-catalog-tag), [Data Catalog Tag Template](./modules/data-catalog-tag-template), [Datafusion](./modules/datafusion), [Dataproc](./modules/dataproc), [GCS](./modules/gcs), [Pub/Sub](./modules/pubsub), [Dataform Repository](./modules/dataform-repository/), [Looker Core](./modules/looker-core) - **development** - [API Gateway](./modules/api-gateway), [Apigee](./modules/apigee), [Artifact Registry](./modules/artifact-registry), [Container Registry](./modules/container-registry), [Cloud Source Repository](./modules/source-repository), [Secure Source Manager instance](./modules/secure-source-manager-instance), [Workstation cluster](./modules/workstation-cluster) diff --git a/blueprints/factories/README.md b/blueprints/factories/README.md index 814ad3086..cfe174392 100644 --- a/blueprints/factories/README.md +++ b/blueprints/factories/README.md @@ -74,3 +74,5 @@ The second factory type is implemented as a standalone module that internally re - **projects** - [`project-factory`](../../modules/project-factory/) +- **VPCs** + - [`net-vpc-factory`](../../modules/net-vpc-factory/) diff --git a/modules/README.md b/modules/README.md index 682419f19..05f9d636d 100644 --- a/modules/README.md +++ b/modules/README.md @@ -60,6 +60,7 @@ These modules are used in the examples included in this repository. If you are u - [NAT](./net-cloudnat) - [Service Directory](./service-directory) - [VPC](./net-vpc) +- [VPC factory](./net-vpc-factory) - [VPC firewall](./net-vpc-firewall) - [VPN dynamic](./net-vpn-dynamic) - [VPC peering](./net-vpc-peering) diff --git a/modules/net-vpc-factory/README.md b/modules/net-vpc-factory/README.md new file mode 100644 index 000000000..aa40b3fd8 --- /dev/null +++ b/modules/net-vpc-factory/README.md @@ -0,0 +1,831 @@ +# Networking Factory + +The factory is implemented as a thin data translation layer for the underlying modules, so that no "magic" or hidden side effects are implemented in code, and debugging or integration of new features are simple. + +The code is meant to be executed by a service account or a user with a wide set of permissions: + +- roles/resourcemanager.projectCreator (only if creating projects) +- roles/billing.user (only if creating projects) +- roles/resourcemanager.folderViewer (if parent is folder) +- roles/orgpolicy.policyAdmin (at the organization level, in case you're configuring Org Policies) +- roles/compute.xpnAdmin (at the parent folder level, if Shared VPC is used) + +This factory module acts as a wrapper around several core Terraform modules: + +- [`../project`](../project/): For managing GCP projects. +- [`../net-vpc`](../net-vpc): For managing VPC networks, subnets, routes, PSA, etc. +- [`../net-vpc-firewall`](../net-vpc-firewall/): For managing VPC firewall rules. +- [`../net-vpn-ha`](../net-vpn-ha): For managing Cloud HA VPN connections. +- [`../net-cloudnat`](../net-cloudnat/): For managing Cloud NAT instances. +- [`../dns`](../dns): For managing Cloud DNS zones and record sets. + +## TOC + + +- [TOC](#toc) +- [Factory configuration](#factory-configuration) + - [Configuration Methods](#configuration-methods) + - [Cross-Referencing Resources](#cross-referencing-resources) +- [Projects](#projects) +- [VPCs](#vpcs) + - [Subnets](#subnets) + - [Firewall Rules](#firewall-rules) + - [Cloud NAT](#cloud-nat) + - [Routes](#routes) + - [Private Service Access (PSA)](#private-service-access-psa) + - [VPC DNS Policy](#vpc-dns-policy) +- [Connectivity](#connectivity) + - [VPC Peering](#vpc-peering) + - [Cloud VPN](#cloud-vpn) + - [GCP to OnPrem](#gcp-to-onprem) + - [GCP to GCP VPN](#gcp-to-gcp-vpn) + - [Network Connectivity Center (NCC)](#network-connectivity-center-ncc) + - [NCC VPC Spokes](#ncc-vpc-spokes) + - [NCC VPN Spokes](#ncc-vpn-spokes) +- [DNS](#dns) + - [Private Zone](#private-zone) + - [Forwarding Zone](#forwarding-zone) + - [Peering Zone](#peering-zone) + - [Public Zone with DNSSEC](#public-zone-with-dnssec) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Factory configuration + +At a high level, the factory consumes all YAML files in a directory (whose path is `var.factories_config.vpcs`). +Each file defines a single project, which can either be created by the factory or pre-created externally, and all the network infrastructure to be deployed in that project, including + +- Project + - NCC Hubs + - VPCs + - Connectivity (including VPNs, VPC peerings, NCC VPC Spokes and NCC VPN hybrid spokes) + - DNS policies + - DNS zones + - Firewall rules/policies (via a sub-factory) + - NAT (public or private) + - PSA configuration + - Routes (including PBRs) + - Subnets (via a sub-factory) + +### Configuration Methods + +- **YAML Factory:** The primary method is to define project and network configurations within individual YAML files placed in the directory specified by `var.factories_config.vpcs`. This factory approach allows for modular and organized management of complex setups. +- **Direct Variable Input (Discouraged):** The module also defines `var.network_project_config`. While it's *technically possible* to populate this variable directly in a .tfvars file, this is **not the recommended approach**. It bypasses the intended factory pattern and makes managing multiple project configurations cumbersome. The variable definition primarily serves as **documentation** for the structure expected within the YAML configuration files. + +### Cross-Referencing Resources + +A key capability of this factory is establishing relationships between resources defined across different VPCs or projects. This is achieved through cross-referencing using logical keys within the YAML configuration. These keys strictly follow the pattern `project_key/vpc_key/resource_key` for resources like VPCs, VPN gateways, routers, or NCC hubs/groups. During Terraform execution, the factory code translates these logical keys into the actual resource IDs or self-links required by the underlying modules. This mechanism avoids hardcoding resource IDs and enables the declarative definition of complex topologies. It's extensively used in configurations for: + +VPC Peering: Specifying the peer_network (e.g., `peer_network: net-land-01/hub`). + +DNS Peering/Forwarding: Defining peer_network or client_networks for DNS zones (e.g., `client_networks: [net-dev-01/dev-spoke]`). + +GCP-to-GCP HA VPN: Referencing the peer HA VPN gateway using its logical key (e.g., `peer_gateways.default.gcp: net-dev-01/dev-spoke/to-hub`). + +Network Connectivity Center (NCC): Linking spokes to hubs (`ncc_config.hub: net-land-01/hub`) and groups (`ncc_config.group: net-land-01/hub/default`), or auto-accepting spokes in hub groups (`auto_accept: [net-prod-01, net-dev-01]`). + +## Projects + +The `project_config` block within each YAML file implements a large subset of the [project module](../project/) variable interface, which allows for project creation or reuse, services enablement and IAM/Organization policies definition. + +Below is a valid YAML file which simply creates a project, enables a minimal set of services, configures the project as a host project, adds an authoritative +role binding and sets an Organization Policy at the project level: + +```hcl +module "net-vpc-factory" { + source = "./fabric/modules/net-vpc-factory" + billing_account = "123456-789012-345678" + parent_id = "folders/123456789012" + prefix = "myprefix" + factories_config = { vpcs = "recipes/examples" } +} +# tftest files=project-config +``` + +```yaml +project_config: + name: project-config + services: + - compute.googleapis.com + - dns.googleapis.com + - networkmanagement.googleapis.com + - networksecurity.googleapis.com + - servicenetworking.googleapis.com + - stackdriver.googleapis.com + - vpcaccess.googleapis.com + shared_vpc_host_config: + enabled: true + iam: + roles/owner: + - group:prj-admins@example.com + org_policies: + compute.restrictLoadBalancerCreationForTypes: + rules: + - allow: + values: + - EXTERNAL_HTTP_HTTPS +# tftest-file id=project-config path=recipes/examples/project-config.yaml schema=network-project.schema.json +``` + +## VPCs + +The `vpc_config` block within each project's YAML file defines one or more VPCs to be created in that project. It implements a large subset of the [net-vpc module](../net-vpc/) variable interface, allowing for the creation of VPCs, subnets, routes, etc. + +Below is a valid YAML file excerpt with a minimal set of configurations creating a project and two VPCs, and pointing to sub-factories for subnets and firewall rules: + +```hcl +module "net-vpc-factory" { + source = "./fabric/modules/net-vpc-factory" + billing_account = "123456-789012-345678" + parent_id = "folders/123456789012" + prefix = "myprefix" + factories_config = { vpcs = "recipes/examples" } +} +# tftest files=vpc-config +``` + +```yaml +project_config: + name: vpc-config + services: + - compute.googleapis.com +vpc_config: + net-00: + subnets_factory_config: + subnets_folder: data/subnets/net-00 + firewall_factory_config: + rules_folder: data/firewall/net-00 +# tftest-file id=vpc-config path=recipes/examples/vpc-config.yaml schema=network-project.schema.json +``` + +### Subnets + +Subnets are managed via the `vpc_config..subnets_factory_config` parameter within the project's YAML file. This leverages the [`net-vpc`](../net-vpc/) module for subnet creation and management. Please refer to its documentation for detailed information on subnet configuration options and the expected YAML structure within the subnets_folder. + +### Firewall Rules + +Firewall rules are managed via the `vpc_config..firewall_factory_config` parameter, which leverages the [net-vpc-firewall](../net-vpc-firewall/) module for firewall rule creation and management. Please refer to its documentation for detailed information on firewall rule configuration options and the expected YAML structure within the rules_folder. + +### Cloud NAT + +Cloud NAT is configured via the `vpc_config..nat_config` block. This uses the [net-cloudnat](../net-cloudnat/) module. + +```hcl +module "net-vpc-factory" { + source = "./fabric/modules/net-vpc-factory" + billing_account = "123456-789012-345678" + parent_id = "folders/123456789012" + prefix = "myprefix" + factories_config = { vpcs = "recipes/examples" } +} +# tftest files=nat-config +``` + +```yaml +project_config: + name: nat-config + services: + - compute.googleapis.com +vpc_config: + net-00: + delete_default_routes_on_create: false + nat_config: + nat-ew8: + region: europe-west8 + subnets_factory_config: + subnets_folder: data/subnets/foobar + firewall_factory_config: + rules_folder: data/firewall/foobar +# tftest-file id=nat-config path=recipes/examples/nat-config.yaml schema=network-project.schema.json +``` + +### Routes + +Static routes and Policy-Based Routes (PBRs) are configured within the `vpc_config..routes` and `vpc_config..policy_based_routes` blocks. + +```hcl +module "net-vpc-factory" { + source = "./fabric/modules/net-vpc-factory" + billing_account = "123456-789012-345678" + parent_id = "folders/123456789012" + prefix = "myprefix" + factories_config = { vpcs = "recipes/examples" } +} +# tftest files=routes-config +``` + +```yaml +project_config: + name: routes-config + services: + - compute.googleapis.com +vpc_config: + net-00: + routes: + default-internet: + dest_range: 0.0.0.0/0 + next_hop_type: gateway + next_hop: default-internet-gateway + priority: 1000 + policy_based_routes: + pbr-to-nva: + filter: + src_range: 10.0.1.0/24 + dest_range: 0.0.0.0/0 + next_hop_ilb_ip: 10.0.100.5 + priority: 100 +# tftest-file id=routes-config path=recipes/examples/routes-config.yaml schema=network-project.schema.json +``` + +Refer to the [net-vpc](../net-vpc/) module documentation for details on routes and policy_based_routes. + +### Private Service Access (PSA) + +PSA configuration is managed via `vpc_config..psa_config`. + +```hcl +module "net-vpc-factory" { + source = "./fabric/modules/net-vpc-factory" + billing_account = "123456-789012-345678" + parent_id = "folders/123456789012" + prefix = "myprefix" + factories_config = { vpcs = "recipes/examples" } +} +# tftest files=psa-config +``` + +```yaml +project_config: + name: psa-config + services: + - compute.googleapis.com +vpc_config: + net-00: + psa_config: + - ranges: + global-psa-range: 10.100.0.0/24 + peered_domains: ["onprem.example."] +# tftest-file id=psa-config path=recipes/examples/psa-config.yaml schema=network-project.schema.json +``` + +Refer to the [net-vpc](../net-vpc) module documentation for details on psa_config. + +### VPC DNS Policy + +Configure VPC-level DNS behavior, such as enabling inbound query forwarding or logging, using the `dns_policy` block. + +```hcl +module "net-vpc-factory" { + source = "./fabric/modules/net-vpc-factory" + billing_account = "123456-789012-345678" + parent_id = "folders/123456789012" + prefix = "myprefix" + factories_config = { vpcs = "recipes/examples" } +} +# tftest files=vpc-dns-policy +``` + +```yaml +project_config: + name: vpc-dns-policy + services: + - compute.googleapis.com + - dns.googleapis.com +vpc_config: + net-dns-policy-demo: + dns_policy: + inbound: true + logging: true +# tftest-file id=vpc-dns-policy path=recipes/examples/vpc-dns-policy.yaml schema=network-project.schema.json +``` + +## Connectivity + +This module supports various connectivity options: + +### VPC Peering + +The example below implements a Hub-and-spoke design, where spokes are connected to the Hub via VPC Peerings: + +```hcl +module "net-vpc-factory" { + source = "./fabric/modules/net-vpc-factory" + billing_account = "123456-789012-345678" + parent_id = "folders/123456789012" + prefix = "myprefix" + factories_config = { vpcs = "recipes/examples" } +} +# tftest files=peerings-hub,peerings-dev,peerings-prod +``` + +Hub: + +```yaml +project_config: + name: net-land-01 + services: + - compute.googleapis.com +vpc_config: + hub: + peering_config: + to-prod: + peer_network: net-prod-01/prod-spoke + to-dev: + peer_network: net-dev-01/dev-spoke + stack_type: IPV4_IPV6 + +# tftest-file id=peerings-hub path=recipes/examples/net-land-01.yaml schema=network-project.schema.json +``` + +Dev Spoke: + +```yaml +project_config: + name: net-dev-01 + services: + - compute.googleapis.com +vpc_config: + dev-spoke: + peering_config: + to-hub: + peer_network: net-land-01/hub + stack_type: IPV4_IPV6 + +# tftest-file id=peerings-dev path=recipes/examples/net-dev-01.yaml schema=network-project.schema.json +``` + +Prod Spoke: + +```yaml +project_config: + name: net-prod-01 + services: + - compute.googleapis.com +vpc_config: + prod-spoke: + peering_config: + to-hub: + peer_network: net-land-01/hub + +# tftest-file id=peerings-prod path=recipes/examples/net-prod-01.yaml schema=network-project.schema.json +``` + +### Cloud VPN + +The example below implements a Hub-and-spoke design, where spokes are connected to the Hub via HA-VPN: + +#### GCP to OnPrem + +This example demonstrates connecting an on-premises network to GCP via HA-VPN. The `vpn_config` implements the same interface as module [net-vpn-ha](../net-vpn-ha/). + +In this example, the configuration `vpc_config.net-00.routers` creates a router named `vpn-router` in europe-west8, and `vpc_config.net-00.vpn_config.to-onprem.router_config.name` refers to it, using the key `//` (e.g., prj-01/net-00/vpn-router. +Per module `net-vpn-ha`, omitting the `router_config` configuration results in the router being automatically created and managed by the VPN module itself. + +Note that - given the limit of 5 Cloud Routers per VPC per region - we recommend creating routers as required and using them across multiple VPNs/Interconnects by setting and referencing the pre-created router. + +```hcl +module "net-vpc-factory" { + source = "./fabric/modules/net-vpc-factory" + billing_account = "123456-789012-345678" + parent_id = "folders/123456789012" + prefix = "myprefix" + factories_config = { vpcs = "recipes/examples" } +} +# tftest files=vpn-config +``` + +```yaml +project_config: + name: prj-01 + services: + - compute.googleapis.com +vpc_config: + net-00: + routers: + vpn-router: + region: europe-west8 + asn: 64514 + vpn_config: + to-onprem: + region: europe-west8 + peer_gateways: + default: + external: + redundancy_type: SINGLE_IP_INTERNALLY_REDUNDANT + interfaces: + - 8.8.8.8 + router_config: + create: false + name: prj-01/net-00/vpn-router + tunnels: + remote-0: + bgp_peer: + address: 169.254.1.1 + asn: 64513 + bgp_session_range: "169.254.1.2/30" + peer_external_gateway_interface: 0 + shared_secret: "mySecret" + vpn_gateway_interface: 0 + remote-1: + bgp_peer: + address: 169.254.2.1 + asn: 64513 + bgp_session_range: "169.254.2.2/30" + peer_external_gateway_interface: 0 + shared_secret: "mySecret" + vpn_gateway_interface: 1 +# tftest-file id=vpn-config path=recipes/examples/vpn-config.yaml schema=network-project.schema.json +``` + +#### GCP to GCP VPN + +These examples demonstrates VPC to VPC connectivity via HA VPN. +In this examples, project `net-land-01` has a VPC named `hub` and project `net-dev-01` has a VPC named `dev-spoke`; the two VPCs are connected together via HA VPN. In order to do so, the `vpc_config.vpn_config.to-hub.peer_gateways.default.gcp` on each side is configured by cross-referencing the VPN gateway in the other side, whose reference is `//`. + +```hcl +module "net-vpc-factory" { + source = "./fabric/modules/net-vpc-factory" + billing_account = "123456-789012-345678" + parent_id = "folders/123456789012" + prefix = "myprefix" + factories_config = { vpcs = "recipes/examples" } +} +# tftest files=vpn-hub,vpn-spoke +``` + +Hub: + +```yaml +project_config: + name: net-land-01 + services: + - compute.googleapis.com +vpc_config: + hub: + vpn_config: + to-dev: + region: europe-west12 + peer_gateways: + default: + gcp: net-dev-01/dev-spoke/to-hub + router_config: + asn: 64521 + tunnels: + remote-0: + shared_secret: foobar + bgp_peer: + address: 169.254.2.2 + asn: 64520 + bgp_session_range: "169.254.2.1/30" + vpn_gateway_interface: 0 + remote-1: + shared_secret: foobar + bgp_peer: + address: 169.254.2.6 + asn: 64520 + bgp_session_range: "169.254.2.5/30" + vpn_gateway_interface: 1 +# tftest-file id=vpn-hub path=recipes/examples/net-land-01.yaml schema=network-project.schema.json +``` + +Dev Spoke: + +```yaml +project_config: + name: net-dev-01 + services: + - compute.googleapis.com +vpc_config: + dev-spoke: + vpn_config: + to-hub: + region: europe-west12 + peer_gateways: + default: + gcp: net-land-01/hub/to-dev + router_config: + asn: 64520 + tunnels: + remote-0: + shared_secret: foobar + bgp_peer: + address: 169.254.2.1 + asn: 64521 + bgp_session_range: "169.254.2.2/30" + vpn_gateway_interface: 0 + remote-1: + shared_secret: foobar + bgp_peer: + address: 169.254.2.5 + asn: 64521 + bgp_session_range: "169.254.2.6/30" + vpn_gateway_interface: 1 +# tftest-file id=vpn-spoke path=recipes/examples/net-dev-01.yaml schema=network-project.schema.json +``` + +### Network Connectivity Center (NCC) + +#### NCC VPC Spokes + +This example demonstrates how to create an NCC hub, and how to connect multiple spokes to it. On the `net-land-01` project, `ncc_hub_config.groups.default.auto_accept` is configured to automatically accept the listed spoke projects. +On the spokes definition, `vpc_config.$env-spoke.ncc_config` cross references the hub and the default group. + +NCC Hub: + +```hcl +module "net-vpc-factory" { + source = "./fabric/modules/net-vpc-factory" + billing_account = "123456-789012-345678" + parent_id = "folders/123456789012" + prefix = "myprefix" + factories_config = { vpcs = "recipes/examples" } +} +# tftest files=ncc-hub,ncc-dev,ncc-prod +``` + +```yaml +project_config: + name: net-land-01 + services: + - compute.googleapis.com + - networkmanagement.googleapis.com +ncc_hub_config: + name: hub + groups: + default: + auto_accept: + - net-prod-01 + - net-dev-01 +# tftest-file id=ncc-hub path=recipes/examples/net-land-01.yaml schema=network-project.schema.json +``` + +Dev Spoke: + +```yaml +project_config: + name: net-dev-01 + services: + - compute.googleapis.com + - networkmanagement.googleapis.com +vpc_config: + dev-spoke: + ncc_config: + hub: net-land-01/hub +# tftest-file id=ncc-dev path=recipes/examples/net-dev-01.yaml schema=network-project.schema.json +``` + +Prod Spoke: + +```yaml +project_config: + name: net-prod-01 + services: + - compute.googleapis.com + - networkmanagement.googleapis.com +vpc_config: + prod-spoke: + ncc_config: + hub: net-land-01/hub + group: net-land-01/hub/default +# tftest-file id=ncc-prod path=recipes/examples/net-prod-01.yaml schema=network-project.schema.json +``` + +#### NCC VPN Spokes + +This example shows how to connect HA VPN tunnels, typically used for on-premises or other cloud connections, as spokes in an NCC Hub. This enables transitive routing between VPC spokes and the VPN connection via the NCC Hub. + +The key is the `ncc_spoke_config` block within the `vpn_config` definition on the hub project (`net-land-01/hub`). + +```hcl +module "net-vpc-factory" { + source = "./fabric/modules/net-vpc-factory" + billing_account = "123456-789012-345678" + parent_id = "folders/123456789012" + prefix = "myprefix" + factories_config = { vpcs = "recipes/examples" } +} +# tftest files=ncc-vpn +``` + +```yaml +project_config: + name: net-land-01 + services: + - compute.googleapis.com + - networkmanagement.googleapis.com +ncc_hub_config: + name: hub +vpc_config: + hub: + mtu: 1500 + routers: + vpn-router: + region: europe-west8 + asn: 64514 + vpn_config: + to-onprem: + ncc_spoke_config: + hub: net-land-01/hub + region: europe-west8 + peer_gateways: + default: + external: + redundancy_type: SINGLE_IP_INTERNALLY_REDUNDANT + interfaces: + - 8.8.8.8 + router_config: + create: false + name: net-land-01/hub/vpn-router + tunnels: + remote-0: + bgp_peer: + address: 169.254.128.1 + asn: 64513 + bgp_session_range: "169.254.128.2/30" + peer_external_gateway_interface: 0 + shared_secret: "mySecret" + vpn_gateway_interface: 0 + remote-1: + bgp_peer: + address: 169.254.128.5 + asn: 64513 + bgp_session_range: "169.254.128.6/30" + peer_external_gateway_interface: 0 + shared_secret: "mySecret" + vpn_gateway_interface: 1 +# tftest-file id=ncc-vpn path=recipes/examples/net-land-01.yaml schema=network-project.schema.json +``` + +## DNS + +This module supports the creation of forwarding zones, peering zones, public and private zones, and recordsets within these zones. `vpc_config.$vpc.dns_zones` implements a large subset of the [net-dns](../dns/) module variable interface. + +Below a few configuration examples: + +### Private Zone + +This example demonstrates how to create a private zone for the internal.example.com domain, and how to add a record set to it. + +```hcl +module "net-vpc-factory" { + source = "./fabric/modules/net-vpc-factory" + billing_account = "123456-789012-345678" + parent_id = "folders/123456789012" + prefix = "myprefix" + factories_config = { vpcs = "recipes/examples" } +} +# tftest files=dns-private +``` + +```yaml +project_config: + name: net-land-01 + services: + - compute.googleapis.com + - dns.googleapis.com +vpc_config: + hub: + dns_zones: + internal-example: + zone_config: + domain: internal.example.com. + private: + client_networks: + - net-land-01/hub + recordsets: + "A localhost": + records: ["127.0.0.1"] +# tftest-file id=dns-private path=recipes/examples/net-land-01.yaml schema=network-project.schema.json +``` + +### Forwarding Zone + +This example demonstrates how to create a forwarding zone that forwards queries for the `example.com` domain to on-premises DNS servers. + +```hcl +module "net-vpc-factory" { + source = "./fabric/modules/net-vpc-factory" + billing_account = "123456-789012-345678" + parent_id = "folders/123456789012" + prefix = "myprefix" + factories_config = { vpcs = "recipes/examples" } +} +# tftest files=dns-fwd +``` + +```yaml +project_config: + name: net-land-01 + services: + - compute.googleapis.com + - dns.googleapis.com +vpc_config: + hub: + dns_zones: + onprem-fwd: + zone_config: + domain: example.com. + forwarding: + forwarders: + "10.0.0.1": default + "10.0.0.2": default + client_networks: + - net-land-01/hub +# tftest-file id=dns-fwd path=recipes/examples/net-land-01.yaml schema=network-project.schema.json +``` + +### Peering Zone + +This example demonstrates how to create a peering zone that allows the dev-spoke VPC to resolve names in the net-land-01 project's hub VPC. + +```hcl +module "net-vpc-factory" { + source = "./fabric/modules/net-vpc-factory" + billing_account = "123456-789012-345678" + parent_id = "folders/123456789012" + prefix = "myprefix" + factories_config = { vpcs = "recipes/examples" } +} +# tftest files=dns-peering +``` + +```yaml +project_config: + name: net-dev-01 + services: + - compute.googleapis.com + - dns.googleapis.com +vpc_config: + dev-spoke: + dns_zones: + root-peering: + zone_config: + domain: . + peering: + peer_network: net-land-01/hub + client_networks: + - net-dev-01/dev-spoke +# tftest-file id=dns-peering path=recipes/examples/net-land-01.yaml schema=network-project.schema.json +``` + +All of the above combined implements a DNS hub-and-spoke design, where the DNS configuration is mostly centralised in the `net-land-01/hub` VPC, including private zones and forwarding to onprem, and spokes (in this case `net-dev-01/dev-spoke`) "delegate" the root zone (`.`, which is DNS for "*") to the central location via DNS peering. + +### Public Zone with DNSSEC + +This example demonstrates creating a public DNS zone and enabling DNSSEC. + +```hcl +module "net-vpc-factory" { + source = "./fabric/modules/net-vpc-factory" + billing_account = "123456-789012-345678" + parent_id = "folders/123456789012" + prefix = "myprefix" + factories_config = { vpcs = "recipes/examples" } +} +# tftest files=dns-public-dnssec +``` + +```yaml +project_config: + name: net-dns-public + services: + - compute.googleapis.com + - dns.googleapis.com +vpc_config: + hub: + dns_zones: + public-example-com: + zone_config: + domain: my-public-domain.example.com. + public: + dnssec_config: + state: "on" + recordsets: + "A www": + records: ["192.0.2.1"] +# tftest-file id=dns-public-dnssec path=recipes/examples/net-dns-public.yaml schema=network-project.schema.json +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [billing_account](variables.tf#L17) | Billing account id. | string | ✓ | | +| [parent_id](variables.tf#L374) | Root node for the projects created by the factory. Must be either organizations/XXXXXXXX or folders/XXXXXXXX. | string | ✓ | | +| [prefix](variables.tf#L379) | Prefix used for projects. | string | ✓ | | +| [factories_config](variables.tf#L22) | Configuration for network resource factories. | object({…}) | | {…} | +| [network_project_config](variables.tf#L33) | Consolidated configuration for project, VPCs and their associated resources. | map(object({…})) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [host_project_ids](outputs.tf#L17) | Network project ids. | | +| [host_project_numbers](outputs.tf#L22) | Network project numbers. | | +| [subnet_ids](outputs.tf#L27) | IDs of subnets created within each VPC. | | +| [subnet_proxy_only_self_links](outputs.tf#L32) | IDs of proxy-only subnets created within each VPC. | | +| [subnet_psc_self_links](outputs.tf#L42) | IDs of PSC subnets created within each VPC. | | +| [vpc_self_links](outputs.tf#L52) | Self-links for the VPCs created on each project. | | +| [vpn_gateway_endpoints](outputs.tf#L57) | External IP Addresses for the GCP VPN gateways. | | + diff --git a/modules/net-vpc-factory/data/firewall/dev-spoke/default-ingress.yaml b/modules/net-vpc-factory/data/firewall/dev-spoke/default-ingress.yaml new file mode 100644 index 000000000..377a74f41 --- /dev/null +++ b/modules/net-vpc-factory/data/firewall/dev-spoke/default-ingress.yaml @@ -0,0 +1,28 @@ +# skip boilerplate check +--- +# start of document (---) avoids errors if the file only contains comments + +# yaml-language-server: $schema=../../../schemas/firewall-rules.schema.json + +ingress: + ingress-icmp-allow: + description: "Allow ICMP from anywhere." + rules: + - protocol: icmp + ports: [] + priority: 1000 + ingress-ssh-from-iap-allow: + description: "Allow SSH connections from IAP ranges." + source_ranges: + - 35.235.240.0/20 + rules: + - protocol: tcp + ports: + - 22 + priority: 1001 + ingress-default-deny: + description: "Deny and log any unmatched ingress traffic." + deny: true + priority: 65535 + enable_logging: + include_metadata: false diff --git a/modules/net-vpc-factory/data/firewall/hub/default-ingress.yaml b/modules/net-vpc-factory/data/firewall/hub/default-ingress.yaml new file mode 100644 index 000000000..377a74f41 --- /dev/null +++ b/modules/net-vpc-factory/data/firewall/hub/default-ingress.yaml @@ -0,0 +1,28 @@ +# skip boilerplate check +--- +# start of document (---) avoids errors if the file only contains comments + +# yaml-language-server: $schema=../../../schemas/firewall-rules.schema.json + +ingress: + ingress-icmp-allow: + description: "Allow ICMP from anywhere." + rules: + - protocol: icmp + ports: [] + priority: 1000 + ingress-ssh-from-iap-allow: + description: "Allow SSH connections from IAP ranges." + source_ranges: + - 35.235.240.0/20 + rules: + - protocol: tcp + ports: + - 22 + priority: 1001 + ingress-default-deny: + description: "Deny and log any unmatched ingress traffic." + deny: true + priority: 65535 + enable_logging: + include_metadata: false diff --git a/modules/net-vpc-factory/data/firewall/prod-spoke/default-ingress.yaml b/modules/net-vpc-factory/data/firewall/prod-spoke/default-ingress.yaml new file mode 100644 index 000000000..377a74f41 --- /dev/null +++ b/modules/net-vpc-factory/data/firewall/prod-spoke/default-ingress.yaml @@ -0,0 +1,28 @@ +# skip boilerplate check +--- +# start of document (---) avoids errors if the file only contains comments + +# yaml-language-server: $schema=../../../schemas/firewall-rules.schema.json + +ingress: + ingress-icmp-allow: + description: "Allow ICMP from anywhere." + rules: + - protocol: icmp + ports: [] + priority: 1000 + ingress-ssh-from-iap-allow: + description: "Allow SSH connections from IAP ranges." + source_ranges: + - 35.235.240.0/20 + rules: + - protocol: tcp + ports: + - 22 + priority: 1001 + ingress-default-deny: + description: "Deny and log any unmatched ingress traffic." + deny: true + priority: 65535 + enable_logging: + include_metadata: false diff --git a/modules/net-vpc-factory/data/subnets/dev-spoke/dev-default.yaml b/modules/net-vpc-factory/data/subnets/dev-spoke/dev-default.yaml new file mode 100644 index 000000000..ebc7037e2 --- /dev/null +++ b/modules/net-vpc-factory/data/subnets/dev-spoke/dev-default.yaml @@ -0,0 +1,8 @@ +# skip boilerplate check + +# yaml-language-server: $schema=../../../schemas/subnet.schema.json + +name: dev-default +region: europe-west8 +ip_cidr_range: 10.68.0.0/24 +description: Default europe-west8 subnet for dev diff --git a/modules/net-vpc-factory/data/subnets/hub/hub-default.yaml b/modules/net-vpc-factory/data/subnets/hub/hub-default.yaml new file mode 100644 index 000000000..e46babe9b --- /dev/null +++ b/modules/net-vpc-factory/data/subnets/hub/hub-default.yaml @@ -0,0 +1,8 @@ +# skip boilerplate check + +# yaml-language-server: $schema=../../../schemas/subnet.schema.json + +name: hub-default +region: europe-west12 +ip_cidr_range: 10.70.0.0/24 +description: Default europe-west12 subnet for hub diff --git a/modules/net-vpc-factory/data/subnets/prod-spoke/prod-default.yaml b/modules/net-vpc-factory/data/subnets/prod-spoke/prod-default.yaml new file mode 100644 index 000000000..50abeb192 --- /dev/null +++ b/modules/net-vpc-factory/data/subnets/prod-spoke/prod-default.yaml @@ -0,0 +1,8 @@ +# skip boilerplate check + +# yaml-language-server: $schema=../../../schemas/subnet.schema.json + +name: prod-default +region: europe-west12 +ip_cidr_range: 10.69.0.0/24 +description: Default europe-west12 subnet for prod diff --git a/modules/net-vpc-factory/factory-dns.tf b/modules/net-vpc-factory/factory-dns.tf new file mode 100644 index 000000000..bf30b580e --- /dev/null +++ b/modules/net-vpc-factory/factory-dns.tf @@ -0,0 +1,87 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description DNS factory. + +locals { + dns_zone_entries = flatten([ + for factory_key, factory_config in local.network_projects : [ + for vpc_key, vpc_config in try(factory_config.vpc_config, {}) : [ + for zone_key, zone in try(vpc_config.dns_zones, {}) : { + key = "${factory_key}/${vpc_key}/${zone_key}" + value = merge( + { + name = replace("${vpc_key}-${zone_key}", "/", "-") + project_id = module.projects[factory_key].id + description = try(zone.description, "Terraform-managed.") + force_destroy = try(zone.force_destroy, null) + iam = try(zone.iam, null) + recordsets = try(zone.recordsets, null) + }, + { + zone_config = merge( + { domain = try(zone.zone_config.domain, null) }, + contains(keys(try(zone.zone_config, {})), "private") ? { + private = { + service_directory_namespace = try(zone.zone_config.private.service_directory_namespace, null) + client_networks = [ + for net in zone.zone_config.private.client_networks : + try(module.vpc[net].self_link, net) + ] + } + } : {}, + contains(keys(try(zone.zone_config, {})), "peering") ? { + peering = { + peer_network = try(module.vpc[zone.zone_config.peering.peer_network].self_link, zone.zone_config.peering.peer_network), + client_networks = [ + for net in zone.zone_config.peering.client_networks : + try(module.vpc[net].self_link, net) + ] + } + } : {}, + contains(keys(try(zone.zone_config, {})), "forwarding") ? { + forwarding = { + forwarders = try(zone.zone_config.forwarding.forwarders, {}), + client_networks = [ + for net in zone.zone_config.forwarding.client_networks : + try(module.vpc[net].self_link, net) + ] + } + } : {} + ) + } + ) + } + ] + ] + ]) + + # Convert the flattened list into a map. + dns_zones = { for entry in local.dns_zone_entries : entry.key => entry.value } +} + +module "dns-zones" { + source = "../dns" + for_each = local.dns_zones + project_id = each.value.project_id + name = each.value.name + description = each.value.description + force_destroy = each.value.force_destroy + iam = each.value.iam + zone_config = each.value.zone_config + recordsets = each.value.recordsets + depends_on = [module.vpc] +} diff --git a/modules/net-vpc-factory/factory-nat.tf b/modules/net-vpc-factory/factory-nat.tf new file mode 100644 index 000000000..eb2e87f6c --- /dev/null +++ b/modules/net-vpc-factory/factory-nat.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Cloud NAT factory. + +locals { + nat_configs = merge(flatten([ + for factory_key, factory_config in local.network_projects : [ + for vpc_key, vpc_config in try(factory_config.vpc_config, {}) : [ + for nat_key, nat_config in try(vpc_config.nat_config, {}) : { + "${factory_key}/${vpc_key}/${nat_key}" = merge(nat_config, { + name = replace("${vpc_key}/${nat_key}", "/", "-") + project_id = module.projects[factory_key].id + addresses = try(nat_config.addresses, []) + config_port_allocation = try(nat_config.config_port_allocation, {}) + config_source_subnetworks = try(nat_config.config_source_subnetworks, {}) + config_timeouts = try(nat_config.config_timeouts, {}) + endpoint_types = try(nat_config.endpoint_types, null) + logging_filter = try(nat_config.logging_filter, null) + router_asn = try(nat_config.router_asn, null) + router_create = try(nat_config.router_create, true) + router_network = module.vpc["${factory_key}/${vpc_key}"].self_link + rules = try(nat_config.rules, []) + type = try(nat_config.type, "PUBLIC") + }) + } + ] + ] + ])...) +} + +module "nat" { + source = "../net-cloudnat" + for_each = local.nat_configs + project_id = each.value.project_id + name = each.value.name + addresses = each.value.addresses + config_port_allocation = each.value.config_port_allocation + config_source_subnetworks = each.value.config_source_subnetworks + config_timeouts = each.value.config_timeouts + endpoint_types = each.value.endpoint_types + logging_filter = each.value.logging_filter + region = each.value.region + router_asn = each.value.router_asn + router_create = each.value.router_create + router_network = each.value.router_network + rules = each.value.rules + type = each.value.type +} diff --git a/modules/net-vpc-factory/factory-ncc.tf b/modules/net-vpc-factory/factory-ncc.tf new file mode 100644 index 000000000..1a2bdc70a --- /dev/null +++ b/modules/net-vpc-factory/factory-ncc.tf @@ -0,0 +1,139 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description NCC factory. + +locals { + ncc_hubs = { for k, v in local.network_projects : "${k}/${v.ncc_hub_config.name}" => + { + name = v.ncc_hub_config.name + project_id = module.projects[k].id + description = try(v.ncc_hub_config.description, "Terraform-managed") + export_psc = try(v.ncc_hub_config.export_psc, true) + preset_topology = try(v.ncc_hub_config.preset_topology, "MESH") + } + if try(v.ncc_hub_config != null, false) + } + + ncc_groups = merge(flatten([for k, v in local.network_projects : + { + for gk, gv in try(v.ncc_hub_config.groups, {}) : "${k}/${v.ncc_hub_config.name}/${gk}" => + { + name = gk + project = module.projects[k].id + hub = google_network_connectivity_hub.default["${k}/${v.ncc_hub_config.name}"].id + description = try(gv.description, "Terraform-managed") + labels = try(gv.labels, {}) + auto_accept = [for project_key in try(gv.auto_accept, []) : module.projects[project_key].id] + } + } + if try(v.ncc_hub_config != null, false) + ])...) + + ncc_vpn_spokes = merge(flatten([ + for factory_key, factory_config in local.network_projects : [ + for vpc_key, vpc_config in try(factory_config.vpc_config, {}) : [ + for vpn_key, vpn_config in try(vpc_config.vpn_config, {}) : { + "${factory_key}/${vpc_key}/${vpn_key}" = { + name = replace("${factory_key}/${vpc_key}/${vpn_key}", "/", "-") + project_id = module.projects[factory_key].id + hub = google_network_connectivity_hub.default[vpn_config.ncc_spoke_config.hub].id + location = vpn_config.region + description = lookup(vpn_config.ncc_spoke_config, "description", "Terraform-managed.") + labels = lookup(vpn_config.ncc_spoke_config, "labels", {}) + tunnel_self_link = [for t, _ in vpn_config.tunnels : module.vpn-ha["${factory_key}/${vpc_key}/${vpn_key}"].tunnel_self_links[t]] + } + } + if try(vpn_config.ncc_spoke_config != null, false) + ] + ] + ])...) + + ncc_vpc_spokes = merge(flatten([ + for factory_key, factory_config in local.network_projects : { + for vpc_key, vpc_config in try(factory_config.vpc_config, {}) : "${factory_key}/${vpc_key}" => merge(vpc_config.ncc_config, { + project_id = module.projects[factory_key].id + network_self_link = module.vpc["${factory_key}/${vpc_key}"].self_link + labels = try(vpc_config.ncc_config.labels, {}) + hub = google_network_connectivity_hub.default[vpc_config.ncc_config.hub].id + description = try(vpc_config.ncc_config.description, "Terraform-managed") + exclude_export_ranges = try(vpc_config.ncc_config.exclude_export_ranges, null) + include_export_ranges = try(vpc_config.ncc_config.include_export_ranges, null) + group = try(google_network_connectivity_group.default[vpc_config.ncc_config.group].id, null) + }) + if try(vpc_config.ncc_config != null, false) + } + ])...) + +} + +resource "google_network_connectivity_hub" "default" { + for_each = local.ncc_hubs + name = each.value.name + description = each.value.description + export_psc = each.value.export_psc + preset_topology = each.value.preset_topology + project = each.value.project_id +} + +resource "google_network_connectivity_spoke" "vpcs" { + for_each = local.ncc_vpc_spokes + project = each.value.project_id + name = replace(each.key, "/", "-") + location = "global" + description = each.value.description + labels = each.value.labels + hub = each.value.hub + linked_vpc_network { + uri = each.value.network_self_link + exclude_export_ranges = each.value.exclude_export_ranges + include_export_ranges = each.value.include_export_ranges + } + depends_on = [google_network_connectivity_hub.default] + group = each.value.group +} + +resource "google_network_connectivity_group" "default" { + for_each = local.ncc_groups + project = each.value.project + name = each.value.name + hub = each.value.hub + labels = each.value.labels + description = each.value.description + dynamic "auto_accept" { + for_each = try(each.value.auto_accept != null, false) ? [""] : [] + content { + auto_accept_projects = each.value.auto_accept + } + } + depends_on = [google_network_connectivity_hub.default] +} + +resource "google_network_connectivity_spoke" "tunnels" { + for_each = local.ncc_vpn_spokes + project = each.value.project_id + name = each.value.name + location = each.value.location + description = each.value.description + labels = each.value.labels + hub = each.value.hub + linked_vpn_tunnels { + uris = each.value.tunnel_self_link + site_to_site_data_transfer = true + include_import_ranges = ["ALL_IPV4_RANGES"] + } + depends_on = [module.vpn-ha] +} diff --git a/modules/net-vpc-factory/factory-peering.tf b/modules/net-vpc-factory/factory-peering.tf new file mode 100644 index 000000000..5cfab8889 --- /dev/null +++ b/modules/net-vpc-factory/factory-peering.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Peering factory. + +locals { + peerings = merge(flatten([ + for factory_key, factory_config in local.network_projects : [ + for vpc_key, vpc_config in try(factory_config.vpc_config, {}) : [ + for k, v in try(vpc_config.peering_config, {}) : { + "${factory_key}/${vpc_key}/${k}" = { + project = factory_key + name = replace("${vpc_key}/${k}", "/", "-") + local_network = module.vpc["${factory_key}/${vpc_key}"].self_link + peer_network = module.vpc[v.peer_network].self_link + export_custom_routes = try(v.routes_config.export, true) + import_custom_routes = try(v.routes_config.import, true) + export_subnet_routes_with_public_ip = try(v.routes_config.public_export, null) + import_subnet_routes_with_public_ip = try(v.routes_config.public_import, null) + stack_type = try(v.stack_type, null) + } + } + ] + ] + ])...) +} + +resource "google_compute_network_peering" "default" { + for_each = local.peerings + name = each.value.name + network = each.value.local_network + peer_network = each.value.peer_network + export_custom_routes = each.value.export_custom_routes + import_custom_routes = each.value.import_custom_routes + export_subnet_routes_with_public_ip = each.value.export_subnet_routes_with_public_ip + import_subnet_routes_with_public_ip = each.value.import_subnet_routes_with_public_ip + stack_type = each.value.stack_type +} diff --git a/modules/net-vpc-factory/factory-project.tf b/modules/net-vpc-factory/factory-project.tf new file mode 100644 index 000000000..00f783ba3 --- /dev/null +++ b/modules/net-vpc-factory/factory-project.tf @@ -0,0 +1,53 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Dedicated project factory. +locals { + + projects = { for k, v in local.network_projects : k => merge( + { + billing_account = try(v.project_config.billing_account, var.billing_account) + prefix = try(v.project_config.prefix, var.prefix) + parent = try(v.project_config.parent, var.parent_id) + shared_vpc_host_config = try(v.project_config.shared_vpc_host_config, null) + iam = try(v.project_config.iam, {}) + iam_bindings = try(v.project_config.iam_bindings, {}) + iam_bindings_additive = try(v.project_config.iam_bindings_additive, {}) + iam_by_principals = try(v.project_config.iam_by_principals, {}) + iam_by_principals_additive = try(v.project_config.iam_by_principals_additive, {}) + services = try(v.project_config.services, []) + org_policies = try(v.project_config.org_policies, {}) + }, + v.project_config) + } +} + +module "projects" { + source = "../project" + for_each = local.projects + billing_account = each.value.billing_account + name = each.value.name + parent = each.value.parent + prefix = each.value.prefix + services = each.value.services + shared_vpc_host_config = each.value.shared_vpc_host_config + iam = each.value.iam + iam_bindings = each.value.iam_bindings + iam_bindings_additive = each.value.iam_bindings_additive + iam_by_principals = each.value.iam_by_principals + iam_by_principals_additive = each.value.iam_by_principals_additive + org_policies = each.value.org_policies +} diff --git a/modules/net-vpc-factory/factory-vpc.tf b/modules/net-vpc-factory/factory-vpc.tf new file mode 100644 index 000000000..59d868e52 --- /dev/null +++ b/modules/net-vpc-factory/factory-vpc.tf @@ -0,0 +1,80 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description VPC and firewall factory. + +locals { + _vpcs_preprocess = [for factory_key, factory_config in local.network_projects : { + for k, v in try(factory_config.vpc_config, {}) : "${factory_key}/${k}" => { + project_id = module.projects[factory_key].id + name = k + auto_create_subnetworks = try(v.auto_create_subnetworks, false) + create_googleapis_routes = try(v.create_googleapis_routes, {}) + delete_default_routes_on_create = try(v.delete_default_routes_on_create, false) + description = try(v.description, "Terraform-managed.") + dns_policy = try(v.dns_policy, {}) + firewall_policy_enforcement_order = try(v.firewall_policy_enforcement_order, "AFTER_CLASSIC_FIREWALL") + ipv6_config = try(v.ipv6_config, {}) + mtu = try(v.mtu, null) + network_attachments = try(v.network_attachments, {}) + policy_based_routes = try(v.policy_based_routes, {}) + psa_config = try(v.psa_config, []) + routes = try(v.routes, {}) + routing_mode = try(v.routing_mode, "GLOBAL") + subnets_factory_config = try(v.subnets_factory_config, {}) + firewall_factory_config = try(v.firewall_factory_config, {}) + peering_config = try(v.peering_config, {}) + vpn_config = try(v.vpn_config, {}) + } + }] + + vpcs = merge( + merge(local._vpcs_preprocess...), + var.network_project_config + ) +} + +module "vpc" { + source = "../net-vpc" + for_each = local.vpcs + project_id = each.value.project_id + name = each.value.name + description = each.value.description + auto_create_subnetworks = each.value.auto_create_subnetworks + create_googleapis_routes = each.value.create_googleapis_routes + delete_default_routes_on_create = each.value.delete_default_routes_on_create + dns_policy = each.value.dns_policy + factories_config = each.value.subnets_factory_config + firewall_policy_enforcement_order = each.value.firewall_policy_enforcement_order + ipv6_config = each.value.ipv6_config + mtu = each.value.mtu + network_attachments = each.value.network_attachments + policy_based_routes = each.value.policy_based_routes + psa_configs = each.value.psa_config + routes = each.value.routes + routing_mode = each.value.routing_mode + depends_on = [module.projects] +} + +module "firewall" { + source = "../net-vpc-firewall" + for_each = { for k, v in local.vpcs : k => v if v.firewall_factory_config != null } + project_id = each.value.project_id + network = each.value.name + factories_config = each.value.firewall_factory_config + default_rules_config = { disabled = true } + depends_on = [module.vpc] +} diff --git a/modules/net-vpc-factory/factory-vpn.tf b/modules/net-vpc-factory/factory-vpn.tf new file mode 100644 index 000000000..937db9a6a --- /dev/null +++ b/modules/net-vpc-factory/factory-vpn.tf @@ -0,0 +1,111 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description VPN factory. + +locals { + routers = merge(flatten([ + for factory_key, factory_config in local.network_projects : [ + for vpc_key, vpc_config in try(factory_config.vpc_config, {}) : [ + for router_key, router_config in try(vpc_config.routers, {}) : { + "${factory_key}/${vpc_key}/${router_key}" = merge(router_config, { + vpc_self_link = module.vpc["${factory_key}/${vpc_key}"].self_link + project_id = module.projects[factory_key].id + custom_advertise = try(router_config.custom_advertise, {}) + advertise_mode = try(router_config.custom_advertise != null, false) ? "CUSTOM" : "DEFAULT" + advertised_groups = try(router_config.custom_advertise.all_subnets, false) ? ["ALL_SUBNETS"] : [] + keepalive = try(router_config.keepalive, null) + asn = try(router_config.asn, null) + }) + } + ] + ] + ])...) + + vpns = merge(flatten([ + for factory_key, factory_config in local.network_projects : [ + for vpc_key, vpc_config in try(factory_config.vpc_config, {}) : [ + for k, v in try(vpc_config.vpn_config, {}) : { + "${factory_key}/${vpc_key}/${k}" = merge(v, { + vpc_name = module.vpc["${factory_key}/${vpc_key}"].name + vpn_name = replace("${factory_key}/${vpc_key}/${k}", "/", "-") + project_id = module.projects[factory_key].id + }, + { + router_config = merge(v.router_config, + try(v.router_config.create, false) == false && can(v.router_config.name) ? { + name = try(google_compute_router.default[v.router_config.name].name, v.router_config.name) + } : {} + ) + } + ) + } + ] + ] + ])...) +} + +resource "google_compute_router" "default" { + for_each = local.routers + name = replace(each.key, "/", "-") + project = each.value.project_id + region = each.value.region + network = each.value.vpc_self_link + bgp { + advertise_mode = each.value.advertise_mode + advertised_groups = each.value.advertised_groups + dynamic "advertised_ip_ranges" { + for_each = try(each.value.custom_advertise.ip_ranges, {}) + iterator = range + content { + range = range.key + description = range.value + } + } + keepalive_interval = each.value.keepalive + asn = each.value.asn + } +} + +resource "google_compute_ha_vpn_gateway" "default" { + for_each = local.vpns + project = each.value.project_id + region = each.value.region + name = replace(each.key, "/", "-") + network = each.value.vpc_name + stack_type = try(each.value.stack_type, null) + depends_on = [module.vpc] +} + +module "vpn-ha" { + source = "../net-vpn-ha" + for_each = local.vpns + project_id = each.value.project_id + name = replace(each.key, "/", "-") + network = each.value.vpc_name + region = each.value.region + router_config = each.value.router_config + tunnels = each.value.tunnels + vpn_gateway = google_compute_ha_vpn_gateway.default[each.key].id + vpn_gateway_create = null + peer_gateways = { + for k, gw in each.value.peer_gateways : k => { + for gw_type, value in gw : gw_type => ( + gw_type == "gcp" ? try(google_compute_ha_vpn_gateway.default[value].id, value) : value + ) + } + } +} diff --git a/modules/net-vpc-factory/main.tf b/modules/net-vpc-factory/main.tf new file mode 100644 index 000000000..393b92c71 --- /dev/null +++ b/modules/net-vpc-factory/main.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Read and process YaML factory files and variables. +locals { + _network_factory_path = try( + pathexpand(var.factories_config.vpcs), null + ) + _network_factory_files = try( + fileset(local._network_factory_path, "**/*.yaml"), + [] + ) + + _network_projects_from_files = { + for f in local._network_factory_files : + f => yamldecode(file("${local._network_factory_path}/${f}")) + } + + _network_projects = { + for _, v in local._network_projects_from_files : + v.project_config.name => v + } + + network_projects = merge(local._network_projects, var.network_project_config) +} diff --git a/modules/net-vpc-factory/outputs.tf b/modules/net-vpc-factory/outputs.tf new file mode 100644 index 000000000..3c54eb6f9 --- /dev/null +++ b/modules/net-vpc-factory/outputs.tf @@ -0,0 +1,64 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "host_project_ids" { + description = "Network project ids." + value = { for k, v in module.projects : k => v.project_id } +} + +output "host_project_numbers" { + description = "Network project numbers." + value = { for k, v in module.projects : k => v.number } +} + +output "subnet_ids" { + description = "IDs of subnets created within each VPC." + value = { for k, v in module.vpc : k => v.subnet_ids } +} + +output "subnet_proxy_only_self_links" { + description = "IDs of proxy-only subnets created within each VPC." + value = { + for k, v in module.vpc : k => + { + for subnet_key, subnet_value in v.subnets_proxy_only : subnet_key => subnet_value.id + } + } +} + +output "subnet_psc_self_links" { + description = "IDs of PSC subnets created within each VPC." + value = { + for k, v in module.vpc : k => + { + for subnet_key, subnet_value in v.subnets_psc : subnet_key => subnet_value.id + } + } +} + +output "vpc_self_links" { + description = "Self-links for the VPCs created on each project." + value = { for k, v in module.vpc : k => v.self_link } +} + +output "vpn_gateway_endpoints" { + description = "External IP Addresses for the GCP VPN gateways." + value = { for k, v in google_compute_ha_vpn_gateway.default : k => + { + for interface_key, interface_value in v.vpn_interfaces : interface_key => interface_value.ip_address + } + } +} diff --git a/modules/net-vpc-factory/recipes/hub-and-spoke-ncc/net-dev-01.yaml b/modules/net-vpc-factory/recipes/hub-and-spoke-ncc/net-dev-01.yaml new file mode 100644 index 000000000..5029be709 --- /dev/null +++ b/modules/net-vpc-factory/recipes/hub-and-spoke-ncc/net-dev-01.yaml @@ -0,0 +1,58 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../schemas/network-project.schema.json + +project_config: + name: net-dev-01 + services: + - container.googleapis.com + - compute.googleapis.com + - dns.googleapis.com + - iap.googleapis.com + - networkmanagement.googleapis.com + - networksecurity.googleapis.com + - servicenetworking.googleapis.com + - stackdriver.googleapis.com + - vpcaccess.googleapis.com + shared_vpc_host_config: + enabled: true +vpc_config: + dev-spoke: + delete_default_routes_on_create: false + mtu: 1500 + nat_config: + nat-ew8: + region: europe-west8 + dns_zones: + root-peering: + zone_config: + domain: . + peering: + peer_network: net-land-01/hub + client_networks: + - net-dev-01/dev-spoke + subnets_factory_config: + subnets_folder: data/subnets/dev-spoke + firewall_factory_config: + rules_folder: data/firewall/dev-spoke + routes: + gateway: + dest_range: "8.8.8.8/32" + priority: 100 + next_hop_type: "gateway" + next_hop: "default-internet-gateway" + ncc_config: + hub: net-land-01/hub + group: net-land-01/hub/default diff --git a/modules/net-vpc-factory/recipes/hub-and-spoke-ncc/net-land-01.yaml b/modules/net-vpc-factory/recipes/hub-and-spoke-ncc/net-land-01.yaml new file mode 100644 index 000000000..9d5c1562f --- /dev/null +++ b/modules/net-vpc-factory/recipes/hub-and-spoke-ncc/net-land-01.yaml @@ -0,0 +1,109 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../schemas/network-project.schema.json + +project_config: + name: net-land-01 + services: + - container.googleapis.com + - compute.googleapis.com + - dns.googleapis.com + - iap.googleapis.com + - networkmanagement.googleapis.com + - networksecurity.googleapis.com + - servicenetworking.googleapis.com + - stackdriver.googleapis.com + - vpcaccess.googleapis.com + shared_vpc_host_config: + enabled: true +ncc_hub_config: + name: hub + groups: + default: + auto_accept: + - net-prod-01 + - net-dev-01 +vpc_config: + test: + delete_default_routes_on_create: false + mtu: 1500 + auto_create_subnetworks: true + hub: + delete_default_routes_on_create: false + mtu: 1500 + routers: + vpn-router: + region: europe-west8 + asn: 64514 + dns_zones: + onprem-fwd: + zone_config: + domain: . + forwarding: + forwarders: + "8.8.8.8": default + "1.1.1.1": default + client_networks: + - net-land-01/hub + dot-test: + zone_config: + domain: test. + private: + client_networks: + - net-land-01/hub + recordsets: + "A localhost": + records: ["127.0.0.1"] + subnets_factory_config: + subnets_folder: data/subnets/hub + firewall_factory_config: + rules_folder: data/firewall/hub + routes: + gateway: + dest_range: "8.8.8.8/32" + priority: 100 + next_hop_type: "gateway" + next_hop: "default-internet-gateway" + vpn_config: + to-onprem: + ncc_spoke_config: + hub: net-land-01/hub + region: europe-west8 + peer_gateways: + default: + external: + redundancy_type: SINGLE_IP_INTERNALLY_REDUNDANT + interfaces: + - 8.8.8.8 + router_config: + create: false + name: net-land-01/hub/vpn-router + tunnels: + remote-0: + bgp_peer: + address: 169.254.128.1 + asn: 64513 + bgp_session_range: "169.254.128.2/30" + peer_external_gateway_interface: 0 + shared_secret: "mySecret" + vpn_gateway_interface: 0 + remote-1: + bgp_peer: + address: 169.254.128.5 + asn: 64513 + bgp_session_range: "169.254.128.6/30" + peer_external_gateway_interface: 0 + shared_secret: "mySecret" + vpn_gateway_interface: 1 diff --git a/modules/net-vpc-factory/recipes/hub-and-spoke-ncc/net-prod-01.yaml b/modules/net-vpc-factory/recipes/hub-and-spoke-ncc/net-prod-01.yaml new file mode 100644 index 000000000..ba37a7393 --- /dev/null +++ b/modules/net-vpc-factory/recipes/hub-and-spoke-ncc/net-prod-01.yaml @@ -0,0 +1,58 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../schemas/network-project.schema.json + +project_config: + name: net-prod-01 + services: + - container.googleapis.com + - compute.googleapis.com + - dns.googleapis.com + - iap.googleapis.com + - networkmanagement.googleapis.com + - networksecurity.googleapis.com + - servicenetworking.googleapis.com + - stackdriver.googleapis.com + - vpcaccess.googleapis.com + shared_vpc_host_config: + enabled: true +vpc_config: + prod-spoke: + delete_default_routes_on_create: false + mtu: 1500 + nat_config: + nat-ew8: + region: europe-west8 + dns_zones: + root-peering: + zone_config: + domain: . + peering: + peer_network: net-land-01/hub + client_networks: + - net-prod-01/prod-spoke + subnets_factory_config: + subnets_folder: data/subnets/prod-spoke + firewall_factory_config: + rules_folder: data/firewall/prod-spoke + routes: + gateway: + dest_range: "8.8.8.8/32" + priority: 100 + next_hop_type: "gateway" + next_hop: "default-internet-gateway" + ncc_config: + hub: net-land-01/hub + group: net-land-01/hub/default diff --git a/modules/net-vpc-factory/recipes/hub-and-spoke-peering/net-dev-01.yaml b/modules/net-vpc-factory/recipes/hub-and-spoke-peering/net-dev-01.yaml new file mode 100644 index 000000000..4128c4374 --- /dev/null +++ b/modules/net-vpc-factory/recipes/hub-and-spoke-peering/net-dev-01.yaml @@ -0,0 +1,58 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../schemas/network-project.schema.json + +project_config: + name: net-dev-01 + services: + - container.googleapis.com + - compute.googleapis.com + - dns.googleapis.com + - iap.googleapis.com + - networkmanagement.googleapis.com + - networksecurity.googleapis.com + - servicenetworking.googleapis.com + - stackdriver.googleapis.com + - vpcaccess.googleapis.com + shared_vpc_host_config: + enabled: true +vpc_config: + dev-spoke: + delete_default_routes_on_create: false + mtu: 1500 + nat_config: + nat-ew8: + region: europe-west8 + dns_zones: + root-peering: + zone_config: + domain: . + peering: + peer_network: net-land-01/hub + client_networks: + - net-dev-01/dev-spoke + subnets_factory_config: + subnets_folder: data/subnets/dev-spoke + firewall_factory_config: + rules_folder: data/firewall/dev-spoke + routes: + gateway: + dest_range: "8.8.8.8/32" + priority: 100 + next_hop_type: "gateway" + next_hop: "default-internet-gateway" + peering_config: + to-hub: + peer_network: net-land-01/hub diff --git a/modules/net-vpc-factory/recipes/hub-and-spoke-peering/net-land-01.yaml b/modules/net-vpc-factory/recipes/hub-and-spoke-peering/net-land-01.yaml new file mode 100644 index 000000000..84a99a393 --- /dev/null +++ b/modules/net-vpc-factory/recipes/hub-and-spoke-peering/net-land-01.yaml @@ -0,0 +1,113 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../schemas/network-project.schema.json + +project_config: + name: net-land-01 + services: + - container.googleapis.com + - compute.googleapis.com + - dns.googleapis.com + - iap.googleapis.com + - networkmanagement.googleapis.com + - networksecurity.googleapis.com + - servicenetworking.googleapis.com + - stackdriver.googleapis.com + - vpcaccess.googleapis.com + shared_vpc_host_config: + enabled: true +vpc_config: + test: + delete_default_routes_on_create: false + mtu: 1500 + auto_create_subnetworks: true + peering_config: + to-hub: + peer_network: net-land-01/hub + hub: + delete_default_routes_on_create: false + mtu: 1500 + nat_config: + nat-ew8: + region: europe-west8 + routers: + vpn-router: + region: europe-west8 + asn: 64514 + dns_zones: + onprem-fwd: + zone_config: + domain: . + forwarding: + forwarders: + "8.8.8.8": default + "1.1.1.1": default + client_networks: + - net-land-01/hub + dot-test: + zone_config: + domain: test. + private: + client_networks: + - net-land-01/hub + recordsets: + "A localhost": + records: ["127.0.0.1"] + subnets_factory_config: + subnets_folder: data/subnets/hub + firewall_factory_config: + rules_folder: data/firewall/hub + routes: + gateway: + dest_range: "8.8.8.8/32" + priority: 100 + next_hop_type: "gateway" + next_hop: "default-internet-gateway" + peering_config: + to-prod: + peer_network: net-prod-01/prod-spoke + to-dev: + peer_network: net-dev-01/dev-spoke + to-test: + peer_network: net-land-01/test + vpn_config: + to-onprem: + region: europe-west8 + peer_gateways: + default: + external: + redundancy_type: SINGLE_IP_INTERNALLY_REDUNDANT + interfaces: + - 8.8.8.8 + router_config: + create: false + name: net-land-01/hub/vpn-router + tunnels: + remote-0: + bgp_peer: + address: 169.254.128.1 + asn: 64513 + bgp_session_range: "169.254.128.2/30" + peer_external_gateway_interface: 0 + shared_secret: "mySecret" + vpn_gateway_interface: 0 + remote-1: + bgp_peer: + address: 169.254.128.5 + asn: 64513 + bgp_session_range: "169.254.128.6/30" + peer_external_gateway_interface: 0 + shared_secret: "mySecret" + vpn_gateway_interface: 1 diff --git a/modules/net-vpc-factory/recipes/hub-and-spoke-peering/net-prod-01.yaml b/modules/net-vpc-factory/recipes/hub-and-spoke-peering/net-prod-01.yaml new file mode 100644 index 000000000..33039b243 --- /dev/null +++ b/modules/net-vpc-factory/recipes/hub-and-spoke-peering/net-prod-01.yaml @@ -0,0 +1,58 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../schemas/network-project.schema.json + +project_config: + name: net-prod-01 + services: + - container.googleapis.com + - compute.googleapis.com + - dns.googleapis.com + - iap.googleapis.com + - networkmanagement.googleapis.com + - networksecurity.googleapis.com + - servicenetworking.googleapis.com + - stackdriver.googleapis.com + - vpcaccess.googleapis.com + shared_vpc_host_config: + enabled: true +vpc_config: + prod-spoke: + delete_default_routes_on_create: false + mtu: 1500 + nat_config: + nat-ew8: + region: europe-west8 + dns_zones: + root-peering: + zone_config: + domain: . + peering: + peer_network: net-land-01/hub + client_networks: + - net-prod-01/prod-spoke + subnets_factory_config: + subnets_folder: data/subnets/prod-spoke + firewall_factory_config: + rules_folder: data/firewall/prod-spoke + routes: + gateway: + dest_range: "8.8.8.8/32" + priority: 100 + next_hop_type: "gateway" + next_hop: "default-internet-gateway" + peering_config: + to-hub: + peer_network: net-land-01/hub diff --git a/modules/net-vpc-factory/recipes/hub-and-spoke-vpn/net-dev-01.yaml b/modules/net-vpc-factory/recipes/hub-and-spoke-vpn/net-dev-01.yaml new file mode 100644 index 000000000..b26f3a005 --- /dev/null +++ b/modules/net-vpc-factory/recipes/hub-and-spoke-vpn/net-dev-01.yaml @@ -0,0 +1,83 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../schemas/network-project.schema.json + +project_config: + name: net-dev-01 + services: + - container.googleapis.com + - compute.googleapis.com + - dns.googleapis.com + - iap.googleapis.com + - networkmanagement.googleapis.com + - networksecurity.googleapis.com + - servicenetworking.googleapis.com + - stackdriver.googleapis.com + - vpcaccess.googleapis.com + shared_vpc_host_config: + enabled: true +vpc_config: + dev-spoke: + delete_default_routes_on_create: false + mtu: 1500 + nat_config: + nat-ew8: + region: europe-west8 + routers: + vpn-router: + region: europe-west8 + asn: 64520 + dns_zones: + root-peering: + zone_config: + domain: . + peering: + peer_network: net-land-01/hub + client_networks: + - net-dev-01/dev-spoke + subnets_factory_config: + subnets_folder: data/subnets/dev-spoke + firewall_factory_config: + rules_folder: data/firewall/dev-spoke + routes: + gateway: + dest_range: "8.8.8.8/32" + priority: 100 + next_hop_type: "gateway" + next_hop: "default-internet-gateway" + vpn_config: + to-hub: + region: europe-west8 + peer_gateways: + default: + gcp: net-land-01/hub/to-dev + router_config: + create: false + name: net-dev-01/dev-spoke/vpn-router + tunnels: + remote-0: + shared_secret: foobar + bgp_peer: + address: 169.254.2.1 + asn: 64514 + bgp_session_range: "169.254.2.2/30" + vpn_gateway_interface: 0 + remote-1: + shared_secret: foobar + bgp_peer: + address: 169.254.2.5 + asn: 64514 + bgp_session_range: "169.254.2.6/30" + vpn_gateway_interface: 1 diff --git a/modules/net-vpc-factory/recipes/hub-and-spoke-vpn/net-land-01.yaml b/modules/net-vpc-factory/recipes/hub-and-spoke-vpn/net-land-01.yaml new file mode 100644 index 000000000..effe127a7 --- /dev/null +++ b/modules/net-vpc-factory/recipes/hub-and-spoke-vpn/net-land-01.yaml @@ -0,0 +1,153 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../schemas/network-project.schema.json + +project_config: + name: net-land-01 + services: + - container.googleapis.com + - compute.googleapis.com + - dns.googleapis.com + - iap.googleapis.com + - networkmanagement.googleapis.com + - networksecurity.googleapis.com + - servicenetworking.googleapis.com + - stackdriver.googleapis.com + - vpcaccess.googleapis.com + shared_vpc_host_config: + enabled: true +vpc_config: + test: + delete_default_routes_on_create: false + mtu: 1500 + auto_create_subnetworks: true + hub: + delete_default_routes_on_create: false + mtu: 1500 + nat_config: + nat-ew8: + region: europe-west8 + routers: + vpn-router: + region: europe-west8 + asn: 64514 + custom_advertise: + all_subnets: false + ip_ranges: + "10.0.0.0/8": "rfc1918_10" + dns_zones: + onprem-fwd: + zone_config: + domain: . + forwarding: + forwarders: + "8.8.8.8": default + "1.1.1.1": default + client_networks: + - net-land-01/hub + dot-test: + zone_config: + domain: test. + private: + client_networks: + - net-land-01/hub + recordsets: + "A localhost": + records: ["127.0.0.1"] + subnets_factory_config: + subnets_folder: data/subnets/hub + firewall_factory_config: + rules_folder: data/firewall/hub + routes: + gateway: + dest_range: "8.8.8.8/32" + priority: 100 + next_hop_type: "gateway" + next_hop: "default-internet-gateway" + vpn_config: + to-onprem: + region: europe-west8 + peer_gateways: + default: + external: + redundancy_type: SINGLE_IP_INTERNALLY_REDUNDANT + interfaces: + - 8.8.8.8 + router_config: + create: false + name: net-land-01/hub/vpn-router + tunnels: + remote-0: + bgp_peer: + address: 169.254.128.1 + asn: 64513 + bgp_session_range: "169.254.128.2/30" + peer_external_gateway_interface: 0 + shared_secret: "mySecret" + vpn_gateway_interface: 0 + remote-1: + bgp_peer: + address: 169.254.128.5 + asn: 64513 + bgp_session_range: "169.254.128.6/30" + peer_external_gateway_interface: 0 + shared_secret: "mySecret" + vpn_gateway_interface: 1 + to-dev: + region: europe-west8 + peer_gateways: + default: + gcp: net-dev-01/dev-spoke/to-hub + router_config: + create: false + name: net-land-01/hub/vpn-router + tunnels: + remote-0: + shared_secret: foobar + bgp_peer: + address: 169.254.2.2 + asn: 64520 + bgp_session_range: "169.254.2.1/30" + vpn_gateway_interface: 0 + remote-1: + shared_secret: foobar + bgp_peer: + address: 169.254.2.6 + asn: 64520 + bgp_session_range: "169.254.2.5/30" + vpn_gateway_interface: 1 + to-prod: + region: europe-west8 + peer_gateways: + default: + gcp: net-prod-01/prod-spoke/to-hub + router_config: + create: false + name: net-land-01/hub/vpn-router + tunnels: + remote-0: + shared_secret: foobar + bgp_peer: + address: 169.254.3.2 + asn: 64523 + bgp_session_range: "169.254.3.1/30" + vpn_gateway_interface: 0 + remote-1: + shared_secret: foobar + bgp_peer: + address: 169.254.3.6 + asn: 64523 + bgp_session_range: "169.254.3.5/30" + vpn_gateway_interface: 1 diff --git a/modules/net-vpc-factory/recipes/hub-and-spoke-vpn/net-prod-01.yaml b/modules/net-vpc-factory/recipes/hub-and-spoke-vpn/net-prod-01.yaml new file mode 100644 index 000000000..4b835e70d --- /dev/null +++ b/modules/net-vpc-factory/recipes/hub-and-spoke-vpn/net-prod-01.yaml @@ -0,0 +1,83 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../schemas/network-project.schema.json + +project_config: + name: net-prod-01 + services: + - container.googleapis.com + - compute.googleapis.com + - dns.googleapis.com + - iap.googleapis.com + - networkmanagement.googleapis.com + - networksecurity.googleapis.com + - servicenetworking.googleapis.com + - stackdriver.googleapis.com + - vpcaccess.googleapis.com + shared_vpc_host_config: + enabled: true +vpc_config: + prod-spoke: + delete_default_routes_on_create: false + mtu: 1500 + nat_config: + nat-ew8: + region: europe-west8 + routers: + vpn-router: + region: europe-west8 + asn: 64523 + dns_zones: + root-peering: + zone_config: + domain: . + peering: + peer_network: net-land-01/hub + client_networks: + - net-prod-01/prod-spoke + subnets_factory_config: + subnets_folder: data/subnets/prod-spoke + firewall_factory_config: + rules_folder: data/firewall/prod-spoke + routes: + gateway: + dest_range: "8.8.8.8/32" + priority: 100 + next_hop_type: "gateway" + next_hop: "default-internet-gateway" + vpn_config: + to-hub: + region: europe-west8 + peer_gateways: + default: + gcp: net-land-01/hub/to-prod + router_config: + create: false + name: net-prod-01/prod-spoke/vpn-router + tunnels: + remote-0: + shared_secret: foobar + bgp_peer: + address: 169.254.3.1 + asn: 64514 + bgp_session_range: "169.254.3.2/30" + vpn_gateway_interface: 0 + remote-1: + shared_secret: foobar + bgp_peer: + address: 169.254.3.5 + asn: 64514 + bgp_session_range: "169.254.3.6/30" + vpn_gateway_interface: 1 diff --git a/modules/net-vpc-factory/recipes/only-projects/net-dev-01.yaml b/modules/net-vpc-factory/recipes/only-projects/net-dev-01.yaml new file mode 100644 index 000000000..fe2f82580 --- /dev/null +++ b/modules/net-vpc-factory/recipes/only-projects/net-dev-01.yaml @@ -0,0 +1,28 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../schemas/network-project.schema.json + +project_config: + name: net-dev-01 + services: + - container.googleapis.com + - compute.googleapis.com + - dns.googleapis.com + - iap.googleapis.com + - networkmanagement.googleapis.com + - networksecurity.googleapis.com + - servicenetworking.googleapis.com + - stackdriver.googleapis.com + - vpcaccess.googleapis.com diff --git a/modules/net-vpc-factory/recipes/only-projects/net-land-01.yaml b/modules/net-vpc-factory/recipes/only-projects/net-land-01.yaml new file mode 100644 index 000000000..55d0cbaec --- /dev/null +++ b/modules/net-vpc-factory/recipes/only-projects/net-land-01.yaml @@ -0,0 +1,30 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../schemas/network-project.schema.json + +# This file creates an empty project, and exists to keep consistency with the other recipes. + +project_config: + name: net-land-01 + services: + - container.googleapis.com + - compute.googleapis.com + - dns.googleapis.com + - iap.googleapis.com + - networkmanagement.googleapis.com + - networksecurity.googleapis.com + - servicenetworking.googleapis.com + - stackdriver.googleapis.com + - vpcaccess.googleapis.com diff --git a/modules/net-vpc-factory/recipes/only-projects/net-prod-01.yaml b/modules/net-vpc-factory/recipes/only-projects/net-prod-01.yaml new file mode 100644 index 000000000..72b7d0145 --- /dev/null +++ b/modules/net-vpc-factory/recipes/only-projects/net-prod-01.yaml @@ -0,0 +1,28 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../schemas/network-project.schema.json + +project_config: + name: net-prod-01 + services: + - container.googleapis.com + - compute.googleapis.com + - dns.googleapis.com + - iap.googleapis.com + - networkmanagement.googleapis.com + - networksecurity.googleapis.com + - servicenetworking.googleapis.com + - stackdriver.googleapis.com + - vpcaccess.googleapis.com diff --git a/modules/net-vpc-factory/recipes/separate-envs/net-dev-01.yaml b/modules/net-vpc-factory/recipes/separate-envs/net-dev-01.yaml new file mode 100644 index 000000000..d3e1b13f9 --- /dev/null +++ b/modules/net-vpc-factory/recipes/separate-envs/net-dev-01.yaml @@ -0,0 +1,72 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../schemas/network-project.schema.json + +project_config: + name: net-dev-01 + services: + - container.googleapis.com + - compute.googleapis.com + - dns.googleapis.com + - iap.googleapis.com + - networkmanagement.googleapis.com + - networksecurity.googleapis.com + - servicenetworking.googleapis.com + - stackdriver.googleapis.com + - vpcaccess.googleapis.com + shared_vpc_host_config: + enabled: true +vpc_config: + dev-spoke: + delete_default_routes_on_create: false + mtu: 1500 + nat_config: + nat-ew8: + region: europe-west8 + routers: + vpn-router: + region: europe-west8 + asn: 64514 + subnets_factory_config: + subnets_folder: data/subnets/dev-spoke + firewall_factory_config: + rules_folder: data/firewall/dev-spoke + routes: + gateway: + dest_range: "8.8.8.8/32" + priority: 100 + next_hop_type: "gateway" + next_hop: "default-internet-gateway" + vpn_config: + to-onprem: + region: europe-west8 + peer_gateways: + default: + external: + redundancy_type: SINGLE_IP_INTERNALLY_REDUNDANT + interfaces: + - 8.8.8.8 + router_config: + create: false + name: net-dev-01/dev-spoke/vpn-router + tunnels: + remote-0: + bgp_peer: + address: 169.254.1.1 + asn: 64513 + bgp_session_range: "169.254.1.2/30" + peer_external_gateway_interface: 0 + shared_secret: "mySecret" + vpn_gateway_interface: 0 diff --git a/modules/net-vpc-factory/recipes/separate-envs/net-land-01.yaml b/modules/net-vpc-factory/recipes/separate-envs/net-land-01.yaml new file mode 100644 index 000000000..55d0cbaec --- /dev/null +++ b/modules/net-vpc-factory/recipes/separate-envs/net-land-01.yaml @@ -0,0 +1,30 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../schemas/network-project.schema.json + +# This file creates an empty project, and exists to keep consistency with the other recipes. + +project_config: + name: net-land-01 + services: + - container.googleapis.com + - compute.googleapis.com + - dns.googleapis.com + - iap.googleapis.com + - networkmanagement.googleapis.com + - networksecurity.googleapis.com + - servicenetworking.googleapis.com + - stackdriver.googleapis.com + - vpcaccess.googleapis.com diff --git a/modules/net-vpc-factory/recipes/separate-envs/net-prod-01.yaml b/modules/net-vpc-factory/recipes/separate-envs/net-prod-01.yaml new file mode 100644 index 000000000..4da9151e0 --- /dev/null +++ b/modules/net-vpc-factory/recipes/separate-envs/net-prod-01.yaml @@ -0,0 +1,72 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# yaml-language-server: $schema=../../schemas/network-project.schema.json + +project_config: + name: net-prod-01 + services: + - container.googleapis.com + - compute.googleapis.com + - dns.googleapis.com + - iap.googleapis.com + - networkmanagement.googleapis.com + - networksecurity.googleapis.com + - servicenetworking.googleapis.com + - stackdriver.googleapis.com + - vpcaccess.googleapis.com + shared_vpc_host_config: + enabled: true +vpc_config: + prod-spoke: + delete_default_routes_on_create: false + mtu: 1500 + nat_config: + nat-ew8: + region: europe-west8 + routers: + vpn-router: + region: europe-west8 + asn: 64514 + subnets_factory_config: + subnets_folder: data/subnets/prod-spoke + firewall_factory_config: + rules_folder: data/firewall/prod-spoke + routes: + gateway: + dest_range: "8.8.8.8/32" + priority: 100 + next_hop_type: "gateway" + next_hop: "default-internet-gateway" + vpn_config: + to-onprem: + region: europe-west8 + peer_gateways: + default: + external: + redundancy_type: SINGLE_IP_INTERNALLY_REDUNDANT + interfaces: + - 8.8.8.8 + router_config: + create: false + name: net-prod-01/prod-spoke/vpn-router + tunnels: + remote-0: + bgp_peer: + address: 169.254.1.1 + asn: 64513 + bgp_session_range: "169.254.1.2/30" + peer_external_gateway_interface: 0 + shared_secret: "mySecret" + vpn_gateway_interface: 0 diff --git a/modules/net-vpc-factory/schemas/network-project.schema.json b/modules/net-vpc-factory/schemas/network-project.schema.json new file mode 100644 index 000000000..130f42868 --- /dev/null +++ b/modules/net-vpc-factory/schemas/network-project.schema.json @@ -0,0 +1,1479 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Network Project Configuration (Single)", + "description": "Schema for a single network project configuration defined at the root.", + "type": "object", + "additionalProperties": false, + "required": [ + "project_config" + ], + "properties": { + "project_config": { + "$ref": "#/$defs/projectConfig" + }, + "ncc_hub_config": { + "$ref": "#/$defs/nccHubConfig" + }, + "vpc_config": { + "$ref": "#/$defs/vpcConfigMap" + } + }, + "$defs": { + "projectConfig": { + "type": "object", + "additionalProperties": false, + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "parent": { + "type": "string" + }, + "billing_account": { + "type": "string" + }, + "deletion_policy": { + "type": "string", + "enum": [ + "DELETE", + "ABANDON" + ] + }, + "default_service_account": { + "type": "string", + "enum": [ + "deprovision", + "disable", + "keep" + ] + }, + "auto_create_network": { + "type": "boolean" + }, + "project_create": { + "type": "boolean" + }, + "shared_vpc_host_config": { + "type": "object", + "additionalProperties": false, + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "service_projects": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "services": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z-]+\\.googleapis\\.com$" + } + }, + "org_policies": { + "$ref": "#/$defs/orgPolicies" + }, + "metric_scopes": { + "type": "array", + "items": { + "type": "string" + } + }, + "iam": { + "$ref": "#/$defs/iam" + }, + "iam_bindings": { + "$ref": "#/$defs/iamBindings" + }, + "iam_bindings_additive": { + "$ref": "#/$defs/iamBindingsAdditive" + }, + "iam_by_principals": { + "$ref": "#/$defs/iamByPrincipals" + }, + "iam_by_principals_additive": { + "$ref": "#/$defs/iamByPrincipals" + } + } + }, + "nccHubConfig": { + "type": "object", + "additionalProperties": false, + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "preset_topology": { + "type": "string", + "enum": [ + "MESH", + "STAR", + "PLANETARY" + ] + }, + "export_psc": { + "type": "boolean" + }, + "groups": { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9_-]+$": { + "$ref": "#/$defs/nccGroup" + } + }, + "additionalProperties": false + } + } + }, + "nccGroup": { + "type": "object", + "additionalProperties": false, + "properties": { + "labels": { + "$ref": "#/$defs/stringMap" + }, + "description": { + "type": "string" + }, + "auto_accept": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "vpcConfigMap": { + "type": "object", + "patternProperties": { + "^[a-z0-9-]+$": { + "$ref": "#/$defs/vpcConfigEntry" + } + }, + "additionalProperties": false + }, + "vpcConfigEntry": { + "type": "object", + "additionalProperties": false, + "properties": { + "auto_create_subnetworks": { + "type": "boolean" + }, + "create_googleapis_routes": { + "type": "object", + "additionalProperties": false, + "properties": { + "private": { + "type": "boolean" + }, + "private-6": { + "type": "boolean" + }, + "restricted": { + "type": "boolean" + }, + "restricted-6": { + "type": "boolean" + } + } + }, + "delete_default_routes_on_create": { + "type": "boolean" + }, + "description": { + "type": "string" + }, + "dns_policy": { + "type": "object", + "additionalProperties": false, + "properties": { + "inbound": { + "type": "boolean" + }, + "logging": { + "type": "boolean" + }, + "outbound": { + "type": "object", + "additionalProperties": false, + "properties": { + "private_ns": { + "type": "array", + "items": { + "type": "string" + } + }, + "public_ns": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + }, + "dns_zones": { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9-]+$": { + "$ref": "#/$defs/dnsZone" + } + }, + "additionalProperties": false + }, + "firewall_policy_enforcement_order": { + "type": "string", + "enum": [ + "AFTER_CLASSIC_FIREWALL", + "BEFORE_CLASSIC_FIREWALL" + ] + }, + "ipv6_config": { + "type": "object", + "additionalProperties": false, + "properties": { + "enable_ula_internal": { + "type": "boolean" + }, + "internal_range": { + "type": "string" + } + } + }, + "mtu": { + "type": "number" + }, + "nat_config": { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9-]+$": { + "$ref": "#/$defs/natConfig" + } + }, + "additionalProperties": false + }, + "network_attachments": { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9-]+$": { + "$ref": "#/$defs/networkAttachment" + } + }, + "additionalProperties": false + }, + "policy_based_routes": { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9-]+$": { + "$ref": "#/$defs/policyBasedRoute" + } + }, + "additionalProperties": false + }, + "psa_config": { + "type": "array", + "items": { + "$ref": "#/$defs/psaConfig" + } + }, + "routers": { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9-]+$": { + "$ref": "#/$defs/routerConfig" + } + }, + "additionalProperties": false + }, + "routes": { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9-]+$": { + "$ref": "#/$defs/routeConfig" + } + }, + "additionalProperties": false + }, + "routing_mode": { + "type": "string", + "enum": [ + "GLOBAL", + "REGIONAL" + ] + }, + "subnets_factory_config": { + "type": "object", + "additionalProperties": false, + "properties": { + "context": { + "type": "object", + "additionalProperties": false, + "properties": { + "regions": { + "$ref": "#/$defs/stringMap" + } + } + }, + "subnets_folder": { + "type": "string" + } + } + }, + "firewall_factory_config": { + "type": "object", + "additionalProperties": false, + "properties": { + "cidr_tpl_file": { + "type": "string" + }, + "rules_folder": { + "type": "string" + } + } + }, + "vpn_config": { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9-]+$": { + "$ref": "#/$defs/vpnConfig" + } + }, + "additionalProperties": false + }, + "peering_config": { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9-]+$": { + "$ref": "#/$defs/peeringConfig" + } + }, + "additionalProperties": false + }, + "ncc_config": { + "$ref": "#/$defs/vpcNccConfig" + } + } + }, + "dnsZone": { + "type": "object", + "additionalProperties": false, + "required": [ + "zone_config" + ], + "properties": { + "force_destroy": { + "type": "boolean" + }, + "description": { + "type": "string" + }, + "iam": { + "$ref": "#/$defs/iam" + }, + "zone_config": { + "$ref": "#/$defs/dnsZoneConfig" + }, + "recordsets": { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9_. -]+$": { + "$ref": "#/$defs/dnsRecordSet" + } + }, + "additionalProperties": false + } + } + }, + "dnsZoneConfig": { + "type": "object", + "additionalProperties": false, + "required": [ + "domain" + ], + "properties": { + "domain": { + "type": "string" + }, + "forwarding": { + "type": "object", + "additionalProperties": false, + "properties": { + "forwarders": { + "$ref": "#/$defs/stringMap" + }, + "client_networks": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "peering": { + "type": "object", + "additionalProperties": false, + "required": [ + "peer_network" + ], + "properties": { + "client_networks": { + "type": "array", + "items": { + "type": "string" + } + }, + "peer_network": { + "type": "string" + } + } + }, + "public": { + "type": "object", + "additionalProperties": false, + "properties": { + "dnssec_config": { + "$ref": "#/$defs/dnssecConfig" + }, + "enable_logging": { + "type": "boolean" + } + } + }, + "private": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_networks": { + "type": "array", + "items": { + "type": "string" + } + }, + "service_directory_namespace": { + "type": "string" + } + } + } + } + }, + "dnssecConfig": { + "type": "object", + "additionalProperties": false, + "required": [ + "state" + ], + "properties": { + "non_existence": { + "type": "string", + "enum": [ + "nsec", + "nsec3" + ] + }, + "state": { + "type": "string", + "enum": [ + "on", + "off", + "transfer" + ] + }, + "key_signing_key": { + "$ref": "#/$defs/dnsKeySpec" + }, + "zone_signing_key": { + "$ref": "#/$defs/dnsKeySpec" + } + } + }, + "dnsKeySpec": { + "type": "object", + "additionalProperties": false, + "required": [ + "algorithm", + "key_length" + ], + "properties": { + "algorithm": { + "type": "string", + "enum": [ + "rsasha1", + "rsasha256", + "rsasha512", + "ecdsap256sha256", + "ecdsap384sha384" + ] + }, + "key_length": { + "type": "number" + } + } + }, + "dnsRecordSet": { + "type": "object", + "additionalProperties": false, + "properties": { + "ttl": { + "type": "number" + }, + "records": { + "type": "array", + "items": { + "type": "string" + } + }, + "geo_routing": { + "type": "array", + "items": { + "$ref": "#/$defs/dnsGeoRoutingRule" + } + }, + "wrr_routing": { + "type": "array", + "items": { + "$ref": "#/$defs/dnsWrrRoutingRule" + } + } + } + }, + "dnsGeoRoutingRule": { + "type": "object", + "additionalProperties": false, + "required": [ + "location" + ], + "properties": { + "location": { + "type": "string" + }, + "records": { + "type": "array", + "items": { + "type": "string" + } + }, + "health_checked_targets": { + "type": "array", + "items": { + "$ref": "#/$defs/dnsHealthCheckedTarget" + } + } + } + }, + "dnsHealthCheckedTarget": { + "type": "object", + "additionalProperties": false, + "required": [ + "load_balancer_type", + "ip_address", + "port", + "ip_protocol", + "network_url", + "project" + ], + "properties": { + "load_balancer_type": { + "type": "string" + }, + "ip_address": { + "type": "string" + }, + "port": { + "type": "string" + }, + "ip_protocol": { + "type": "string" + }, + "network_url": { + "type": "string" + }, + "project": { + "type": "string" + }, + "region": { + "type": "string" + } + } + }, + "dnsWrrRoutingRule": { + "type": "object", + "additionalProperties": false, + "required": [ + "weight", + "records" + ], + "properties": { + "weight": { + "type": "number" + }, + "records": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "natConfig": { + "type": "object", + "additionalProperties": false, + "required": [ + "region" + ], + "properties": { + "region": { + "type": "string" + }, + "router_create": { + "type": "boolean" + }, + "router_name": { + "type": "string" + }, + "router_network": { + "type": "string" + }, + "router_asn": { + "type": "number" + }, + "type": { + "type": "string", + "enum": [ + "PUBLIC", + "PRIVATE" + ] + }, + "addresses": { + "type": "array", + "items": { + "type": "string" + } + }, + "endpoint_types": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "ENDPOINT_TYPE_VM", + "ENDPOINT_TYPE_SWG", + "ENDPOINT_TYPE_MANAGED_PROXY_LB" + ] + } + }, + "logging_filter": { + "type": "string", + "enum": [ + "ERRORS_ONLY", + "TRANSLATIONS_ONLY", + "ALL" + ] + }, + "config_port_allocation": { + "type": "object", + "additionalProperties": false, + "properties": { + "enable_endpoint_independent_mapping": { + "type": "boolean" + }, + "enable_dynamic_port_allocation": { + "type": "boolean" + }, + "min_ports_per_vm": { + "type": "number" + }, + "max_ports_per_vm": { + "type": "number" + } + } + }, + "config_source_subnetworks": { + "type": "object", + "additionalProperties": false, + "properties": { + "all": { + "type": "boolean" + }, + "primary_ranges_only": { + "type": "boolean" + }, + "subnetworks": { + "type": "array", + "items": { + "$ref": "#/$defs/natSourceSubnetwork" + } + } + } + }, + "config_timeouts": { + "type": "object", + "additionalProperties": false, + "properties": { + "icmp": { + "type": "number" + }, + "tcp_established": { + "type": "number" + }, + "tcp_time_wait": { + "type": "number" + }, + "tcp_transitory": { + "type": "number" + }, + "udp": { + "type": "number" + } + } + }, + "rules": { + "type": "array", + "items": { + "$ref": "#/$defs/natRule" + } + } + } + }, + "natSourceSubnetwork": { + "type": "object", + "additionalProperties": false, + "required": [ + "self_link" + ], + "properties": { + "self_link": { + "type": "string" + }, + "all_ranges": { + "type": "boolean" + }, + "primary_range": { + "type": "boolean" + }, + "secondary_ranges": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "natRule": { + "type": "object", + "additionalProperties": false, + "required": [ + "match" + ], + "properties": { + "description": { + "type": "string" + }, + "match": { + "type": "string" + }, + "source_ips": { + "type": "array", + "items": { + "type": "string" + } + }, + "source_ranges": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "networkAttachment": { + "type": "object", + "additionalProperties": false, + "required": [ + "subnet" + ], + "properties": { + "subnet": { + "type": "string" + }, + "automatic_connection": { + "type": "boolean" + }, + "description": { + "type": "string" + }, + "producer_accept_lists": { + "type": "array", + "items": { + "type": "string" + } + }, + "producer_reject_lists": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "policyBasedRoute": { + "type": "object", + "additionalProperties": false, + "properties": { + "description": { + "type": "string" + }, + "labels": { + "$ref": "#/$defs/stringMap" + }, + "priority": { + "type": "number" + }, + "next_hop_ilb_ip": { + "type": "string" + }, + "use_default_routing": { + "type": "boolean" + }, + "filter": { + "type": "object", + "additionalProperties": false, + "properties": { + "ip_protocol": { + "type": "string" + }, + "dest_range": { + "type": "string" + }, + "src_range": { + "type": "string" + } + } + }, + "target": { + "type": "object", + "additionalProperties": false, + "properties": { + "interconnect_attachment": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + }, + "psaConfig": { + "type": "object", + "additionalProperties": false, + "required": [ + "ranges" + ], + "properties": { + "deletion_policy": { + "type": "string", + "enum": [ + "delete", + "abandon" + ] + }, + "ranges": { + "$ref": "#/$defs/stringMap" + }, + "export_routes": { + "type": "boolean" + }, + "import_routes": { + "type": "boolean" + }, + "peered_domains": { + "type": "array", + "items": { + "type": "string" + } + }, + "range_prefix": { + "type": "string" + }, + "service_producer": { + "type": "string" + } + } + }, + "routerConfig": { + "type": "object", + "additionalProperties": false, + "required": [ + "region" + ], + "properties": { + "region": { + "type": "string" + }, + "asn": { + "type": "number" + }, + "custom_advertise": { + "$ref": "#/$defs/customAdvertiseConfig" + }, + "keepalive": { + "type": "number" + }, + "name": { + "type": "string" + } + } + }, + "routeConfig": { + "type": "object", + "additionalProperties": false, + "required": [ + "dest_range", + "next_hop_type", + "next_hop" + ], + "properties": { + "description": { + "type": "string" + }, + "dest_range": { + "type": "string" + }, + "next_hop_type": { + "type": "string" + }, + "next_hop": { + "type": "string" + }, + "priority": { + "type": "number" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "vpnConfig": { + "type": "object", + "additionalProperties": false, + "required": [ + "region", + "peer_gateways", + "router_config", + "tunnels" + ], + "properties": { + "region": { + "type": "string" + }, + "ncc_spoke_config": { + "type": "object", + "additionalProperties": false, + "properties": { + "hub": { + "type": "string" + }, + "description": { + "type": "string" + }, + "labels": { + "$ref": "#/$defs/stringMap" + } + } + }, + "peer_gateways": { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9-]+$": { + "$ref": "#/$defs/peerGateway" + } + }, + "additionalProperties": false + }, + "router_config": { + "$ref": "#/$defs/vpnRouterConfig" + }, + "stack_type": { + "type": "string", + "enum": [ + "IPV4_ONLY", + "IPV4_IPV6" + ] + }, + "tunnels": { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9-]+$": { + "$ref": "#/$defs/vpnTunnel" + } + }, + "additionalProperties": false + } + } + }, + "peerGateway": { + "type": "object", + "additionalProperties": false, + "properties": { + "external": { + "$ref": "#/$defs/externalPeerGateway" + }, + "gcp": { + "type": "string" + } + } + }, + "externalPeerGateway": { + "type": "object", + "additionalProperties": false, + "required": [ + "redundancy_type", + "interfaces" + ], + "properties": { + "redundancy_type": { + "type": "string", + "enum": [ + "SINGLE_IP_INTERNALLY_REDUNDANT", + "TWO_IPS_REDUNDANCY", + "FOUR_IPS_REDUNDANCY" + ] + }, + "interfaces": { + "type": "array", + "items": { + "type": "string" + } + }, + "description": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "vpnRouterConfig": { + "type": "object", + "additionalProperties": false, + "properties": { + "asn": { + "type": "number" + }, + "create": { + "type": "boolean" + }, + "custom_advertise": { + "$ref": "#/$defs/customAdvertiseConfig" + }, + "keepalive": { + "type": "number" + }, + "name": { + "type": "string" + }, + "override_name": { + "type": "string" + } + } + }, + "vpnTunnel": { + "type": "object", + "additionalProperties": false, + "required": [ + "bgp_peer", + "bgp_session_range", + "vpn_gateway_interface" + ], + "properties": { + "bgp_peer": { + "$ref": "#/$defs/bgpPeerConfig" + }, + "bgp_session_range": { + "type": "string" + }, + "ike_version": { + "type": "number", + "enum": [ + 1, + 2 + ] + }, + "name": { + "type": "string" + }, + "peer_external_gateway_interface": { + "type": "number" + }, + "peer_router_interface_name": { + "type": "string" + }, + "peer_gateway": { + "type": "string" + }, + "router": { + "type": "string" + }, + "shared_secret": { + "type": "string" + }, + "vpn_gateway_interface": { + "type": "number" + } + } + }, + "bgpPeerConfig": { + "type": "object", + "additionalProperties": false, + "required": [ + "address", + "asn" + ], + "properties": { + "address": { + "type": "string" + }, + "asn": { + "type": "number" + }, + "route_priority": { + "type": "number" + }, + "custom_advertise": { + "$ref": "#/$defs/customAdvertiseConfig" + }, + "md5_authentication_key": { + "type": "object", + "additionalProperties": false, + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "key": { + "type": "string" + } + } + }, + "ipv6": { + "type": "object", + "additionalProperties": false, + "properties": { + "nexthop_address": { + "type": "string" + }, + "peer_nexthop_address": { + "type": "string" + } + } + }, + "name": { + "type": "string" + } + } + }, + "customAdvertiseConfig": { + "type": "object", + "additionalProperties": false, + "required": [ + "all_subnets" + ], + "properties": { + "all_subnets": { + "type": "boolean" + }, + "ip_ranges": { + "$ref": "#/$defs/stringMap" + } + } + }, + "peeringConfig": { + "type": "object", + "additionalProperties": false, + "required": [ + "peer_network" + ], + "properties": { + "peer_network": { + "type": "string" + }, + "routes_config": { + "type": "object", + "additionalProperties": false, + "properties": { + "export": { + "type": "boolean" + }, + "import": { + "type": "boolean" + }, + "public_export": { + "type": "boolean" + }, + "public_import": { + "type": "boolean" + } + } + }, + "stack_type": { + "type": "string", + "enum": [ + "IPV4_ONLY", + "IPV4_IPV6" + ] + } + } + }, + "vpcNccConfig": { + "type": "object", + "additionalProperties": false, + "required": [ + "hub" + ], + "properties": { + "hub": { + "type": "string" + }, + "description": { + "type": "string" + }, + "labels": { + "$ref": "#/$defs/stringMap" + }, + "group": { + "type": "string" + }, + "exclude_export_ranges": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + }, + "include_export_ranges": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + } + } + }, + "stringMap": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "condition": { + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "title" + ], + "properties": { + "expression": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + } + } + }, + "principalPattern": { + "type": "string", + "pattern": "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|[a-z])" + }, + "rolePattern": { + "type": "string", + "pattern": "^roles/" + }, + "iam": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^roles/": { + "type": "array", + "items": { + "$ref": "#/$defs/principalPattern" + } + } + } + }, + "iamBindings": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "object", + "additionalProperties": false, + "required": [ + "members", + "role" + ], + "properties": { + "members": { + "type": "array", + "items": { + "$ref": "#/$defs/principalPattern" + } + }, + "role": { + "$ref": "#/$defs/rolePattern" + }, + "condition": { + "$ref": "#/$defs/condition" + } + } + } + } + }, + "iamBindingsAdditive": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9_-]+$": { + "type": "object", + "additionalProperties": false, + "required": [ + "member", + "role" + ], + "properties": { + "member": { + "$ref": "#/$defs/principalPattern" + }, + "role": { + "$ref": "#/$defs/rolePattern" + }, + "condition": { + "$ref": "#/$defs/condition" + } + } + } + } + }, + "iamByPrincipals": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|[a-z])": { + "type": "array", + "items": { + "$ref": "#/$defs/rolePattern" + } + } + } + }, + "orgPolicies": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z]+\\.": { + "$ref": "#/$defs/orgPolicyConfig" + } + } + }, + "orgPolicyConfig": { + "type": "object", + "properties": { + "inherit_from_parent": { + "type": "boolean" + }, + "reset": { + "type": "boolean" + }, + "rules": { + "type": "array", + "items": { + "$ref": "#/$defs/orgPolicyRule" + } + } + } + }, + "orgPolicyRule": { + "type": "object", + "additionalProperties": false, + "properties": { + "allow": { + "$ref": "#/$defs/orgPolicyRuleAllowDeny" + }, + "deny": { + "$ref": "#/$defs/orgPolicyRuleAllowDeny" + }, + "enforce": { + "type": "boolean" + }, + "condition": { + "type": "object", + "additionalProperties": false, + "properties": { + "description": { + "type": "string" + }, + "expression": { + "type": "string" + }, + "location": { + "type": "string" + }, + "title": { + "type": "string" + } + } + } + } + }, + "orgPolicyRuleAllowDeny": { + "type": "object", + "additionalProperties": false, + "properties": { + "all": { + "type": "boolean" + }, + "values": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } +} \ No newline at end of file diff --git a/modules/net-vpc-factory/variables.tf b/modules/net-vpc-factory/variables.tf new file mode 100644 index 000000000..efab6f08c --- /dev/null +++ b/modules/net-vpc-factory/variables.tf @@ -0,0 +1,382 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "billing_account" { + description = "Billing account id." + type = string +} + +variable "factories_config" { + description = "Configuration for network resource factories." + type = object({ + vpcs = optional(string, "recipes/hub-and-spoke-ncc") + firewall_policy_name = optional(string, "net-default") + }) + default = { + vpcs = "recipes/hub-and-spoke-ncc" + } +} + +variable "network_project_config" { + description = "Consolidated configuration for project, VPCs and their associated resources." + type = map(object({ + project_config = object({ + name = string + prefix = optional(string) + parent = optional(string) + billing_account = optional(string) + deletion_policy = optional(string, "DELETE") + default_service_account = optional(string, "keep") + auto_create_network = optional(bool, false) + project_create = optional(bool, true) + shared_vpc_host_config = optional(object({ + enabled = bool + service_projects = optional(list(string), []) + })) + services = optional(list(string), ) + org_policies = optional(map(object({ + inherit_from_parent = optional(bool) + reset = optional(bool) + rules = optional(list(object({ + allow = optional(object({ + all = optional(bool) + values = optional(list(string)) + })) + deny = optional(object({ + all = optional(bool) + values = optional(list(string)) + })) + enforce = optional(bool) + condition = optional(object({ + description = optional(string) + expression = optional(string) + location = optional(string) + title = optional(string) + }), {}) + })), ) + })), {}) + metric_scopes = optional(list(string), []) + iam = optional(map(list(string)), {}) + iam_bindings = optional(map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + iam_bindings_additive = optional(map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + iam_by_principals_additive = optional(map(list(string)), {}) + iam_by_principals = optional(map(list(string)), {}) + }) + ncc_hub_config = optional(object({ + name = string + description = optional(string, "Terraform-managed.") + preset_topology = optional(string, "MESH") + export_psc = optional(bool, true) + groups = optional(map(object({ + labels = optional(map(string)) + description = optional(string, "Terraform-managed.") + auto_accept = optional(list(string), []) + }))) + })) + vpc_config = optional(map(object({ + auto_create_subnetworks = optional(bool, false) + create_googleapis_routes = optional(object({ + private = optional(bool, true) + private-6 = optional(bool, false) + restricted = optional(bool, true) + restricted-6 = optional(bool, false) + }), {}) + delete_default_routes_on_create = optional(bool, false) + description = optional(string, "Terraform-managed.") + dns_policy = optional(object({ + inbound = optional(bool) + logging = optional(bool) + outbound = optional(object({ + private_ns = list(string) + public_ns = list(string) + })) + })) + dns_zones = optional(map(object({ + force_destroy = optional(bool) + description = optional(string, "Terraform managed.") + iam = optional(map(list(string)), {}) + zone_config = object({ + domain = string + forwarding = optional(object({ + forwarders = optional(map(string), {}) + client_networks = optional(list(string), ) + })) + peering = optional(object({ + client_networks = optional(list(string), ) + peer_network = string + })) + public = optional(object({ + dnssec_config = optional(object({ + non_existence = optional(string, "nsec3") + state = string + key_signing_key = optional(object( + { algorithm = string, key_length = number }), + { algorithm = "rsasha256", key_length = 2048 } + ) + zone_signing_key = optional(object( + { algorithm = string, key_length = number }), + { algorithm = "rsasha256", key_length = 1024 } + ) + })) + enable_logging = optional(bool, false) + })) + private = optional(object({ + client_networks = optional(list(string), ) + service_directory_namespace = optional(string) + })) + }) + recordsets = optional(map(object({ + ttl = optional(number, 300) + records = optional(list(string)) + geo_routing = optional(list(object({ + location = string + records = optional(list(string)) + health_checked_targets = optional(list(object({ + load_balancer_type = string + ip_address = string + port = string + ip_protocol = string + network_url = string + project = string + region = optional(string) + }))) + }))) + wrr_routing = optional(list(object({ + weight = number + records = list(string) + }))) + })), {}) + }))) + firewall_policy_enforcement_order = optional(string, "AFTER_CLASSIC_FIREWALL") + ipv6_config = optional(object({ + enable_ula_internal = optional(bool) + internal_range = optional(string) + }), {}) + mtu = optional(number) + name = string + nat_config = optional(map(object({ + region = string + router_create = optional(bool, true) + router_name = optional(string) + router_network = optional(string) + router_asn = optional(number) + type = optional(string, "PUBLIC") + addresses = optional(list(string), []) + endpoint_types = optional(list(string)) + logging_filter = optional(string) + config_port_allocation = optional(object({ + enable_endpoint_independent_mapping = optional(bool, true) + enable_dynamic_port_allocation = optional(bool, false) + min_ports_per_vm = optional(number) + max_ports_per_vm = optional(number, 65536) + }), {}) + config_source_subnetworks = optional(object({ + all = optional(bool, true) + primary_ranges_only = optional(bool) + subnetworks = optional(list(object({ + self_link = string + all_ranges = optional(bool, true) + primary_range = optional(bool, false) + secondary_ranges = optional(list(string)) + })), []) + }), {}) + config_timeouts = optional(object({ + icmp = optional(number) + tcp_established = optional(number) + tcp_time_wait = optional(number) + tcp_transitory = optional(number) + udp = optional(number) + }), {}) + rules = optional(list(object({ + description = optional(string) + match = string + source_ips = optional(list(string)) + source_ranges = optional(list(string)) + })), []) + + }))) + network_attachments = optional(map(object({ + subnet = string + automatic_connection = optional(bool, false) + description = optional(string, "Terraform-managed.") + producer_accept_lists = optional(list(string)) + producer_reject_lists = optional(list(string)) + })), {}) + policy_based_routes = optional(map(object({ + description = optional(string, "Terraform-managed.") + labels = optional(map(string)) + priority = optional(number) + next_hop_ilb_ip = optional(string) + use_default_routing = optional(bool, false) + filter = optional(object({ + ip_protocol = optional(string) + dest_range = optional(string) + src_range = optional(string) + }), {}) + target = optional(object({ + interconnect_attachment = optional(string) + tags = optional(list(string)) + }), {}) + })), {}) + psa_config = optional(list(object({ + deletion_policy = optional(string, null) + ranges = map(string) + export_routes = optional(bool, false) + import_routes = optional(bool, false) + peered_domains = optional(list(string), []) + range_prefix = optional(string) + service_producer = optional(string, "servicenetworking.googleapis.com") + })), []) + routers = optional(map(object({ + region = string + asn = optional(number) + custom_advertise = optional(object({ + all_subnets = bool + ip_ranges = map(string) + })) + keepalive = optional(number) + name = optional(string) + }))) + routes = optional(map(object({ + description = optional(string, "Terraform-managed.") + dest_range = string + next_hop_type = string + next_hop = string + priority = optional(number) + tags = optional(list(string)) + })), {}) + routing_mode = optional(string, "GLOBAL") + subnets_factory_config = optional(object({ + context = optional(object({ + regions = optional(map(string), {}) + }), {}) + subnets_folder = optional(string) + }), {}) + firewall_factory_config = optional(object({ + cidr_tpl_file = optional(string) + rules_folder = optional(string) + }), {}) + vpn_config = optional(map(object({ + #TOFIX: are we even using name? + name = string + region = string + ncc_spoke_config = optional(object({ + hub = string + description = string + labels = map(string) + })) + peer_gateways = map(object({ + external = optional(object({ + redundancy_type = string + interfaces = list(string) + description = optional(string, "Terraform managed external VPN gateway") + name = optional(string) + })) + gcp = optional(string) + })) + router_config = object({ + asn = optional(number) + create = optional(bool, true) + custom_advertise = optional(object({ + all_subnets = bool + ip_ranges = map(string) + })) + keepalive = optional(number) + name = optional(string) + override_name = optional(string) + }) + stack_type = optional(string) + tunnels = map(object({ + bgp_peer = object({ + address = string + asn = number + route_priority = optional(number, 1000) + custom_advertise = optional(object({ + all_subnets = bool + ip_ranges = map(string) + })) + md5_authentication_key = optional(object({ + name = string + key = optional(string) + })) + ipv6 = optional(object({ + nexthop_address = optional(string) + peer_nexthop_address = optional(string) + })) + name = optional(string) + }) + # each BGP session on the same Cloud Router must use a unique /30 CIDR + # from the 169.254.0.0/16 block. + bgp_session_range = string + ike_version = optional(number, 2) + name = optional(string) + peer_external_gateway_interface = optional(number) + peer_router_interface_name = optional(string) + peer_gateway = optional(string, "default") + router = optional(string) + shared_secret = optional(string) + vpn_gateway_interface = number + })) + })), {}) + peering_config = optional(map(object({ + peer_network = string + routes_config = optional(object({ + export = optional(bool, true) + import = optional(bool, true) + public_export = optional(bool) + public_import = optional(bool) + } + ), {}) + stack_type = optional(string) + })), {}) + ncc_config = optional(object({ + hub = string + description = optional(string, "Terraform-managed.") + labels = optional(map(string)) + group = optional(string) + exclude_export_ranges = optional(list(string), null) + include_export_ranges = optional(list(string), null) + })) + }))) + })) + default = null +} + +variable "parent_id" { + description = "Root node for the projects created by the factory. Must be either organizations/XXXXXXXX or folders/XXXXXXXX." + type = string +} + +variable "prefix" { + description = "Prefix used for projects." + type = string +} diff --git a/tests/modules/net_vpc_factory/common.tfvars b/tests/modules/net_vpc_factory/common.tfvars new file mode 100644 index 000000000..4abab596e --- /dev/null +++ b/tests/modules/net_vpc_factory/common.tfvars @@ -0,0 +1,3 @@ +billing_account = "123456-789012-345678" +parent_id = "folders/123456789012" +prefix = "myprefix" diff --git a/tests/modules/net_vpc_factory/ncc.tfvars b/tests/modules/net_vpc_factory/ncc.tfvars new file mode 100644 index 000000000..bd034cf87 --- /dev/null +++ b/tests/modules/net_vpc_factory/ncc.tfvars @@ -0,0 +1,2 @@ +factories_config = { vpcs = "recipes/hub-and-spoke-ncc" } + diff --git a/tests/modules/net_vpc_factory/ncc.yaml b/tests/modules/net_vpc_factory/ncc.yaml new file mode 100644 index 000000000..30a070167 --- /dev/null +++ b/tests/modules/net_vpc_factory/ncc.yaml @@ -0,0 +1,40 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +counts: + google_compute_external_vpn_gateway: 1 + google_compute_firewall: 9 + google_compute_ha_vpn_gateway: 1 + google_compute_network: 4 + google_compute_route: 15 + google_compute_router: 3 + google_compute_router_interface: 2 + google_compute_router_nat: 2 + google_compute_router_peer: 2 + google_compute_shared_vpc_host_project: 3 + google_compute_subnetwork: 3 + google_compute_vpn_tunnel: 2 + google_dns_managed_zone: 4 + google_dns_policy: 4 + google_dns_record_set: 1 + google_network_connectivity_group: 1 + google_network_connectivity_hub: 1 + google_network_connectivity_spoke: 3 + google_project: 3 + google_project_iam_member: 21 + google_project_service: 27 + google_project_service_identity: 21 + modules: 17 + random_id: 3 + resources: 136 diff --git a/tests/modules/net_vpc_factory/only_projects.tfvars b/tests/modules/net_vpc_factory/only_projects.tfvars new file mode 100644 index 000000000..8e5157506 --- /dev/null +++ b/tests/modules/net_vpc_factory/only_projects.tfvars @@ -0,0 +1,2 @@ +factories_config = { vpcs = "recipes/only-projects" } + diff --git a/tests/modules/net_vpc_factory/only_projects.yaml b/tests/modules/net_vpc_factory/only_projects.yaml new file mode 100644 index 000000000..e67e4ff6d --- /dev/null +++ b/tests/modules/net_vpc_factory/only_projects.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +counts: + google_project: 3 + google_project_iam_member: 21 + google_project_service: 27 + google_project_service_identity: 21 + modules: 3 + resources: 72 diff --git a/tests/modules/net_vpc_factory/peering.tfvars b/tests/modules/net_vpc_factory/peering.tfvars new file mode 100644 index 000000000..1209220ef --- /dev/null +++ b/tests/modules/net_vpc_factory/peering.tfvars @@ -0,0 +1,2 @@ +factories_config = { vpcs = "recipes/hub-and-spoke-peering" } + diff --git a/tests/modules/net_vpc_factory/peering.yaml b/tests/modules/net_vpc_factory/peering.yaml new file mode 100644 index 000000000..fd734ffcd --- /dev/null +++ b/tests/modules/net_vpc_factory/peering.yaml @@ -0,0 +1,38 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +counts: + google_compute_external_vpn_gateway: 1 + google_compute_firewall: 9 + google_compute_ha_vpn_gateway: 1 + google_compute_network: 4 + google_compute_network_peering: 6 + google_compute_route: 15 + google_compute_router: 4 + google_compute_router_interface: 2 + google_compute_router_nat: 3 + google_compute_router_peer: 2 + google_compute_shared_vpc_host_project: 3 + google_compute_subnetwork: 3 + google_compute_vpn_tunnel: 2 + google_dns_managed_zone: 4 + google_dns_policy: 4 + google_dns_record_set: 1 + google_project: 3 + google_project_iam_member: 21 + google_project_service: 27 + google_project_service_identity: 21 + modules: 18 + random_id: 3 + resources: 139 diff --git a/tests/modules/net_vpc_factory/separate_envs.tfvars b/tests/modules/net_vpc_factory/separate_envs.tfvars new file mode 100644 index 000000000..041cce5ba --- /dev/null +++ b/tests/modules/net_vpc_factory/separate_envs.tfvars @@ -0,0 +1,2 @@ +factories_config = { vpcs = "recipes/separate-envs" } + diff --git a/tests/modules/net_vpc_factory/separate_envs.yaml b/tests/modules/net_vpc_factory/separate_envs.yaml new file mode 100644 index 000000000..f3b1399a0 --- /dev/null +++ b/tests/modules/net_vpc_factory/separate_envs.yaml @@ -0,0 +1,35 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +counts: + google_compute_external_vpn_gateway: 2 + google_compute_firewall: 6 + google_compute_ha_vpn_gateway: 2 + google_compute_network: 2 + google_compute_route: 8 + google_compute_router: 4 + google_compute_router_interface: 2 + google_compute_router_nat: 2 + google_compute_router_peer: 2 + google_compute_shared_vpc_host_project: 2 + google_compute_subnetwork: 2 + google_compute_vpn_tunnel: 2 + google_dns_policy: 2 + google_project: 3 + google_project_iam_member: 21 + google_project_service: 27 + google_project_service_identity: 21 + modules: 11 + random_id: 4 + resources: 114 diff --git a/tests/modules/net_vpc_factory/tftest.yaml b/tests/modules/net_vpc_factory/tftest.yaml new file mode 100644 index 000000000..3c90b73d8 --- /dev/null +++ b/tests/modules/net_vpc_factory/tftest.yaml @@ -0,0 +1,23 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module: modules/net-vpc-factory +common_tfvars: + - common.tfvars +tests: + ncc: + only_projects: + peering: + separate_envs: + vpn: diff --git a/tests/modules/net_vpc_factory/vpn.tfvars b/tests/modules/net_vpc_factory/vpn.tfvars new file mode 100644 index 000000000..0d5c56b95 --- /dev/null +++ b/tests/modules/net_vpc_factory/vpn.tfvars @@ -0,0 +1 @@ +factories_config = { vpcs = "recipes/hub-and-spoke-vpn" } diff --git a/tests/modules/net_vpc_factory/vpn.yaml b/tests/modules/net_vpc_factory/vpn.yaml new file mode 100644 index 000000000..7598121b7 --- /dev/null +++ b/tests/modules/net_vpc_factory/vpn.yaml @@ -0,0 +1,37 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +counts: + google_compute_external_vpn_gateway: 1 + google_compute_firewall: 9 + google_compute_ha_vpn_gateway: 5 + google_compute_network: 4 + google_compute_route: 15 + google_compute_router: 6 + google_compute_router_interface: 10 + google_compute_router_nat: 3 + google_compute_router_peer: 10 + google_compute_shared_vpc_host_project: 3 + google_compute_subnetwork: 3 + google_compute_vpn_tunnel: 10 + google_dns_managed_zone: 4 + google_dns_policy: 4 + google_dns_record_set: 1 + google_project: 3 + google_project_iam_member: 21 + google_project_service: 27 + google_project_service_identity: 21 + modules: 22 + random_id: 15 + resources: 175 From f1a3cac8ca516677064c9d3cce8fa32078324e8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Fri, 11 Apr 2025 08:34:48 +0000 Subject: [PATCH 4/6] Use factory-projects-object to normalize inputs for project module --- modules/net-vpc-factory/factory-project.tf | 23 +- .../factory-projects-object.tf | 257 ++++++++++++++++++ tools/duplicate-diff.py | 4 +- 3 files changed, 267 insertions(+), 17 deletions(-) create mode 100644 modules/net-vpc-factory/factory-projects-object.tf diff --git a/modules/net-vpc-factory/factory-project.tf b/modules/net-vpc-factory/factory-project.tf index 00f783ba3..4e7ae1992 100644 --- a/modules/net-vpc-factory/factory-project.tf +++ b/modules/net-vpc-factory/factory-project.tf @@ -17,22 +17,15 @@ # tfdoc:file:description Dedicated project factory. locals { - projects = { for k, v in local.network_projects : k => merge( - { - billing_account = try(v.project_config.billing_account, var.billing_account) - prefix = try(v.project_config.prefix, var.prefix) - parent = try(v.project_config.parent, var.parent_id) - shared_vpc_host_config = try(v.project_config.shared_vpc_host_config, null) - iam = try(v.project_config.iam, {}) - iam_bindings = try(v.project_config.iam_bindings, {}) - iam_bindings_additive = try(v.project_config.iam_bindings_additive, {}) - iam_by_principals = try(v.project_config.iam_by_principals, {}) - iam_by_principals_additive = try(v.project_config.iam_by_principals_additive, {}) - services = try(v.project_config.services, []) - org_policies = try(v.project_config.org_policies, {}) - }, - v.project_config) + _projects_input = { for k, v in local.network_projects : k => v.project_config } + _projects_config = { + data_defaults = { + billing_account = var.billing_account + prefix = var.prefix + parent = var.parent_id + } } + projects = local._projects_output } module "projects" { diff --git a/modules/net-vpc-factory/factory-projects-object.tf b/modules/net-vpc-factory/factory-projects-object.tf new file mode 100644 index 000000000..3e9c2d714 --- /dev/null +++ b/modules/net-vpc-factory/factory-projects-object.tf @@ -0,0 +1,257 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# inputs +# local._projects_input - parsed data from yaml as map +# local._projects_config = object({ +# data_overrides = ... +# data_defaults = ... +# }) +# outputs: +# local._projects_output - map +locals { + __projects_config = { + data_defaults = merge({ + billing_account = null + contacts = {} + factories_config = merge({ + custom_roles = null + observability = null + org_policies = null + quotas = null + }, try(local._projects_config.data_defaults.factories_config, { + custom_roles = null + observability = null + org_policies = null + quotas = null + }) + ) + labels = {} + metric_scopes = [] + parent = null + prefix = null + service_encryption_key_ids = {} + services = [] + shared_vpc_service_config = merge({ + host_project = null + network_users = [] + service_agent_iam = {} + service_agent_subnet_iam = {} + service_iam_grants = [] + network_subnet_users = {} + }, try(local._projects_config.data_defaults.shared_vpc_service_config, { + host_project = null + network_users = [] + service_agent_iam = {} + service_agent_subnet_iam = {} + service_iam_grants = [] + network_subnet_users = {} + }) + ) + storage_location = null + tag_bindings = {} + service_accounts = {} + vpc_sc = merge({ + perimeter_name = null + perimeter_bridges = [] + is_dry_run = false + }, try(local._projects_config.data_defaults.vpc_sc, { + perimeter_name = null + perimeter_bridges = [] + is_dry_run = false + }) + ) + logging_data_access = {} + }, + try(local._projects_config.data_defaults, {}) + ) + # data_overrides default to null's, to mark that they should not override + data_overrides = merge({ + billing_account = null + contacts = null + factories_config = merge({ + custom_roles = null + observability = null + org_policies = null + quotas = null + }, try(local._projects_config.data_overrides.factories_config, { + custom_roles = null + observability = null + org_policies = null + quotas = null + }) + ) + parent = null + prefix = null + service_encryption_key_ids = null + storage_location = null + tag_bindings = null + services = null + service_accounts = null + vpc_sc = try( + merge( + { + perimeter_name = null + perimeter_bridges = [] + is_dry_run = false + }, + local._projects_config.data_overrides.vpc_sc + ), + null + ) + logging_data_access = null + }, + try(local._projects_config.data_overrides, {}) + ) + } + _projects_output = { + # Semantics of the merges are: + # * if data_overrides. is not null, use this value + # * if _projects_inputs. is not null, use this value + # * use data_default value, which if not set, will provide "empty" type + # This logic is easily implemented using coalesce, even on maps and list and allows to + # set data_overrides. to "", [] or {} to ensure, that empty value is always passed, or do + # the same in _projects_input to prevent falling back to default value + for k, v in local._projects_input : k => merge(v, { + billing_account = try(coalesce( # type: string + local.__projects_config.data_overrides.billing_account, + try(v.billing_account, null), + local.__projects_config.data_defaults.billing_account + ), null) + contacts = coalesce( # type: map + local.__projects_config.data_overrides.contacts, + try(v.contacts, null), + local.__projects_config.data_defaults.contacts + ) + factories_config = { # type: object + custom_roles = try( # type: string + coalesce( + local.__projects_config.data_overrides.factories_config.custom_roles, + try(v.factories_config.custom_roles, null), + local.__projects_config.data_defaults.factories_config.custom_roles + ), + null + ) + observability = try( # type: string + coalesce( + local.__projects_config.data_overrides.factories_config.observability, + try(v.factories_config.observability, null), + local.__projects_config.data_defaults.factories_config.observability + ), + null) + org_policies = try( # type: string + coalesce( + local.__projects_config.data_overrides.factories_config.org_policies, + try(v.factories_config.org_policies, null), + local.__projects_config.data_defaults.factories_config.org_policies + ), + null) + quotas = try( # type: string + coalesce( + local.__projects_config.data_overrides.factories_config.quotas, + try(v.factories_config.quotas, null), + local.__projects_config.data_defaults.factories_config.quotas + ), + null) + } + iam = try(v.iam, {}) # type: map(list(string)) + iam_bindings = try(v.iam_bindings, {}) # type: map(object({...})) + iam_bindings_additive = try(v.iam_bindings_additive, {}) # type: map(object({...})) + iam_by_principals_additive = try(v.iam_by_principals_additive, {}) # type: map(list(string)) + iam_by_principals = try(v.iam_by_principals, {}) # map(list(string)) + labels = coalesce( # type: map(string) + try(v.labels, null), + local.__projects_config.data_defaults.labels + ) + metric_scopes = coalesce( # type: list(string) + try(v.metric_scopes, null), + local.__projects_config.data_defaults.metric_scopes + ) + name = lookup(v, "name", k) # type: string + org_policies = try(v.org_policies, {}) # type: map(object({...})) + parent = try( # type: string, nullable + coalesce( + local.__projects_config.data_overrides.parent, + try(v.parent, null), + local.__projects_config.data_defaults.parent + ), null + ) + prefix = try( # type: string, nullable + coalesce( + local.__projects_config.data_overrides.prefix, + try(v.prefix, null), + local.__projects_config.data_defaults.prefix + ), null + ) + service_encryption_key_ids = coalesce( # type: map(list(string)) + local.__projects_config.data_overrides.service_encryption_key_ids, + try(v.service_encryption_key_ids, null), + local.__projects_config.data_defaults.service_encryption_key_ids + ) + services = coalesce( # type: list(string) + local.__projects_config.data_overrides.services, + try(v.services, null), + local.__projects_config.data_defaults.services + ) + shared_vpc_host_config = ( # type: object({...}) + try(v.shared_vpc_host_config, null) != null + ? merge( + { service_projects = [] }, + v.shared_vpc_host_config + ) + : null + ) + shared_vpc_service_config = ( # type: object({...}) + try(v.shared_vpc_service_config, null) != null + ? merge( + { + host_project = null + network_users = [] + service_agent_iam = {} + service_agent_subnet_iam = {} + service_iam_grants = [] + network_subnet_users = {} + }, + v.shared_vpc_service_config + ) + : local.__projects_config.data_defaults.shared_vpc_service_config + ) + tag_bindings = coalesce( # type: map(string) + local.__projects_config.data_overrides.tag_bindings, + try(v.tag_bindings, null), + local.__projects_config.data_defaults.tag_bindings + ) + vpc_sc = ( # type: object + local.__projects_config.data_overrides.vpc_sc != null + ? local.__projects_config.data_overrides.vpc_sc + : ( + try(v.vpc_sc, null) != null + ? merge({ + perimeter_name = null + perimeter_bridges = [] + is_dry_run = false + }, v.vpc_sc) + : local.__projects_config.data_defaults.vpc_sc + ) + ) + logging_data_access = coalesce( # type: map(object({...})) + local.__projects_config.data_overrides.logging_data_access, + try(v.logging_data_access, null), + local.__projects_config.data_defaults.logging_data_access + ) + }) + } +} diff --git a/tools/duplicate-diff.py b/tools/duplicate-diff.py index 895dcc04a..a87febfc6 100644 --- a/tools/duplicate-diff.py +++ b/tools/duplicate-diff.py @@ -19,9 +19,9 @@ import sys duplicates = [ [ - # "modules/net-vpc-factory/factory-projects-object.tf", - # data factory + "modules/net-vpc-factory/factory-projects-object.tf", "modules/project-factory/factory-projects-object.tf", + # data factory ], ] From a747653e88d1e45cdd8fe4418322fc7afb6a1f08 Mon Sep 17 00:00:00 2001 From: simonebruzzechesse <60114646+simonebruzzechesse@users.noreply.github.com> Date: Fri, 11 Apr 2025 15:14:05 +0200 Subject: [PATCH 5/6] SecOps Anonymization improvements (#3013) * update secops anonymization pipeline with new chronicle APIs --- .../secops-anonymization-pipeline/README.md | 41 ++-- .../secops-anonymization-pipeline/main.tf | 87 ++------- .../source/main.py | 179 +++++++----------- .../source/requirements.txt | 1 + .../source/shared/secops.py | 111 ----------- .../source/shared/utils.py | 39 +--- .../variables.tf | 24 ++- 7 files changed, 137 insertions(+), 345 deletions(-) delete mode 100644 fast/project-templates/secops-anonymization-pipeline/source/shared/secops.py diff --git a/fast/project-templates/secops-anonymization-pipeline/README.md b/fast/project-templates/secops-anonymization-pipeline/README.md index 92c0ca02b..8be973103 100644 --- a/fast/project-templates/secops-anonymization-pipeline/README.md +++ b/fast/project-templates/secops-anonymization-pipeline/README.md @@ -22,13 +22,15 @@ The following diagram illustrates the high-level design of the solution, which c The use case is a SecOps deployment composed of 2 tenants (one for production and one for development/testing). There might be the need to export production data from the prod tenant and import them back in DEV (possibly anonymizing it) for rules and/or parser development, that is why this pipeline might be convenient for speeding up the data migration process. +The solution is based on a custom Python script responsible for implementing the aforementioned logic. The script leverages the new [SecOps API Wrapper](https://github.com/google/secops-wrapper) available also in [PyPi](https://pypi.org/project/secops/). + ### Pipeline Steps -- **SecOps Export**: Triggered via the corresponding TRIGGER-EXPORT action. Call SecOps Export API to trigger raw logs export on a GCS bucket based on either all the log types or one o more of them for a specific time frame. By default, the export will be for the previous day, otherwise the following parameters can be specified to change the time frame: +- **SecOps Export**: Triggered via the corresponding TRIGGER-EXPORT action. Call [SecOps Export API](https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.dataExports) to trigger raw logs export on a GCS bucket based on either all the log types or one o more of them for a specific time frame. By default, the export will be for the previous day, otherwise the following parameters can be specified to change the time frame: * `EXPORT_DATE` date for the export (format %Y-%m-%d) * `EXPORT_START_DATETIME` and `EXPORT_END_DATETIME` start and end datetime for the export (format %Y-%m-%dT%H:%M:%SZ). This is useful for verbose log source with GB/TB of raw logs ingested on a daily basis - **Anonymize Data**: Triggered via the corresponding ANONYMIZE-DATA action. Split the exported CSV files to one or more CSV files where the size of each file is less than 60MB (which is the maximum file size supported by DLP). It also renames those files in .log for better handling by the DLP Job. It will then trigger an asynchronous DLP job to anonymize data. -- **Import Data**: Triggered via the corresponding IMPORT-DATA action. Import the exported raw logs (or anonymized ones according to the pipeline configuration) data into the target SecOps tenant leveraging the [Ingestion API](https://cloud.google.com/chronicle/docs/reference/ingestion-api). +- **Import Data**: Triggered via the corresponding IMPORT-DATA action. Import the exported raw logs (or anonymized ones according to the pipeline configuration) data into the target SecOps tenant leveraging the new [SecOps Ingestion API](https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.logTypes.logs/import). ### Limitations @@ -56,11 +58,11 @@ information (for more precise configuration see the Variables section): * GCP Project ID for SecOps anonymization pipeline deployment * SecOps tenants information: - * GCP projects of SecOps tenants - * customer ID - * deployment region for both the tenants (must be the same) - * SA credentials with export permissions on source tenant - * SA credentials with ingestion API grants on target tenant + * GCP projects of both source and target SecOps tenants + * SecOps customer IDs for both source and target SecOps tenants + * SecOps deployment region for both the tenants (must be the same) + * SecOps Forwarder ID for target tenant (this is mandatory for new ingestion APIs and requires at least an empty collector forwarder to be setup in target tenant) + * **Grant Pipeline SA Chronicle API Editor role on both source and target tenant** (this might be restricred to data export permissions on source and import logs permissions on target tenant) #### Step 2: Prepare the variables @@ -92,19 +94,21 @@ terraform apply #### Step 5: Test solution Test the solution triggering an export from the Cloud Scheduler page, after few hours (accoding to the size of the export) logs should be available on secops-export bucket. Please check for any issue during export using the corresponding APIs and the export ID. + ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [prefix](variables.tf#L59) | Prefix used for resource names. | string | ✓ | | -| [project_id](variables.tf#L69) | Project id, references existing project if `project_create` is null. | string | ✓ | | -| [secops_config](variables.tf#L86) | SecOps config. | object({…}) | ✓ | | +| [project_id](variables.tf#L78) | Project id that references existing project. | string | ✓ | | +| [secops_config](variables.tf#L95) | SecOps config. | object({…}) | ✓ | | | [anonymization_scheduler](variables.tf#L17) | Schedule for triggering export, anonymization and import of data. | object({…}) | | {…} | | [cloud_function_config](variables.tf#L31) | Optional Cloud Function configuration. | object({…}) | | {} | | [dlp_config](variables.tf#L49) | Data Loss prevention configuration. | object({…}) | | null | -| [regions](variables.tf#L74) | Regions: primary for all resources and secondary for clouds scheduler since the latter is available in few regions. | object({…}) | | {…} | -| [skip_anonymization](variables.tf#L103) | Whether to skip anonymization step and just import data exported from source tenant. | bool | | false | +| [project_create_config](variables.tf#L69) | Create project instead of using an existing one. | object({…}) | | null | +| [regions](variables.tf#L83) | Regions: primary for all resources and secondary for clouds scheduler since the latter is available in few regions. | object({…}) | | {…} | +| [skip_anonymization](variables.tf#L111) | Whether to skip anonymization step and just import data exported from source tenant. | bool | | false | ## Outputs @@ -121,22 +125,25 @@ module "test" { region = "europe" alpha_apis_region = "eu" source_tenant = { - gcp_project = "SOURCE_PROJECT_ID" - export_sa_key_base64 = "dGVzdAo=" + gcp_project = "SOURCE_PROJECT_ID" + customer_id = "xxx-xxxxxx-xxxxx" } target_tenant = { - gcp_project = "TARGET_PROJECT_ID" - customer_id = "xxx-xxxxxx-xxxxx" - ingestion_sa_key_base64 = "dGVzdAo=" + gcp_project = "TARGET_PROJECT_ID" + customer_id = "xxx-xxxxxx-xxxxx" + forwarder_id = "xxxxxxxxx" } } skip_anonymization = false prefix = "pre" project_id = "gcp-project-id" + project_create_config = { + billing_account = "12345-ABCDE-12345" + } regions = { primary = "europe-west1" secondary = "europe-west1" } } -# tftest modules=8 resources=54 +# tftest modules=7 resources=49 ``` diff --git a/fast/project-templates/secops-anonymization-pipeline/main.tf b/fast/project-templates/secops-anonymization-pipeline/main.tf index 39afb864d..fc39cc27f 100644 --- a/fast/project-templates/secops-anonymization-pipeline/main.tf +++ b/fast/project-templates/secops-anonymization-pipeline/main.tf @@ -20,13 +20,14 @@ locals { deidentify_template_id = google_data_loss_prevention_deidentify_template.dlp_deidentify_template.0.id inspect_template_id = google_data_loss_prevention_inspect_template.dlp_inspect_template.0.id } : var.dlp_config - secops_anonymization_export_secret_id = "secops-export-secret-json" - secops_anonymization_import_secret_id = "secops-import-secret-json" } module "project" { - source = "../../../modules/project" - name = var.project_id + source = "../../../modules/project" + name = var.project_id + billing_account = try(var.project_create_config.billing_account, null) + parent = try(var.project_create_config.parent, null) + project_reuse = var.project_create_config != null ? null : {} services = concat([ "secretmanager.googleapis.com", "run.googleapis.com", @@ -41,6 +42,7 @@ module "project" { "roles/dlp.reader" = [module.function.service_account_iam_email] "roles/dlp.jobsEditor" = [module.function.service_account_iam_email] "roles/serviceusage.serviceUsageConsumer" = [module.function.service_account_iam_email] + "roles/chronicle.editor" = [module.function.service_account_iam_email] } iam_bindings_additive = { function-log-writer = { @@ -50,45 +52,6 @@ module "project" { } } -module "secrets" { - source = "../../../modules/secret-manager" - project_id = module.project.project_id - secrets = { - (local.secops_anonymization_export_secret_id) = { - locations = [var.regions.primary] - } - (local.secops_anonymization_import_secret_id) = { - locations = [var.regions.primary] - } - } - versions = { - (local.secops_anonymization_export_secret_id) = { - latest = { - enabled = true, - data = base64decode(var.secops_config.source_tenant.export_sa_key_base64) - } - } - (local.secops_anonymization_import_secret_id) = { - latest = { - enabled = true, - data = base64decode(var.secops_config.target_tenant.ingestion_sa_key_base64) - } - } - } - iam = { - (local.secops_anonymization_export_secret_id) = { - "roles/secretmanager.secretAccessor" = [ - "serviceAccount:${module.function.service_account_email}" - ] - } - (local.secops_anonymization_import_secret_id) = { - "roles/secretmanager.secretAccessor" = [ - "serviceAccount:${module.function.service_account_email}" - ] - } - } -} - module "export-bucket" { source = "../../../modules/gcs" project_id = module.project.project_id @@ -177,15 +140,16 @@ module "function" { path = "${path.module}/source" } environment_variables = merge({ - GCP_PROJECT = module.project.project_id - SKIP_ANONYMIZATION = var.skip_anonymization - SECOPS_SOURCE_SA_KEY_SECRET_PATH = "/app/secrets/source/latest" - SECOPS_TARGET_SA_KEY_SECRET_PATH = "/app/secrets/target/latest" - SECOPS_TARGET_CUSTOMER_ID = var.secops_config.target_tenant.customer_id - SECOPS_REGION = var.secops_config.region - SECOPS_ALPHA_APIS_REGION = var.secops_config.alpha_apis_region - SECOPS_EXPORT_BUCKET = module.export-bucket.name - LOG_EXECUTION_ID = "true" + GCP_PROJECT = module.project.project_id + SKIP_ANONYMIZATION = var.skip_anonymization + SECOPS_SOURCE_PROJECT = var.secops_config.source_tenant.gcp_project + SECOPS_TARGET_PROJECT = var.secops_config.target_tenant.gcp_project + SECOPS_SOURCE_CUSTOMER_ID = var.secops_config.source_tenant.customer_id + SECOPS_TARGET_CUSTOMER_ID = var.secops_config.target_tenant.customer_id + SECOPS_TARGET_FORWARDER_ID = var.secops_config.target_tenant.forwarder_id + SECOPS_REGION = var.secops_config.region + SECOPS_EXPORT_BUCKET = module.export-bucket.name + LOG_EXECUTION_ID = "true" }, var.skip_anonymization ? {} : { SECOPS_OUTPUT_BUCKET = module.anonymized-bucket.0.name DLP_DEIDENTIFY_TEMPLATE_ID = local.dlp_config.deidentify_template_id @@ -202,24 +166,7 @@ module "function" { "serviceAccount:${module.scheduler-sa.email}" ] } - secrets = { - "/app/secrets/source" = { - is_volume = true - project_id = module.project.number - secret = local.secops_anonymization_export_secret_id - versions = [ - "latest:latest" - ] - } - "/app/secrets/target" = { - is_volume = true - project_id = module.project.number - secret = local.secops_anonymization_import_secret_id - versions = [ - "latest:latest" - ] - } - } + secrets = {} vpc_connector = ( var.cloud_function_config.vpc_connector == null ? {} diff --git a/fast/project-templates/secops-anonymization-pipeline/source/main.py b/fast/project-templates/secops-anonymization-pipeline/source/main.py index b02f16e50..d7949cef4 100644 --- a/fast/project-templates/secops-anonymization-pipeline/source/main.py +++ b/fast/project-templates/secops-anonymization-pipeline/source/main.py @@ -17,16 +17,13 @@ import json import os import click import logging -import sys import google.cloud.logging -from google.auth.transport.requests import AuthorizedSession -from google.oauth2 import service_account -from shared.secops import SecOpsUtils from jinja2 import Template from shared import utils from google.cloud import dlp_v2 from google.cloud import storage -from datetime import date, timedelta +from datetime import date, timedelta, datetime +from secops import SecOpsClient client = google.cloud.logging.Client() client.setup_logging() @@ -37,79 +34,53 @@ logging.basicConfig( format='[%(levelname)-8s] - %(asctime)s - %(message)s') logging.root.setLevel(logging.DEBUG) -SCOPES = [ - "https://www.googleapis.com/auth/chronicle-backstory", - "https://www.googleapis.com/auth/malachite-ingestion" -] - -# Threshold value in bytes for ingesting the logs to the SecOps. -# SecOps Ingestion API allows the maximum 1MB of payload and we kept 0.5MB as a buffer. -SIZE_THRESHOLD_BYTES = 950000 - SECOPS_REGION = os.environ.get("SECOPS_REGION") -SECOPS_ALPHA_APIS_REGION = os.environ.get("SECOPS_ALPHA_APIS_REGION") GCP_PROJECT_ID = os.environ.get("GCP_PROJECT") SECOPS_EXPORT_BUCKET = os.environ.get("SECOPS_EXPORT_BUCKET") SECOPS_OUTPUT_BUCKET = os.environ.get("SECOPS_OUTPUT_BUCKET") -SECOPS_SOURCE_SA_KEY_SECRET_PATH = os.environ.get( - "SECOPS_SOURCE_SA_KEY_SECRET_PATH") -SECOPS_TARGET_SA_KEY_SECRET_PATH = os.environ.get( - "SECOPS_TARGET_SA_KEY_SECRET_PATH") +SECOPS_SOURCE_PROJECT = os.environ.get("SECOPS_SOURCE_PROJECT") +SECOPS_TARGET_PROJECT = os.environ.get("SECOPS_TARGET_PROJECT") +SECOPS_SOURCE_CUSTOMER_ID = os.environ.get("SECOPS_SOURCE_CUSTOMER_ID") SECOPS_TARGET_CUSTOMER_ID = os.environ.get("SECOPS_TARGET_CUSTOMER_ID") - -SKIP_ANONYMIZATION = False if (os.environ.get( - "SKIP_ANONYMIZATION", "false").lower() == "false") else True +SECOPS_TARGET_FORWARDER_ID = os.environ.get("SECOPS_TARGET_FORWARDER_ID") +SKIP_ANONYMIZATION = False if (os.environ.get("SKIP_ANONYMIZATION", "false").lower() == "false") else True DLP_DEIDENTIFY_TEMPLATE_ID = os.environ.get("DLP_DEIDENTIFY_TEMPLATE_ID") DLP_INSPECT_TEMPLATE_ID = os.environ.get("DLP_INSPECT_TEMPLATE_ID") DLP_REGION = os.environ.get("DLP_REGION") -INGESTION_API_URL = F"https://{SECOPS_REGION}-malachiteingestion-pa.googleapis.com" -URI_UNSTRUCTURED = f"{INGESTION_API_URL}/v2/unstructuredlogentries:batchCreate" - def import_logs(export_date): + client = SecOpsClient() + chronicle = client.chronicle(customer_id=SECOPS_TARGET_CUSTOMER_ID, project_id=SECOPS_TARGET_PROJECT, region=SECOPS_REGION) + storage_client = storage.Client() BUCKET = SECOPS_OUTPUT_BUCKET if not SKIP_ANONYMIZATION else SECOPS_EXPORT_BUCKET bucket = storage_client.bucket(BUCKET) export_ids = utils.get_secops_export_folders_for_date(BUCKET, export_date) - backstory_credentials = service_account.Credentials.from_service_account_file( - SECOPS_TARGET_SA_KEY_SECRET_PATH, scopes=SCOPES) - authed_session = AuthorizedSession(backstory_credentials) for export_id in export_ids: for folder in utils.list_anonymized_folders(BUCKET, export_id): log_type = folder.split("-")[0] for log_file in utils.list_log_files(BUCKET, f"{export_id}/{folder}"): - blob = bucket.blob(log_file) # Directly get the blob object - with blob.open("r") as f: - cur_entries = [] - body = { - "customer_id": SECOPS_TARGET_CUSTOMER_ID, - "log_type": log_type, - "entries": cur_entries - } - size_of_empty_payload = sys.getsizeof(json.dumps(body)) - for line in f: - next_entries = cur_entries + [{"logText": line.rstrip('\n')}] - if size_of_empty_payload + sys.getsizeof( - json.dumps(next_entries)) >= SIZE_THRESHOLD_BYTES: - body["entries"] = cur_entries - LOGGER.debug(body) - LOGGER.debug(sys.getsizeof(json.dumps(body))) - response = authed_session.post(URI_UNSTRUCTURED, json=body) - LOGGER.debug(response) - cur_entries = [{"logText": line.rstrip('\n')}] - else: - cur_entries.append({"logText": line.rstrip('\n')}) + try: + blob = bucket.blob(log_file) # Directly get the blob object + with blob.open("r") as f: + logs = [] + for line in f: + logs.append(line.rstrip('\n')) + if len(logs) == 1000: + response = chronicle.ingest_log(log_message=logs, log_type=log_type, forwarder_id=SECOPS_TARGET_FORWARDER_ID) + LOGGER.debug(response) + logs = [] - # Send any remaining entries - if cur_entries: - body["entries"] = cur_entries - LOGGER.debug(sys.getsizeof(json.dumps(body))) - LOGGER.debug(body) - response = authed_session.post(URI_UNSTRUCTURED, json=body) - LOGGER.debug(response) + # Send any remaining entries + if len(logs) > 0: + response = chronicle.ingest_log(log_message=logs, log_type=log_type, forwarder_id=SECOPS_TARGET_FORWARDER_ID) + LOGGER.debug(response) + except Exception as e: + LOGGER.error(f"Error during log ingestion': {e}") + raise SystemExit(f'Error during log ingestion: {e}') # delete both export and anonymized buckets after ingesting logs utils.delete_folder(BUCKET, export_id) @@ -120,7 +91,7 @@ def import_logs(export_date): def trigger_export(export_date: str, export_start_datetime: str, - export_end_datetime: str, log_types: list): + export_end_datetime: str, log_types: str): """ Trigger secops export using Data Export API for a specific date :param secops_source_sa_key_secret_path: @@ -133,36 +104,35 @@ def trigger_export(export_date: str, export_start_datetime: str, :param date: datetime (as string) with DD-MM-YYYY format :return: """ - backstory_credentials = service_account.Credentials.from_service_account_file( - SECOPS_SOURCE_SA_KEY_SECRET_PATH, scopes=SCOPES) - secops_utils = SecOpsUtils(backstory_credentials) + + client = SecOpsClient() + chronicle = client.chronicle(customer_id=SECOPS_SOURCE_CUSTOMER_ID, project_id=SECOPS_SOURCE_PROJECT, region=SECOPS_REGION) export_ids = [] + + if export_start_datetime and export_end_datetime: + start_time, end_time = datetime.strptime(export_start_datetime, "%Y-%m-%dT%H:%M:%SZ"), datetime.strptime(export_end_datetime, "%Y-%m-%dT%H:%M:%SZ") + else: + start_time, end_time = utils.format_date_time_range(date_input=export_date) + gcs_bucket = f"projects/{GCP_PROJECT_ID}/buckets/{SECOPS_EXPORT_BUCKET}" + try: - if log_types is None: - export_response = secops_utils.create_data_export( - project=GCP_PROJECT_ID, export_date=export_date, - export_start_datetime=export_start_datetime, - export_end_datetime=export_end_datetime) + if log_types is None or log_types == "": + export_response = chronicle.create_data_export(start_time=start_time, end_time=end_time, gcs_bucket=gcs_bucket, export_all_logs=True) LOGGER.info(export_response) - export_ids.append(export_response["dataExportId"]) - LOGGER.info( - f"Triggered export with ID: {export_response['dataExportId']}") + export_id = export_response["dataExportStatus"]["name"].split("/")[-1] + export_ids.append(export_id) + LOGGER.info(f"Triggered export with ID: {export_id}") else: - for log_type in log_types: - export_response = secops_utils.create_data_export( - project=GCP_PROJECT_ID, export_date=export_date, - export_start_datetime=export_start_datetime, - export_end_datetime=export_end_datetime, log_type=log_type) - LOGGER.info(export_response) - export_ids.append(export_response["dataExportId"]) - LOGGER.info( - f"Triggered export with ID: {export_response['dataExportId']}") + for log_type in log_types.split(","): + export_response = chronicle.create_data_export(start_time=start_time, end_time=end_time, gcs_bucket=gcs_bucket, log_type=log_type) + export_id = export_response["dataExportStatus"]["name"].split("/")[-1] + export_ids.append(export_id) + LOGGER.info(f"Triggered export with ID: {export_id}") except Exception as e: LOGGER.error(f"Error during export': {e}") raise SystemExit(f'Error during secops export: {e}') - LOGGER.info(f"Export IDs: {export_response['dataExportId']}") return export_ids @@ -172,19 +142,21 @@ def anonymize_data(export_date): :param export_date: date for which data should be anonymized :return: """ - backstory_credentials = service_account.Credentials.from_service_account_file( - SECOPS_SOURCE_SA_KEY_SECRET_PATH, scopes=SCOPES) - secops_utils = SecOpsUtils(backstory_credentials) - export_ids = utils.get_secops_export_folders_for_date(SECOPS_EXPORT_BUCKET, - export_date=export_date) + + client = SecOpsClient() + chronicle = client.chronicle(customer_id=SECOPS_SOURCE_CUSTOMER_ID, project_id=SECOPS_SOURCE_PROJECT, region=SECOPS_REGION) + export_ids = utils.get_secops_export_folders_for_date(SECOPS_EXPORT_BUCKET, export_date=export_date) export_finished = True for export_id in export_ids: - export = secops_utils.get_data_export(export_id=export_id) - export_state = export["dataExportStatus"]["stage"] - LOGGER.info(f"Export status: {export_state}.") - if export_state != "FINISHED_SUCCESS": + export = chronicle.get_data_export(data_export_id=export_id) + LOGGER.info(f"Export response: {export}.") + if "dataExportStatus"in export and export["dataExportStatus"]["stage"] == "FINISHED_SUCCESS": + export_state = export["dataExportStatus"]["stage"] + LOGGER.info(f"Export status: {export_state}.") + else: export_finished = False + break if export_finished: for export_id in export_ids: @@ -209,10 +181,13 @@ def anonymize_data(export_date): "inspect_job": dlp_job } - dlp_client = dlp_v2.DlpServiceClient( - client_options={'quota_project_id': GCP_PROJECT_ID}) - response = dlp_client.create_dlp_job(request=job_request) - LOGGER.info(response) + try: + dlp_client = dlp_v2.DlpServiceClient(client_options={'quota_project_id': GCP_PROJECT_ID}) + response = dlp_client.create_dlp_job(request=job_request) + LOGGER.info(response) + except Exception as e: + LOGGER.error(f"Error during export': {e}") + raise SystemExit(f'Error during secops export: {e}') else: LOGGER.error("Export is not finished yet, please try again later.") @@ -259,21 +234,13 @@ def main(request): @click.command() -@click.option('--export-date', '-d', required=False, type=str, - help='Date for secops export and anonymization.') -@click.option('--export-start-datetime', '-d', required=False, type=str, - help='Start datetime for secops export and anonymization.') -@click.option('--export-end-datetime', '-d', required=False, type=str, - help='End datetime for secops export and anonymization.') +@click.option('--export-date', '-d', required=False, type=str, help='Date for secops export and anonymization.') +@click.option('--export-start-datetime', '-d', required=False, type=str, help='Start datetime for secops export and anonymization.') +@click.option('--export-end-datetime', '-d', required=False, type=str, help='End datetime for secops export and anonymization.') @click.option('--log-type', type=str, multiple=True) -@click.option( - '--action', - type=click.Choice(['TRIGGER-EXPORT', 'ANONYMIZE-DATA', - 'IMPORT-DATA']), required=True) -@click.option('--debug', is_flag=True, default=False, - help='Turn on debug logging.') -def main_cli(export_date, export_start_datetime, export_end_datetime, - log_type: list, action: str, debug=False): +@click.option('--action', type=click.Choice(['TRIGGER-EXPORT', 'ANONYMIZE-DATA', 'IMPORT-DATA']), required=True) +@click.option('--debug', is_flag=True, default=False, help='Turn on debug logging.') +def main_cli(export_date, export_start_datetime, export_end_datetime, log_type: list, action: str, debug=False): """ CLI entry point. :param date: date for secops export and anonymization @@ -286,7 +253,7 @@ def main_cli(export_date, export_start_datetime, export_end_datetime, trigger_export(export_date=export_date, export_start_datetime=export_start_datetime, export_end_datetime=export_end_datetime, - log_types=log_type) + log_types=','.join(log_type)) case "ANONYMIZE-DATA": anonymize_data(export_date=export_date) case "IMPORT-DATA": diff --git a/fast/project-templates/secops-anonymization-pipeline/source/requirements.txt b/fast/project-templates/secops-anonymization-pipeline/source/requirements.txt index a61d7ec92..c82eef8f5 100644 --- a/fast/project-templates/secops-anonymization-pipeline/source/requirements.txt +++ b/fast/project-templates/secops-anonymization-pipeline/source/requirements.txt @@ -23,3 +23,4 @@ google-cloud-storage click==8.1.3 google-cloud-dlp google-cloud-logging +secops \ No newline at end of file diff --git a/fast/project-templates/secops-anonymization-pipeline/source/shared/secops.py b/fast/project-templates/secops-anonymization-pipeline/source/shared/secops.py deleted file mode 100644 index 21df22859..000000000 --- a/fast/project-templates/secops-anonymization-pipeline/source/shared/secops.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import google.auth -import logging -import requests -import os -from . import utils -from google.auth.transport.requests import AuthorizedSession -"""SecOps utility functions.""" - -LOGGER = logging.getLogger("secops") -SECOPS_REGION = os.environ.get("SECOPS_REGION") -SECOPS_EXPORT_BUCKET = os.environ.get("SECOPS_EXPORT_BUCKET") -SECOPS_OUTPUT_BUCKET = os.environ.get("SECOPS_OUTPUT_BUCKET") - - -class SecOpsUtils: - - def __init__(self, credentials=None): - self.BACKSTORY_API_URL = f"https://{SECOPS_REGION}-backstory.googleapis.com/v1/tools/dataexport" - self.INGESTION_API_URL = F"https://{SECOPS_REGION}-malachiteingestion-pa.googleapis.com" - self.HTTP = AuthorizedSession(credentials=credentials if credentials - is not None else google.auth.default()[0]) - - def create_data_export(self, project, export_date, export_start_datetime, - export_end_datetime, log_type: str = None): - """ - Trigger Chronicle data export for the given date and log types. - - :param export_start_datetime: - :param export_date: - :param project: - :param session: auth session for API call - :param date: date for which data will be exported - :return: Chronicle Data export response. - """ - if export_start_datetime and export_end_datetime: - start_time, end_time = export_start_datetime, export_end_datetime - else: - start_time, end_time = utils.format_date_time_range( - date_input=export_date) - gcs_bucket = f"projects/{project}/buckets/{SECOPS_EXPORT_BUCKET}" - - body = { - "startTime": start_time, - "endTime": end_time, - "logType": "ALL_TYPES" if log_type is None else log_type, - "gcsBucket": gcs_bucket, - } - - response = self.HTTP.post(self.BACKSTORY_API_URL, json=body) - response.raise_for_status() - print(f"Data export created successfully.") - return response.json() - - def get_data_export(self, export_id: str) -> str: - """ - Get Chronicle data export information. - - :param export_id: ID of Chronicle export to get information from - :return: Data Export status - :raises requests.exceptions.HTTPError: If the API request fails. - """ - try: - response = self.HTTP.get(f"{self.BACKSTORY_API_URL}/{export_id}") - response.raise_for_status( - ) # Raise HTTPError for bad responses (4xx or 5xx) - print( - f"Data export for '{export_id}' retrieved, content is {response.json()}" - ) - return response.json() - except requests.exceptions.HTTPError as e: - print(f"Error fetching data export '{export_id}': {e}") - # You can choose to handle the error in a more specific way here, - # like retrying the request, logging the error, or raising a custom exception. - raise # Re-raise the exception to be handled by the caller - - def list_log_types(self, date): - start_date, end_date = utils.format_date_time_range(date) - params = { - "startTime": start_date, - "endTime": end_date, - } - response = self.HTTP.get(f"{self.BACKSTORY_API_URL}/listavailablelogtypes") - response.raise_for_status() - if response.status_code == 200: - logging.info(f"Log types for date: {date} is {response.json()}") - log_types = response.json()["availableLogTypes"] - else: - error_message = response.json().get("error", - {}).get("message", "Unknown error") - status_code = response.status_code - logging.error( - f"Error listing log types on {date} (Status code: {status_code}) Error message: {error_message}" - ) - raise Exception("Error while listing log types.") - - return log_types diff --git a/fast/project-templates/secops-anonymization-pipeline/source/shared/utils.py b/fast/project-templates/secops-anonymization-pipeline/source/shared/utils.py index 8da1bf1b9..e14f9efcc 100644 --- a/fast/project-templates/secops-anonymization-pipeline/source/shared/utils.py +++ b/fast/project-templates/secops-anonymization-pipeline/source/shared/utils.py @@ -17,7 +17,7 @@ import os import logging import math import csv -from google.cloud import secretmanager, storage, exceptions +from google.cloud import storage from datetime import datetime, timedelta, timezone, time LOGGER = logging.getLogger('secops') @@ -25,22 +25,6 @@ LOGGER = logging.getLogger('secops') MAX_FILE_SIZE = 61440000 # Max size supported by DLP -def get_value_from_secret_manager(resource_path: str) -> str: - """Retrieve the value of the secret from the Google Cloud Secret Manager. - - Args: - resource_path (str): Path of the secret with version included. Ex.: - "projects//secrets//versions/1", - "projects//secrets//versions/latest" - - Returns: - str: Payload for secret. - """ - client = secretmanager.SecretManagerServiceClient() - response = client.access_secret_version(name=resource_path) - return response.payload.data.decode("UTF-8") - - def format_date_time_range(date_input): """ Creates datetime objects for the beginning and end of the input date @@ -56,15 +40,10 @@ def format_date_time_range(date_input): """ date_obj = datetime.strptime(date_input, "%Y-%m-%d") - start_of_day = datetime.combine(date_obj.date(), time.min, - tzinfo=timezone.utc) + start_of_day = datetime.combine(date_obj.date(), time.min,tzinfo=timezone.utc) end_of_day = start_of_day + timedelta(days=1, seconds=-1) - # Format both datetime objects - formatted_start = start_of_day.strftime("%Y-%m-%dT%H:%M:%SZ") - formatted_end = end_of_day.strftime("%Y-%m-%dT%H:%M:%SZ") - - return formatted_start, formatted_end + return start_of_day, end_of_day def list_anonymized_folders(bucket_name, folder_name): @@ -94,16 +73,10 @@ def delete_folder(bucket_name, folder_name): bucket_name: The name of the bucket. folder_name: The name of the folder to delete. """ - storage_client = storage.Client() bucket = storage_client.bucket(bucket_name) - - # List all blobs with the given prefix (folder name) blobs = list(bucket.list_blobs(prefix=folder_name)) - - # Delete the blobs in parallel bucket.delete_blobs(blobs) - print(f"Folder {folder_name} deleted from bucket {bucket_name}") @@ -210,9 +183,9 @@ def get_secops_export_folders_for_date(bucket_name, export_date): export_ids = [] for blob in storage_client.list_blobs(bucket_name): - if blob.time_created.strftime( - "%Y-%m-%d") == export_date and blob.name.split( - '/')[0] not in export_ids: + if "_$folder$" in blob.name: + continue + if blob.time_created.strftime("%Y-%m-%d") == export_date and blob.name.split('/')[0] not in export_ids: export_ids.append(blob.name.split('/')[0]) return export_ids diff --git a/fast/project-templates/secops-anonymization-pipeline/variables.tf b/fast/project-templates/secops-anonymization-pipeline/variables.tf index 587baaedc..56d8c323c 100644 --- a/fast/project-templates/secops-anonymization-pipeline/variables.tf +++ b/fast/project-templates/secops-anonymization-pipeline/variables.tf @@ -66,8 +66,17 @@ variable "prefix" { } } +variable "project_create_config" { + description = "Create project instead of using an existing one." + type = object({ + billing_account = string + parent = optional(string) + }) + default = null +} + variable "project_id" { - description = "Project id, references existing project if `project_create` is null." + description = "Project id that references existing project." type = string } @@ -86,16 +95,15 @@ variable "regions" { variable "secops_config" { description = "SecOps config." type = object({ - region = string - alpha_apis_region = string + region = string source_tenant = object({ - gcp_project = string - export_sa_key_base64 = string + customer_id = string + gcp_project = string }) target_tenant = object({ - gcp_project = string - customer_id = string - ingestion_sa_key_base64 = string + gcp_project = string + customer_id = string + forwarder_id = string }) }) } From 03db2e45cf923d57f0fc26c4b7cefebe268e34d7 Mon Sep 17 00:00:00 2001 From: simonebruzzechesse <60114646+simonebruzzechesse@users.noreply.github.com> Date: Fri, 11 Apr 2025 15:44:31 +0200 Subject: [PATCH 6/6] secops-rules module (#3023) * secops-rules module --- modules/secops-rules/README.md | 217 ++++++++++++++++++ modules/secops-rules/main.tf | 72 ++++++ modules/secops-rules/variables.tf | 72 ++++++ modules/secops-rules/versions.tf | 35 +++ modules/secops-rules/versions.tofu | 35 +++ tests/examples/variables.tf | 7 + .../modules/secops_rules/examples/basic.yaml | 76 ++++++ .../secops_rules/examples/factory.yaml | 76 ++++++ 8 files changed, 590 insertions(+) create mode 100644 modules/secops-rules/README.md create mode 100644 modules/secops-rules/main.tf create mode 100644 modules/secops-rules/variables.tf create mode 100644 modules/secops-rules/versions.tf create mode 100644 modules/secops-rules/versions.tofu create mode 100644 tests/modules/secops_rules/examples/basic.yaml create mode 100644 tests/modules/secops_rules/examples/factory.yaml diff --git a/modules/secops-rules/README.md b/modules/secops-rules/README.md new file mode 100644 index 000000000..e99aa3858 --- /dev/null +++ b/modules/secops-rules/README.md @@ -0,0 +1,217 @@ +# SecOps Rules + +This module allows creation and management of [custom rules](https://cloud.google.com/chronicle/docs/detection/view-all-rules) as well as [reference lists](https://cloud.google.com/chronicle/docs/reference/reference-lists) in Google SecOps. + +- rule definition (yaral code) and reference list entries are managed as files in data folder as per the `factories_config` variable and sample code +- rule and reference list deployments can leverage both `rules_config` and `reference_lists_config` variables or YAML file still specified in the `factories_config` variable. + + +- [Examples](#examples) + - [Sample SecOps Rules and reference list deployment](#sample-secops-rules-and-reference-list-deployment) + - [SecOps Rules Factory](#secops-rules-factory) +- [Variables](#variables) + + +## Examples + +### Sample SecOps Rules and reference list deployment + +This is a sample usage of the secops-rules module for deploying a rule (network_traffic_to_specific_country) and a reference list (private_ip_ranges), definition of the rule in yaral is available in the corresponding file in the `data/rules` folder and the reference list in the `data/reference_lists` folder. Deployment configuration for both is passed as an input to the module using the `rules_config` and `reference_lists_config` variables. + +```hcl +module "secops" { + source = "./fabric/modules/secops-rules" + project_id = var.project_id + tenant_config = var.secops_tenant_config + reference_lists_config = { + "private_ip_ranges" = { + description = "Private CIDR ranges" + type = "CIDR" + } + } + rules_config = { + "network_traffic_to_specific_country" = { + enabled = true + alerting = true + archived = false + run_frequency = "LIVE" + } + } + factories_config = { + rules_defs = "./data/rules" + reference_lists_defs = "./data/reference_lists" + } +} +# tftest modules=1 resources=3 files=reference,rule inventory=basic.yaml +``` + +``` +rule network_traffic_to_specific_country { + +meta: + author = "Google Cloud Security" + description = "Identify network traffic based on target country" + type = "alert" + tags = "geoip enrichment" + data_source = "microsoft windows events" + severity = "Low" + priority = "Low" + +events: + $network.metadata.event_type = "NETWORK_CONNECTION" + //Specify a country of interest to monitor or add additional countries using an or statement + $network.target.ip_geo_artifact.location.country_or_region = "France" nocase + $network.target.ip = $ip + +match: + $ip over 30m + +outcome: + $risk_score = max(35) + $event_count = count_distinct($network.metadata.id) + + // added to populate alert graph with additional context + $principal_ip = array_distinct($network.principal.ip) + + // Commented out target.ip because it is already represented in graph as match variable. If match changes, can uncomment to add to results + //$target_ip = array_distinct($network.target.ip) + $principal_process_pid = array_distinct($network.principal.process.pid) + $principal_process_command_line = array_distinct($network.principal.process.command_line) + $principal_process_file_sha256 = array_distinct($network.principal.process.file.sha256) + $principal_process_file_full_path = array_distinct($network.principal.process.file.full_path) + $principal_process_product_specfic_process_id = array_distinct($network.principal.process.product_specific_process_id) + $principal_process_parent_process_product_specfic_process_id = array_distinct($network.principal.process.parent_process.product_specific_process_id) + $target_process_pid = array_distinct($network.target.process.pid) + $target_process_command_line = array_distinct($network.target.process.command_line) + $target_process_file_sha256 = array_distinct($network.target.process.file.sha256) + $target_process_file_full_path = array_distinct($network.target.process.file.full_path) + $target_process_product_specfic_process_id = array_distinct($network.target.process.product_specific_process_id) + $target_process_parent_process_product_specfic_process_id = array_distinct($network.target.process.parent_process.product_specific_process_id) + $principal_user_userid = array_distinct($network.principal.user.userid) + $target_user_userid = array_distinct($network.target.user.userid) + +condition: + $network +} +# tftest-file id=rule path=data/rules/network_traffic_to_specific_country.yaral +``` + +``` +10.0.0.0/8 +172.16.0.0/12 +192.168.0.0/16 +127.0.0.1/32 +::1/128 +fc00::/7 +fe80::/10 +# tftest-file id=reference path=data/reference_lists/private_ip_ranges.txt +``` + +### SecOps Rules Factory + +The module includes a secops rules and reference list factory (see [Resource Factories](../../blueprints/factories/)) for the configuration of rules and reference lists leveraging YAML configuration files. Each configuration file for rules and reference lists contains more than one rule with a structure that reflects the `rules_config` and `reference_lists_config` variables. Again rules and reference list definition is available in the corresponding yaral and txt files in the data folder. + +```hcl +module "secops" { + source = "./fabric/modules/secops-rules" + project_id = var.project_id + tenant_config = var.secops_tenant_config + factories_config = { + rules = "./secops_rules.yaml" + rules_defs = "./data/rules" + reference_lists = "./secops_reference_lists.yaml" + reference_lists_defs = "./data/reference_lists" + } +} +# tftest modules=1 resources=3 files=1,2,reference,rule inventory=factory.yaml + +``` + +```yaml +network_traffic_to_specific_country: + enabled: true + alerting: true + archived: false + run_frequency: "DAILY" +# tftest-file id=1 path=secops_rules.yaml +``` + +```yaml +private_ip_ranges: + description: "Private CIDR ranges" + type: CIDR # either CIDR, STRING, REGEX +# tftest-file id=2 path=secops_reference_lists.yaml +``` + +``` +rule network_traffic_to_specific_country { + +meta: + author = "Google Cloud Security" + description = "Identify network traffic based on target country" + type = "alert" + tags = "geoip enrichment" + data_source = "microsoft windows events" + severity = "Low" + priority = "Low" + +events: + $network.metadata.event_type = "NETWORK_CONNECTION" + //Specify a country of interest to monitor or add additional countries using an or statement + $network.target.ip_geo_artifact.location.country_or_region = "France" nocase + $network.target.ip = $ip + +match: + $ip over 30m + +outcome: + $risk_score = max(35) + $event_count = count_distinct($network.metadata.id) + + // added to populate alert graph with additional context + $principal_ip = array_distinct($network.principal.ip) + + // Commented out target.ip because it is already represented in graph as match variable. If match changes, can uncomment to add to results + //$target_ip = array_distinct($network.target.ip) + $principal_process_pid = array_distinct($network.principal.process.pid) + $principal_process_command_line = array_distinct($network.principal.process.command_line) + $principal_process_file_sha256 = array_distinct($network.principal.process.file.sha256) + $principal_process_file_full_path = array_distinct($network.principal.process.file.full_path) + $principal_process_product_specfic_process_id = array_distinct($network.principal.process.product_specific_process_id) + $principal_process_parent_process_product_specfic_process_id = array_distinct($network.principal.process.parent_process.product_specific_process_id) + $target_process_pid = array_distinct($network.target.process.pid) + $target_process_command_line = array_distinct($network.target.process.command_line) + $target_process_file_sha256 = array_distinct($network.target.process.file.sha256) + $target_process_file_full_path = array_distinct($network.target.process.file.full_path) + $target_process_product_specfic_process_id = array_distinct($network.target.process.product_specific_process_id) + $target_process_parent_process_product_specfic_process_id = array_distinct($network.target.process.parent_process.product_specific_process_id) + $principal_user_userid = array_distinct($network.principal.user.userid) + $target_user_userid = array_distinct($network.target.user.userid) + +condition: + $network +} +# tftest-file id=rule path=data/rules/network_traffic_to_specific_country.yaral +``` + +``` +10.0.0.0/8 +172.16.0.0/12 +192.168.0.0/16 +127.0.0.1/32 +::1/128 +fc00::/7 +fe80::/10 +# tftest-file id=reference path=data/reference_lists/private_ip_ranges.txt +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [project_id](variables.tf#L29) | Project used for resources. | string | ✓ | | +| [tenant_config](variables.tf#L66) | SecOps Tenant configuration. | object({…}) | ✓ | | +| [factories_config](variables.tf#L17) | Paths to YAML config expected in 'rules' and 'reference_lists'. Path to folders containing rules definitions (yaral files) and reference lists content (txt files) for the corresponding _defs keys. | object({…}) | | {} | +| [reference_lists_config](variables.tf#L34) | SecOps Reference lists configuration. | map(object({…})) | | {} | +| [rules_config](variables.tf#L49) | SecOps Detection rules configuration. | map(object({…})) | | {} | + diff --git a/modules/secops-rules/main.tf b/modules/secops-rules/main.tf new file mode 100644 index 000000000..b76191bb4 --- /dev/null +++ b/modules/secops-rules/main.tf @@ -0,0 +1,72 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + reference_lists = try(yamldecode(file(var.factories_config.reference_lists)), var.reference_lists_config) + reference_lists_entries = { + for k, v in local.reference_lists : k => split("\n", file("${var.factories_config.reference_lists_defs}/${k}.txt")) + } + reference_list_type_mapping = { + STRING = "REFERENCE_LIST_SYNTAX_TYPE_PLAIN_TEXT_STRING" + REGEX = "REFERENCE_LIST_SYNTAX_TYPE_REGEX" + CIDR = "REFERENCE_LIST_SYNTAX_TYPE_CIDR" + } + secops_rules = { + for file_name in fileset(var.factories_config.rules_defs, "*.yaral") : + replace(file_name, ".yaral", "") => file("${var.factories_config.rules_defs}/${file_name}") + } + secops_rule_deployment = try(yamldecode(file(var.factories_config.rules)), var.rules_config) +} + +resource "google_chronicle_reference_list" "default" { + for_each = local.reference_lists + project = var.project_id + location = var.tenant_config.region + instance = var.tenant_config.customer_id + reference_list_id = each.key + description = each.value.description + dynamic "entries" { + for_each = local.reference_lists_entries[each.key] + content { + value = entries.value + } + } + syntax_type = local.reference_list_type_mapping[each.value.type] +} + +resource "google_chronicle_rule" "default" { + for_each = local.secops_rule_deployment + project = var.project_id + location = var.tenant_config.region + instance = var.tenant_config.customer_id + text = local.secops_rules[each.key] + deletion_policy = "FORCE" + depends_on = [ + google_chronicle_reference_list.default + ] +} + +resource "google_chronicle_rule_deployment" "default" { + for_each = local.secops_rule_deployment + project = var.project_id + location = var.tenant_config.region + instance = var.tenant_config.customer_id + rule = google_chronicle_rule.default[each.key].rule_id + enabled = each.value.enabled + alerting = each.value.alerting + archived = each.value.archived + run_frequency = each.value.run_frequency +} diff --git a/modules/secops-rules/variables.tf b/modules/secops-rules/variables.tf new file mode 100644 index 000000000..163e29865 --- /dev/null +++ b/modules/secops-rules/variables.tf @@ -0,0 +1,72 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "factories_config" { + description = "Paths to YAML config expected in 'rules' and 'reference_lists'. Path to folders containing rules definitions (yaral files) and reference lists content (txt files) for the corresponding _defs keys." + type = object({ + rules = optional(string) + rules_defs = optional(string, "data/rules") + reference_lists = optional(string) + reference_lists_defs = optional(string, "data/reference_lists") + }) + nullable = false + default = {} +} + +variable "project_id" { + description = "Project used for resources." + type = string +} + +variable "reference_lists_config" { + description = "SecOps Reference lists configuration." + type = map(object({ + description = string + type = string + })) + default = {} + validation { + condition = alltrue([ + for config in var.reference_lists_config : contains(["CIDR", "STRING", "REGEX"], config.type) + ]) + error_message = "The 'type' attribute for each reference list must be one of: CIDR, STRING, REGEX." + } +} + +variable "rules_config" { + description = "SecOps Detection rules configuration." + type = map(object({ + enabled = optional(bool, true) + alerting = optional(bool, false) + archived = optional(bool, false) + run_frequency : optional(string) + })) + default = {} + validation { + condition = alltrue([ + for config in var.rules_config : contains(["LIVE", "HOURLY", "DAILY"], config.run_frequency) + ]) + error_message = "The 'type' attribute for each reference list must be one of: CIDR, STRING, REGEX." + } +} + +variable "tenant_config" { + description = "SecOps Tenant configuration." + type = object({ + customer_id = string + region = string + }) +} diff --git a/modules/secops-rules/versions.tf b/modules/secops-rules/versions.tf new file mode 100644 index 000000000..469fa2ddb --- /dev/null +++ b/modules/secops-rules/versions.tf @@ -0,0 +1,35 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fabric release: v38.1.0 + +terraform { + required_version = ">= 1.10.2" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.28.0, < 7.0.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.28.0, < 7.0.0" # tftest + } + } + provider_meta "google" { + module_name = "google-pso-tool/cloud-foundation-fabric/path:v38.1.0-tf" + } + provider_meta "google-beta" { + module_name = "google-pso-tool/cloud-foundation-fabric/path:v38.1.0-tf" + } +} diff --git a/modules/secops-rules/versions.tofu b/modules/secops-rules/versions.tofu new file mode 100644 index 000000000..33635f966 --- /dev/null +++ b/modules/secops-rules/versions.tofu @@ -0,0 +1,35 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fabric release: v38.1.0 + +terraform { + required_version = ">= 1.9.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.28.0, < 7.0.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.28.0, < 7.0.0" # tftest + } + } + provider_meta "google" { + module_name = "google-pso-tool/cloud-foundation-fabric/path:v38.1.0-tofu" + } + provider_meta "google-beta" { + module_name = "google-pso-tool/cloud-foundation-fabric/path:v38.1.0-tofu" + } +} diff --git a/tests/examples/variables.tf b/tests/examples/variables.tf index 80f76c234..6ba4c363f 100644 --- a/tests/examples/variables.tf +++ b/tests/examples/variables.tf @@ -61,6 +61,13 @@ variable "regions" { } } +variable "secops_tenant_config" { + default = { + customer_id = "customer-id" + region = "europe" + } +} + variable "service_account" { default = { id = "service_account_id" diff --git a/tests/modules/secops_rules/examples/basic.yaml b/tests/modules/secops_rules/examples/basic.yaml new file mode 100644 index 000000000..9c9f6d423 --- /dev/null +++ b/tests/modules/secops_rules/examples/basic.yaml @@ -0,0 +1,76 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.secops.google_chronicle_reference_list.default["private_ip_ranges"]: + description: Private CIDR ranges + entries: + - value: 10.0.0.0/8 + - value: 172.16.0.0/12 + - value: 192.168.0.0/16 + - value: 127.0.0.1/32 + - value: ::1/128 + - value: fc00::/7 + - value: fe80::/10 + - value: '# tftest-file id=reference path=data/reference_lists/private_ip_ranges.txt' + - value: '' + instance: customer-id + location: europe + project: project-id + reference_list_id: private_ip_ranges + syntax_type: REFERENCE_LIST_SYNTAX_TYPE_CIDR + timeouts: null + module.secops.google_chronicle_rule.default["network_traffic_to_specific_country"]: + deletion_policy: FORCE + instance: customer-id + location: europe + project: project-id + scope: null + text: "rule network_traffic_to_specific_country {\n\nmeta:\n author = \"Google\ + \ Cloud Security\"\n description = \"Identify network traffic based on target\ + \ country\"\n type = \"alert\"\n tags = \"geoip enrichment\"\n data_source\ + \ = \"microsoft windows events\"\n severity = \"Low\"\n priority = \"Low\"\ + \n\nevents:\n $network.metadata.event_type = \"NETWORK_CONNECTION\"\n //Specify\ + \ a country of interest to monitor or add additional countries using an or statement\n\ + \ $network.target.ip_geo_artifact.location.country_or_region = \"France\" nocase\n\ + \ $network.target.ip = $ip\n\nmatch:\n $ip over 30m\n\noutcome:\n $risk_score\ + \ = max(35)\n $event_count = count_distinct($network.metadata.id)\n\n // added\ + \ to populate alert graph with additional context\n $principal_ip = array_distinct($network.principal.ip)\n\ + \n // Commented out target.ip because it is already represented in graph as\ + \ match variable. If match changes, can uncomment to add to results\n //$target_ip\ + \ = array_distinct($network.target.ip)\n $principal_process_pid = array_distinct($network.principal.process.pid)\n\ + \ $principal_process_command_line = array_distinct($network.principal.process.command_line)\n\ + \ $principal_process_file_sha256 = array_distinct($network.principal.process.file.sha256)\n\ + \ $principal_process_file_full_path = array_distinct($network.principal.process.file.full_path)\n\ + \ $principal_process_product_specfic_process_id = array_distinct($network.principal.process.product_specific_process_id)\n\ + \ $principal_process_parent_process_product_specfic_process_id = array_distinct($network.principal.process.parent_process.product_specific_process_id)\n\ + \ $target_process_pid = array_distinct($network.target.process.pid)\n $target_process_command_line\ + \ = array_distinct($network.target.process.command_line)\n $target_process_file_sha256\ + \ = array_distinct($network.target.process.file.sha256)\n $target_process_file_full_path\ + \ = array_distinct($network.target.process.file.full_path)\n $target_process_product_specfic_process_id\ + \ = array_distinct($network.target.process.product_specific_process_id)\n $target_process_parent_process_product_specfic_process_id\ + \ = array_distinct($network.target.process.parent_process.product_specific_process_id)\n\ + \ $principal_user_userid = array_distinct($network.principal.user.userid)\n\ + \ $target_user_userid = array_distinct($network.target.user.userid)\n\ncondition:\n\ + \ $network\n}\n# tftest-file id=rule path=data/rules/network_traffic_to_specific_country.yaral\n" + timeouts: null + module.secops.google_chronicle_rule_deployment.default["network_traffic_to_specific_country"]: + alerting: true + archived: false + enabled: true + instance: customer-id + location: europe + project: project-id + run_frequency: LIVE + timeouts: null diff --git a/tests/modules/secops_rules/examples/factory.yaml b/tests/modules/secops_rules/examples/factory.yaml new file mode 100644 index 000000000..aaca9c6f0 --- /dev/null +++ b/tests/modules/secops_rules/examples/factory.yaml @@ -0,0 +1,76 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.secops.google_chronicle_reference_list.default["private_ip_ranges"]: + description: Private CIDR ranges + entries: + - value: 10.0.0.0/8 + - value: 172.16.0.0/12 + - value: 192.168.0.0/16 + - value: 127.0.0.1/32 + - value: ::1/128 + - value: fc00::/7 + - value: fe80::/10 + - value: '# tftest-file id=reference path=data/reference_lists/private_ip_ranges.txt' + - value: '' + instance: customer-id + location: europe + project: project-id + reference_list_id: private_ip_ranges + syntax_type: REFERENCE_LIST_SYNTAX_TYPE_CIDR + timeouts: null + module.secops.google_chronicle_rule.default["network_traffic_to_specific_country"]: + deletion_policy: FORCE + instance: customer-id + location: europe + project: project-id + scope: null + text: "rule network_traffic_to_specific_country {\n\nmeta:\n author = \"Google\ + \ Cloud Security\"\n description = \"Identify network traffic based on target\ + \ country\"\n type = \"alert\"\n tags = \"geoip enrichment\"\n data_source\ + \ = \"microsoft windows events\"\n severity = \"Low\"\n priority = \"Low\"\ + \n\nevents:\n $network.metadata.event_type = \"NETWORK_CONNECTION\"\n //Specify\ + \ a country of interest to monitor or add additional countries using an or statement\n\ + \ $network.target.ip_geo_artifact.location.country_or_region = \"France\" nocase\n\ + \ $network.target.ip = $ip\n\nmatch:\n $ip over 30m\n\noutcome:\n $risk_score\ + \ = max(35)\n $event_count = count_distinct($network.metadata.id)\n\n // added\ + \ to populate alert graph with additional context\n $principal_ip = array_distinct($network.principal.ip)\n\ + \n // Commented out target.ip because it is already represented in graph as\ + \ match variable. If match changes, can uncomment to add to results\n //$target_ip\ + \ = array_distinct($network.target.ip)\n $principal_process_pid = array_distinct($network.principal.process.pid)\n\ + \ $principal_process_command_line = array_distinct($network.principal.process.command_line)\n\ + \ $principal_process_file_sha256 = array_distinct($network.principal.process.file.sha256)\n\ + \ $principal_process_file_full_path = array_distinct($network.principal.process.file.full_path)\n\ + \ $principal_process_product_specfic_process_id = array_distinct($network.principal.process.product_specific_process_id)\n\ + \ $principal_process_parent_process_product_specfic_process_id = array_distinct($network.principal.process.parent_process.product_specific_process_id)\n\ + \ $target_process_pid = array_distinct($network.target.process.pid)\n $target_process_command_line\ + \ = array_distinct($network.target.process.command_line)\n $target_process_file_sha256\ + \ = array_distinct($network.target.process.file.sha256)\n $target_process_file_full_path\ + \ = array_distinct($network.target.process.file.full_path)\n $target_process_product_specfic_process_id\ + \ = array_distinct($network.target.process.product_specific_process_id)\n $target_process_parent_process_product_specfic_process_id\ + \ = array_distinct($network.target.process.parent_process.product_specific_process_id)\n\ + \ $principal_user_userid = array_distinct($network.principal.user.userid)\n\ + \ $target_user_userid = array_distinct($network.target.user.userid)\n\ncondition:\n\ + \ $network\n}\n# tftest-file id=rule path=data/rules/network_traffic_to_specific_country.yaral\n" + timeouts: null + module.secops.google_chronicle_rule_deployment.default["network_traffic_to_specific_country"]: + alerting: true + archived: false + enabled: true + instance: customer-id + location: europe + project: project-id + run_frequency: DAILY + timeouts: null