From 2dea1224e537152338b9a47462280cc294058dce Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 9 Oct 2025 15:36:47 +0200 Subject: [PATCH] Align FAST project templates project definitions to new format (#3399) * fix os apt registries template * align project templates to new pf format * align project templates to new pf format --- fast/project-templates/.gitignore | 1 + .../data-mongodb/project.yaml | 38 ++++++++----- .../managed-kafka/project.yaml | 40 ++++++++------ .../os-apt-registries/project.yaml | 32 ++++++----- .../secops-anonymization-pipeline/README.md | 21 ++++---- .../project.yaml | 54 ++++++++++--------- 6 files changed, 109 insertions(+), 77 deletions(-) diff --git a/fast/project-templates/.gitignore b/fast/project-templates/.gitignore index cc8997e22..0d4deaa9c 100644 --- a/fast/project-templates/.gitignore +++ b/fast/project-templates/.gitignore @@ -1,2 +1,3 @@ **/*-providers.tf **/*.tfvars +**/*.tfvars.json diff --git a/fast/project-templates/data-mongodb/project.yaml b/fast/project-templates/data-mongodb/project.yaml index 3c7a6e073..7d9f83f30 100644 --- a/fast/project-templates/data-mongodb/project.yaml +++ b/fast/project-templates/data-mongodb/project.yaml @@ -14,34 +14,44 @@ # yaml-language-server: $schema=../../stages/2-project-factory/schemas/project.schema.json -# edit parent to suit desired folder where project is created -parent: shared -name: prod-shared-mongodb-0 +# TODO: edit and uncomment the following line to create the project in a folder +# parent: $folder_ids:shared + +# project id can be customized here if file name cannot be changed +# name: prod-shared-mongodb-0 + services: - compute.googleapis.com - logging.googleapis.com - monitoring.googleapis.com -# if automation resources are not used, grant these roles to the principal -# that will be used to apply this Terraform setup + iam: roles/compute.admin: - automation/rw roles/servicedirectory.admin: - automation/rw + automation: - project: foo-prod-shared-iac-0 + # TODO: edit the automation project and optionally edit resource names + project: $project_ids:iac-0 service_accounts: rw: description: Read/write automation service account for MongoDB. bucket: - description: Terraform state bucket for MongoDB. - iam: - roles/storage.objectAdmin: - - automation/rw - roles/storage.objectViewer: - - automation/rw + description: Terraform state bucket for apt registries. + # this reuses the existing stage state bucket and creates a folder in it + name: iac-stage-state + create: false + managed_folders: + data-mongodb: + iam: + roles/storage.objectCreator: + # the project id in the service account ref matches this file name + - $iam_principals:service_accounts/data-mongodb/automation/rw + roles/storage.objectViewer: + - $iam_principals:service_accounts/data-mongodb/automation/rw # edit or comment shared VPC service host shared_vpc_service_config: - host_project: dev-spoke-0 + host_project: $projject_ids:dev-spoke-0 network_users: - - automation/rw + - $iam_principals:service_accounts/data-mongodb/automation/rw diff --git a/fast/project-templates/managed-kafka/project.yaml b/fast/project-templates/managed-kafka/project.yaml index d0a88bf73..5e618668d 100644 --- a/fast/project-templates/managed-kafka/project.yaml +++ b/fast/project-templates/managed-kafka/project.yaml @@ -13,37 +13,47 @@ # limitations under the License. # FAST-compliant project definition for the Managed Kafka cluster -parent: shared -name: prod-shared-managed-kafka-0 +# TODO: edit and uncomment the following line to create the project in a folder +# parent: $folder_ids:shared + +# project id can be customized here if file name cannot be changed +# name: prod-shared-managed-kafka-0 + services: - compute.googleapis.com - logging.googleapis.com - monitoring.googleapis.com - kafka.googleapis.com - dns.googleapis.com -# If automation resources are not used, grant these roles to the principal -# that will be used to apply this Terraform setup + iam: roles/compute.admin: - - automation/rw + - $iam_principals:service_accounts/managed-kafka/automation/rw roles/servicedirectory.admin: - - automation/rw + - $iam_principals:service_accounts/managed-kafka/automation/rw roles/managedkafka.client: - - automation/rw + - $iam_principals:service_accounts/managed-kafka/automation/rw automation: - project: foo-prod-shared-iac-0 + # TODO: edit the automation project and optionally edit resource names + project: $project_ids:iac-0 service_accounts: rw: description: Read/write automation service account for Managed Kafka. bucket: description: Terraform state bucket for Managed Kafka. - iam: - roles/storage.objectAdmin: - - automation/rw - roles/storage.objectViewer: - - automation/rw + # this reuses the existing stage state bucket and creates a folder in it + name: iac-stage-state + create: false + managed_folders: + managed-kafka: + iam: + roles/storage.objectAdmin: + # the project id in the service account ref matches this file name + - $iam_principals:service_accounts/managed-kafka/automation/rw + roles/storage.objectViewer: + - $iam_principals:service_accounts/managed-kafka/automation/rw # Edit or comment shared VPC service host shared_vpc_service_config: - host_project: dev-spoke-0 + host_project: $project_ids:dev-spoke-0 network_users: - - automation/rw + - $iam_principals:service_accounts/managed-kafka/automation/rw diff --git a/fast/project-templates/os-apt-registries/project.yaml b/fast/project-templates/os-apt-registries/project.yaml index 649fa8a08..8ce64d571 100644 --- a/fast/project-templates/os-apt-registries/project.yaml +++ b/fast/project-templates/os-apt-registries/project.yaml @@ -15,31 +15,39 @@ # yaml-language-server: $schema=../../stages/2-project-factory/schemas/project.schema.json # TODO: edit and uncomment the following line to create the project in a folder -# parent: shared +# parent: $folder_ids:shared + +# project id can be customized here if file name cannot be changed +# name: prod-os-apt-0 -name: prod-os-apt-0 services: - accesscontextmanager.googleapis.com - artifactregistry.googleapis.com automation: # TODO: edit the automation project and optionally edit resource names - project: prod-pf-iac-0 + project: $project_ids:iac-0 service_accounts: rw: description: Read/write automation service account for apt registries. bucket: description: Terraform state bucket for apt registries. - iam: - roles/storage.objectCreator: - - rw - roles/storage.objectViewer: - - rw + # this reuses the existing stage state bucket and creates a folder in it + name: iac-stage-state + create: false + managed_folders: + os-apt: + iam: + roles/storage.objectCreator: + # the project id in the service account ref matches this file name + - $iam_principals:service_accounts/os-apt-registries/automation/rw + roles/storage.objectViewer: + - $iam_principals:service_accounts/os-apt-registries/automation/rw iam: roles/viewer: - - prod-os-apt-0/rw + - $iam_principals:service_accounts/os-apt-registries/automation/rw roles/artifactregistry.admin: - - prod-os-apt-0/rw - # TODO: add instance service accounts that need access to the registries + - $iam_principals:service_accounts/os-apt-registries/automation/rw + # TODO: add instance service accounts, or use principalSets for folders/projects # roles/artifactregistry.writer: - # - serviceAccount:foo@bar \ No newline at end of file + # - principalSet://cloudresourcemanager.googleapis.com/folders/210938489642/type/ServiceAccount diff --git a/fast/project-templates/secops-anonymization-pipeline/README.md b/fast/project-templates/secops-anonymization-pipeline/README.md index 8be973103..e92978f19 100644 --- a/fast/project-templates/secops-anonymization-pipeline/README.md +++ b/fast/project-templates/secops-anonymization-pipeline/README.md @@ -13,7 +13,6 @@ This Terraform can of course be deployed using any pre-existing project. In that - enable the APIs listed under `services` - grant the permissions listed under `iam` to the principal running Terraform, either machine (service account) or human - ### High level architecture The following diagram illustrates the high-level design of the solution, which can be adapted to specific requirements via variables and/or simple terraform and Python code customizations: @@ -22,13 +21,13 @@ The following diagram illustrates the high-level design of the solution, which c The use case is a SecOps deployment composed of 2 tenants (one for production and one for development/testing). There might be the need to export production data from the prod tenant and import them back in DEV (possibly anonymizing it) for rules and/or parser development, that is why this pipeline might be convenient for speeding up the data migration process. -The solution is based on a custom Python script responsible for implementing the aforementioned logic. The script leverages the new [SecOps API Wrapper](https://github.com/google/secops-wrapper) available also in [PyPi](https://pypi.org/project/secops/). +The solution is based on a custom Python script responsible for implementing the aforementioned logic. The script leverages the new [SecOps API Wrapper](https://github.com/google/secops-wrapper) available also in [PyPi](https://pypi.org/project/secops/). ### Pipeline Steps - **SecOps Export**: Triggered via the corresponding TRIGGER-EXPORT action. Call [SecOps Export API](https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.dataExports) to trigger raw logs export on a GCS bucket based on either all the log types or one o more of them for a specific time frame. By default, the export will be for the previous day, otherwise the following parameters can be specified to change the time frame: - * `EXPORT_DATE` date for the export (format %Y-%m-%d) - * `EXPORT_START_DATETIME` and `EXPORT_END_DATETIME` start and end datetime for the export (format %Y-%m-%dT%H:%M:%SZ). This is useful for verbose log source with GB/TB of raw logs ingested on a daily basis + - `EXPORT_DATE` date for the export (format %Y-%m-%d) + - `EXPORT_START_DATETIME` and `EXPORT_END_DATETIME` start and end datetime for the export (format %Y-%m-%dT%H:%M:%SZ). This is useful for verbose log source with GB/TB of raw logs ingested on a daily basis - **Anonymize Data**: Triggered via the corresponding ANONYMIZE-DATA action. Split the exported CSV files to one or more CSV files where the size of each file is less than 60MB (which is the maximum file size supported by DLP). It also renames those files in .log for better handling by the DLP Job. It will then trigger an asynchronous DLP job to anonymize data. - **Import Data**: Triggered via the corresponding IMPORT-DATA action. Import the exported raw logs (or anonymized ones according to the pipeline configuration) data into the target SecOps tenant leveraging the new [SecOps Ingestion API](https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.logTypes.logs/import). @@ -56,13 +55,13 @@ git clone REPO_URL Before you deploy the architecture, you will need at least the following information (for more precise configuration see the Variables section): -* GCP Project ID for SecOps anonymization pipeline deployment -* SecOps tenants information: - * GCP projects of both source and target SecOps tenants - * SecOps customer IDs for both source and target SecOps tenants - * SecOps deployment region for both the tenants (must be the same) - * SecOps Forwarder ID for target tenant (this is mandatory for new ingestion APIs and requires at least an empty collector forwarder to be setup in target tenant) - * **Grant Pipeline SA Chronicle API Editor role on both source and target tenant** (this might be restricred to data export permissions on source and import logs permissions on target tenant) +- GCP Project ID for SecOps anonymization pipeline deployment +- SecOps tenants information: + - GCP projects of both source and target SecOps tenants + - SecOps customer IDs for both source and target SecOps tenants + - SecOps deployment region for both the tenants (must be the same) + - SecOps Forwarder ID for target tenant (this is mandatory for new ingestion APIs and requires at least an empty collector forwarder to be setup in target tenant) + - **Grant Pipeline SA Chronicle API Editor role on both source and target tenant** (this might be restricred to data export permissions on source and import logs permissions on target tenant) #### Step 2: Prepare the variables diff --git a/fast/project-templates/secops-anonymization-pipeline/project.yaml b/fast/project-templates/secops-anonymization-pipeline/project.yaml index ce5ec3fe8..eb29e134f 100644 --- a/fast/project-templates/secops-anonymization-pipeline/project.yaml +++ b/fast/project-templates/secops-anonymization-pipeline/project.yaml @@ -15,38 +15,42 @@ # yaml-language-server: $schema=../../stages/2-project-factory/schemas/project.schema.json # TODO: edit and uncomment the following line to create the project in a folder -# parent: shared +# parent: $folder_ids:shared + +# project id can be customized here if file name cannot be changed +# name: secops-anonym-0 -name: secops-anonym-0 services: - - "secretmanager.googleapis.com" - - "run.googleapis.com" - - "cloudscheduler.googleapis.com" - - "cloudbuild.googleapis.com" - - "cloudresourcemanager.googleapis.com" - - "vpcaccess.googleapis.com" - - "dlp.googleapis.com" - - "vpcaccess.googleapis.com" + - secretmanager.googleapis.com + - run.googleapis.com + - cloudscheduler.googleapis.com + - cloudbuild.googleapis.com + - cloudresourcemanager.googleapis.com + - vpcaccess.googleapis.com + - dlp.googleapis.com + - vpcaccess.googleapis.com automation: # TODO: edit the automation project and optionally edit resource names - project: pf-automation-0 + project: $project_ids:iac-0 service_accounts: rw: - description: Read/write automation service account for apt registries. - buckets: - tf-state: - description: Terraform state bucket for apt registries. - iam: - roles/storage.objectCreator: - - rw - roles/storage.objectViewer: - - rw + description: Read/write automation service account for secops pipeline. + bucket: + description: Terraform state bucket for apt registries. + # this reuses the existing stage state bucket and creates a folder in it + name: iac-stage-state + create: false + managed_folders: + secops-anonymization-pipeline: + iam: + roles/storage.objectCreator: + # the project id in the service account ref matches this file name + - $iam_principals:service_accounts/secops-anonymization-pipeline/automation/rw + roles/storage.objectViewer: + - $iam_principals:service_accounts/secops-anonymization-pipeline/automation/rw iam: roles/viewer: - - rw + - $iam_principals:service_accounts/secops-anonymization-pipeline/automation/rw roles/owner: - - rw - # TODO: add instance service accounts that need access to the registries - # roles/artifactregistry.writer: - # - serviceAccount:foo@bar + - $iam_principals:service_accounts/secops-anonymization-pipeline/automation/rw