diff --git a/fast/project-templates/secops-anonymization-pipeline/README.md b/fast/project-templates/secops-anonymization-pipeline/README.md index 93ff571c7..8be973103 100644 --- a/fast/project-templates/secops-anonymization-pipeline/README.md +++ b/fast/project-templates/secops-anonymization-pipeline/README.md @@ -22,6 +22,8 @@ The following diagram illustrates the high-level design of the solution, which c The use case is a SecOps deployment composed of 2 tenants (one for production and one for development/testing). There might be the need to export production data from the prod tenant and import them back in DEV (possibly anonymizing it) for rules and/or parser development, that is why this pipeline might be convenient for speeding up the data migration process. +The solution is based on a custom Python script responsible for implementing the aforementioned logic. The script leverages the new [SecOps API Wrapper](https://github.com/google/secops-wrapper) available also in [PyPi](https://pypi.org/project/secops/). + ### Pipeline Steps - **SecOps Export**: Triggered via the corresponding TRIGGER-EXPORT action. Call [SecOps Export API](https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.dataExports) to trigger raw logs export on a GCS bucket based on either all the log types or one o more of them for a specific time frame. By default, the export will be for the previous day, otherwise the following parameters can be specified to change the time frame: @@ -92,6 +94,7 @@ terraform apply #### Step 5: Test solution Test the solution triggering an export from the Cloud Scheduler page, after few hours (accoding to the size of the export) logs should be available on secops-export bucket. Please check for any issue during export using the corresponding APIs and the export ID. + ## Variables diff --git a/fast/project-templates/secops-anonymization-pipeline/source/main.py b/fast/project-templates/secops-anonymization-pipeline/source/main.py index 350edfd69..80712bbf6 100644 --- a/fast/project-templates/secops-anonymization-pipeline/source/main.py +++ b/fast/project-templates/secops-anonymization-pipeline/source/main.py @@ -17,16 +17,13 @@ import json import os import click import logging -import sys import google.cloud.logging -from google.auth.transport.requests import AuthorizedSession -from google.oauth2 import service_account from jinja2 import Template from shared import utils from google.cloud import dlp_v2 from google.cloud import storage -from datetime import date, timedelta -from shared import secops +from datetime import date, timedelta, datetime +from secops import SecOpsClient client = google.cloud.logging.Client() client.setup_logging() @@ -37,11 +34,6 @@ logging.basicConfig( format='[%(levelname)-8s] - %(asctime)s - %(message)s') logging.root.setLevel(logging.DEBUG) -SCOPES = [ - "https://www.googleapis.com/auth/chronicle-backstory", - "https://www.googleapis.com/auth/malachite-ingestion" -] - SECOPS_REGION = os.environ.get("SECOPS_REGION") GCP_PROJECT_ID = os.environ.get("GCP_PROJECT") SECOPS_EXPORT_BUCKET = os.environ.get("SECOPS_EXPORT_BUCKET") @@ -51,7 +43,6 @@ SECOPS_TARGET_PROJECT = os.environ.get("SECOPS_TARGET_PROJECT") SECOPS_SOURCE_CUSTOMER_ID = os.environ.get("SECOPS_SOURCE_CUSTOMER_ID") SECOPS_TARGET_CUSTOMER_ID = os.environ.get("SECOPS_TARGET_CUSTOMER_ID") SECOPS_TARGET_FORWARDER_ID = os.environ.get("SECOPS_TARGET_FORWARDER_ID") - SKIP_ANONYMIZATION = False if (os.environ.get( "SKIP_ANONYMIZATION", "false").lower() == "false") else True DLP_DEIDENTIFY_TEMPLATE_ID = os.environ.get("DLP_DEIDENTIFY_TEMPLATE_ID") @@ -60,16 +51,10 @@ DLP_REGION = os.environ.get("DLP_REGION") def import_logs(export_date): - # Initialize with default credentials - will automatically use the service account - # assigned to your Google Cloud resource - client = secops.SecOpsClient() - - # Initialize Chronicle client - chronicle = client.chronicle( - customer_id=SECOPS_TARGET_CUSTOMER_ID, # Your Chronicle instance ID - project_id=SECOPS_TARGET_PROJECT, # Your GCP project ID - region=SECOPS_REGION # Chronicle API region - ) + client = SecOpsClient() + chronicle = client.chronicle(customer_id=SECOPS_TARGET_CUSTOMER_ID, + project_id=SECOPS_TARGET_PROJECT, + region=SECOPS_REGION) storage_client = storage.Client() BUCKET = SECOPS_OUTPUT_BUCKET if not SKIP_ANONYMIZATION else SECOPS_EXPORT_BUCKET @@ -81,20 +66,28 @@ def import_logs(export_date): log_type = folder.split("-")[0] for log_file in utils.list_log_files(BUCKET, f"{export_id}/{folder}"): - blob = bucket.blob(log_file) # Directly get the blob object - with blob.open("r") as f: - logs = [] - for line in f: - logs.append(line.rstrip('\n')) - if len(logs) == 1000: - response = chronicle.ingest_logs(logs=logs, log_type=log_type, forwarder_id=SECOPS_TARGET_FORWARDER_ID) - LOGGER.debug(response) - logs = [] + try: + blob = bucket.blob(log_file) # Directly get the blob object + with blob.open("r") as f: + logs = [] + for line in f: + logs.append(line.rstrip('\n')) + if len(logs) == 1000: + response = chronicle.ingest_log( + log_message=logs, log_type=log_type, + forwarder_id=SECOPS_TARGET_FORWARDER_ID) + LOGGER.debug(response) + logs = [] - # Send any remaining entries - if len(logs) > 0: - response = chronicle.ingest_logs(logs=logs, log_type=log_type, forwarder_id=SECOPS_TARGET_FORWARDER_ID) + # Send any remaining entries + if len(logs) > 0: + response = chronicle.ingest_log( + log_message=logs, log_type=log_type, + forwarder_id=SECOPS_TARGET_FORWARDER_ID) LOGGER.debug(response) + except Exception as e: + LOGGER.error(f"Error during log ingestion': {e}") + raise SystemExit(f'Error during log ingestion: {e}') # delete both export and anonymized buckets after ingesting logs utils.delete_folder(BUCKET, export_id) @@ -119,35 +112,38 @@ def trigger_export(export_date: str, export_start_datetime: str, :return: """ - - # Initialize with default credentials - will automatically use the service account - # assigned to your Google Cloud resource - client = secops.SecOpsClient() - - # Initialize Chronicle client - chronicle = client.chronicle( - customer_id=SECOPS_SOURCE_CUSTOMER_ID, # Your Chronicle instance ID - project_id=SECOPS_SOURCE_PROJECT, # Your GCP project ID - region=SECOPS_REGION # Chronicle API region - ) + client = SecOpsClient() + chronicle = client.chronicle(customer_id=SECOPS_SOURCE_CUSTOMER_ID, + project_id=SECOPS_SOURCE_PROJECT, + region=SECOPS_REGION) export_ids = [] + + if export_start_datetime and export_end_datetime: + start_time, end_time = datetime.strptime( + export_start_datetime, + "%Y-%m-%dT%H:%M:%SZ"), datetime.strptime(export_end_datetime, + "%Y-%m-%dT%H:%M:%SZ") + else: + start_time, end_time = utils.format_date_time_range(date_input=export_date) + gcs_bucket = f"projects/{GCP_PROJECT_ID}/buckets/{SECOPS_EXPORT_BUCKET}" + try: - if log_types is None: - export_response = chronicle.create_data_export( - project=GCP_PROJECT_ID, export_date=export_date, - export_start_datetime=export_start_datetime, - export_end_datetime=export_end_datetime) + if log_types is None or log_types == "": + export_response = chronicle.create_data_export(start_time=start_time, + end_time=end_time, + gcs_bucket=gcs_bucket, + export_all_logs=True) LOGGER.info(export_response) export_id = export_response["dataExportStatus"]["name"].split("/")[-1] export_ids.append(export_id) LOGGER.info(f"Triggered export with ID: {export_id}") else: for log_type in log_types.split(","): - export_response = chronicle.create_data_export( - project=GCP_PROJECT_ID, export_date=export_date, - export_start_datetime=export_start_datetime, - export_end_datetime=export_end_datetime, log_type=log_type) + export_response = chronicle.create_data_export(start_time=start_time, + end_time=end_time, + gcs_bucket=gcs_bucket, + log_type=log_type) export_id = export_response["dataExportStatus"]["name"].split("/")[-1] export_ids.append(export_id) LOGGER.info(f"Triggered export with ID: {export_id}") @@ -164,24 +160,20 @@ def anonymize_data(export_date): :param export_date: date for which data should be anonymized :return: """ - # Initialize with default credentials - will automatically use the service account - # assigned to your Google Cloud resource - client = secops.SecOpsClient() - # Initialize Chronicle client - chronicle = client.chronicle( - customer_id=SECOPS_SOURCE_CUSTOMER_ID, # Your Chronicle instance ID - project_id=SECOPS_SOURCE_PROJECT, # Your GCP project ID - region=SECOPS_REGION # Chronicle API region - ) + client = SecOpsClient() + chronicle = client.chronicle(customer_id=SECOPS_SOURCE_CUSTOMER_ID, + project_id=SECOPS_SOURCE_PROJECT, + region=SECOPS_REGION) export_ids = utils.get_secops_export_folders_for_date(SECOPS_EXPORT_BUCKET, export_date=export_date) export_finished = True for export_id in export_ids: - export = chronicle.get_data_export(name=export_id) + export = chronicle.get_data_export(data_export_id=export_id) LOGGER.info(f"Export response: {export}.") - if "dataExportStatus"in export and export["dataExportStatus"]["stage"] == "FINISHED_SUCCESS": + if "dataExportStatus" in export and export["dataExportStatus"][ + "stage"] == "FINISHED_SUCCESS": export_state = export["dataExportStatus"]["stage"] LOGGER.info(f"Export status: {export_state}.") else: @@ -211,10 +203,14 @@ def anonymize_data(export_date): "inspect_job": dlp_job } - dlp_client = dlp_v2.DlpServiceClient( - client_options={'quota_project_id': GCP_PROJECT_ID}) - response = dlp_client.create_dlp_job(request=job_request) - LOGGER.info(response) + try: + dlp_client = dlp_v2.DlpServiceClient( + client_options={'quota_project_id': GCP_PROJECT_ID}) + response = dlp_client.create_dlp_job(request=job_request) + LOGGER.info(response) + except Exception as e: + LOGGER.error(f"Error during export': {e}") + raise SystemExit(f'Error during secops export: {e}') else: LOGGER.error("Export is not finished yet, please try again later.") @@ -288,7 +284,7 @@ def main_cli(export_date, export_start_datetime, export_end_datetime, trigger_export(export_date=export_date, export_start_datetime=export_start_datetime, export_end_datetime=export_end_datetime, - log_types=log_type) + log_types=','.join(log_type)) case "ANONYMIZE-DATA": anonymize_data(export_date=export_date) case "IMPORT-DATA": diff --git a/fast/project-templates/secops-anonymization-pipeline/source/requirements.txt b/fast/project-templates/secops-anonymization-pipeline/source/requirements.txt index a61d7ec92..c82eef8f5 100644 --- a/fast/project-templates/secops-anonymization-pipeline/source/requirements.txt +++ b/fast/project-templates/secops-anonymization-pipeline/source/requirements.txt @@ -23,3 +23,4 @@ google-cloud-storage click==8.1.3 google-cloud-dlp google-cloud-logging +secops \ No newline at end of file diff --git a/fast/project-templates/secops-anonymization-pipeline/source/shared/utils.py b/fast/project-templates/secops-anonymization-pipeline/source/shared/utils.py index e966c83b5..362816f99 100644 --- a/fast/project-templates/secops-anonymization-pipeline/source/shared/utils.py +++ b/fast/project-templates/secops-anonymization-pipeline/source/shared/utils.py @@ -17,7 +17,7 @@ import os import logging import math import csv -from google.cloud import secretmanager, storage, exceptions +from google.cloud import storage from datetime import datetime, timedelta, timezone, time LOGGER = logging.getLogger('secops') @@ -25,22 +25,6 @@ LOGGER = logging.getLogger('secops') MAX_FILE_SIZE = 61440000 # Max size supported by DLP -def get_value_from_secret_manager(resource_path: str) -> str: - """Retrieve the value of the secret from the Google Cloud Secret Manager. - - Args: - resource_path (str): Path of the secret with version included. Ex.: - "projects//secrets//versions/1", - "projects//secrets//versions/latest" - - Returns: - str: Payload for secret. - """ - client = secretmanager.SecretManagerServiceClient() - response = client.access_secret_version(name=resource_path) - return response.payload.data.decode("UTF-8") - - def format_date_time_range(date_input): """ Creates datetime objects for the beginning and end of the input date @@ -60,11 +44,7 @@ def format_date_time_range(date_input): tzinfo=timezone.utc) end_of_day = start_of_day + timedelta(days=1, seconds=-1) - # Format both datetime objects - formatted_start = start_of_day.strftime("%Y-%m-%dT%H:%M:%SZ") - formatted_end = end_of_day.strftime("%Y-%m-%dT%H:%M:%SZ") - - return formatted_start, formatted_end + return start_of_day, end_of_day def list_anonymized_folders(bucket_name, folder_name): @@ -94,16 +74,10 @@ def delete_folder(bucket_name, folder_name): bucket_name: The name of the bucket. folder_name: The name of the folder to delete. """ - storage_client = storage.Client() bucket = storage_client.bucket(bucket_name) - - # List all blobs with the given prefix (folder name) blobs = list(bucket.list_blobs(prefix=folder_name)) - - # Delete the blobs in parallel bucket.delete_blobs(blobs) - print(f"Folder {folder_name} deleted from bucket {bucket_name}") diff --git a/modules/net-vpc-factory/factory-project.tf b/modules/net-vpc-factory/factory-project.tf index 00f783ba3..4e7ae1992 100644 --- a/modules/net-vpc-factory/factory-project.tf +++ b/modules/net-vpc-factory/factory-project.tf @@ -17,22 +17,15 @@ # tfdoc:file:description Dedicated project factory. locals { - projects = { for k, v in local.network_projects : k => merge( - { - billing_account = try(v.project_config.billing_account, var.billing_account) - prefix = try(v.project_config.prefix, var.prefix) - parent = try(v.project_config.parent, var.parent_id) - shared_vpc_host_config = try(v.project_config.shared_vpc_host_config, null) - iam = try(v.project_config.iam, {}) - iam_bindings = try(v.project_config.iam_bindings, {}) - iam_bindings_additive = try(v.project_config.iam_bindings_additive, {}) - iam_by_principals = try(v.project_config.iam_by_principals, {}) - iam_by_principals_additive = try(v.project_config.iam_by_principals_additive, {}) - services = try(v.project_config.services, []) - org_policies = try(v.project_config.org_policies, {}) - }, - v.project_config) + _projects_input = { for k, v in local.network_projects : k => v.project_config } + _projects_config = { + data_defaults = { + billing_account = var.billing_account + prefix = var.prefix + parent = var.parent_id + } } + projects = local._projects_output } module "projects" { diff --git a/modules/net-vpc-factory/factory-projects-object.tf b/modules/net-vpc-factory/factory-projects-object.tf new file mode 100644 index 000000000..3e9c2d714 --- /dev/null +++ b/modules/net-vpc-factory/factory-projects-object.tf @@ -0,0 +1,257 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# inputs +# local._projects_input - parsed data from yaml as map +# local._projects_config = object({ +# data_overrides = ... +# data_defaults = ... +# }) +# outputs: +# local._projects_output - map +locals { + __projects_config = { + data_defaults = merge({ + billing_account = null + contacts = {} + factories_config = merge({ + custom_roles = null + observability = null + org_policies = null + quotas = null + }, try(local._projects_config.data_defaults.factories_config, { + custom_roles = null + observability = null + org_policies = null + quotas = null + }) + ) + labels = {} + metric_scopes = [] + parent = null + prefix = null + service_encryption_key_ids = {} + services = [] + shared_vpc_service_config = merge({ + host_project = null + network_users = [] + service_agent_iam = {} + service_agent_subnet_iam = {} + service_iam_grants = [] + network_subnet_users = {} + }, try(local._projects_config.data_defaults.shared_vpc_service_config, { + host_project = null + network_users = [] + service_agent_iam = {} + service_agent_subnet_iam = {} + service_iam_grants = [] + network_subnet_users = {} + }) + ) + storage_location = null + tag_bindings = {} + service_accounts = {} + vpc_sc = merge({ + perimeter_name = null + perimeter_bridges = [] + is_dry_run = false + }, try(local._projects_config.data_defaults.vpc_sc, { + perimeter_name = null + perimeter_bridges = [] + is_dry_run = false + }) + ) + logging_data_access = {} + }, + try(local._projects_config.data_defaults, {}) + ) + # data_overrides default to null's, to mark that they should not override + data_overrides = merge({ + billing_account = null + contacts = null + factories_config = merge({ + custom_roles = null + observability = null + org_policies = null + quotas = null + }, try(local._projects_config.data_overrides.factories_config, { + custom_roles = null + observability = null + org_policies = null + quotas = null + }) + ) + parent = null + prefix = null + service_encryption_key_ids = null + storage_location = null + tag_bindings = null + services = null + service_accounts = null + vpc_sc = try( + merge( + { + perimeter_name = null + perimeter_bridges = [] + is_dry_run = false + }, + local._projects_config.data_overrides.vpc_sc + ), + null + ) + logging_data_access = null + }, + try(local._projects_config.data_overrides, {}) + ) + } + _projects_output = { + # Semantics of the merges are: + # * if data_overrides. is not null, use this value + # * if _projects_inputs. is not null, use this value + # * use data_default value, which if not set, will provide "empty" type + # This logic is easily implemented using coalesce, even on maps and list and allows to + # set data_overrides. to "", [] or {} to ensure, that empty value is always passed, or do + # the same in _projects_input to prevent falling back to default value + for k, v in local._projects_input : k => merge(v, { + billing_account = try(coalesce( # type: string + local.__projects_config.data_overrides.billing_account, + try(v.billing_account, null), + local.__projects_config.data_defaults.billing_account + ), null) + contacts = coalesce( # type: map + local.__projects_config.data_overrides.contacts, + try(v.contacts, null), + local.__projects_config.data_defaults.contacts + ) + factories_config = { # type: object + custom_roles = try( # type: string + coalesce( + local.__projects_config.data_overrides.factories_config.custom_roles, + try(v.factories_config.custom_roles, null), + local.__projects_config.data_defaults.factories_config.custom_roles + ), + null + ) + observability = try( # type: string + coalesce( + local.__projects_config.data_overrides.factories_config.observability, + try(v.factories_config.observability, null), + local.__projects_config.data_defaults.factories_config.observability + ), + null) + org_policies = try( # type: string + coalesce( + local.__projects_config.data_overrides.factories_config.org_policies, + try(v.factories_config.org_policies, null), + local.__projects_config.data_defaults.factories_config.org_policies + ), + null) + quotas = try( # type: string + coalesce( + local.__projects_config.data_overrides.factories_config.quotas, + try(v.factories_config.quotas, null), + local.__projects_config.data_defaults.factories_config.quotas + ), + null) + } + iam = try(v.iam, {}) # type: map(list(string)) + iam_bindings = try(v.iam_bindings, {}) # type: map(object({...})) + iam_bindings_additive = try(v.iam_bindings_additive, {}) # type: map(object({...})) + iam_by_principals_additive = try(v.iam_by_principals_additive, {}) # type: map(list(string)) + iam_by_principals = try(v.iam_by_principals, {}) # map(list(string)) + labels = coalesce( # type: map(string) + try(v.labels, null), + local.__projects_config.data_defaults.labels + ) + metric_scopes = coalesce( # type: list(string) + try(v.metric_scopes, null), + local.__projects_config.data_defaults.metric_scopes + ) + name = lookup(v, "name", k) # type: string + org_policies = try(v.org_policies, {}) # type: map(object({...})) + parent = try( # type: string, nullable + coalesce( + local.__projects_config.data_overrides.parent, + try(v.parent, null), + local.__projects_config.data_defaults.parent + ), null + ) + prefix = try( # type: string, nullable + coalesce( + local.__projects_config.data_overrides.prefix, + try(v.prefix, null), + local.__projects_config.data_defaults.prefix + ), null + ) + service_encryption_key_ids = coalesce( # type: map(list(string)) + local.__projects_config.data_overrides.service_encryption_key_ids, + try(v.service_encryption_key_ids, null), + local.__projects_config.data_defaults.service_encryption_key_ids + ) + services = coalesce( # type: list(string) + local.__projects_config.data_overrides.services, + try(v.services, null), + local.__projects_config.data_defaults.services + ) + shared_vpc_host_config = ( # type: object({...}) + try(v.shared_vpc_host_config, null) != null + ? merge( + { service_projects = [] }, + v.shared_vpc_host_config + ) + : null + ) + shared_vpc_service_config = ( # type: object({...}) + try(v.shared_vpc_service_config, null) != null + ? merge( + { + host_project = null + network_users = [] + service_agent_iam = {} + service_agent_subnet_iam = {} + service_iam_grants = [] + network_subnet_users = {} + }, + v.shared_vpc_service_config + ) + : local.__projects_config.data_defaults.shared_vpc_service_config + ) + tag_bindings = coalesce( # type: map(string) + local.__projects_config.data_overrides.tag_bindings, + try(v.tag_bindings, null), + local.__projects_config.data_defaults.tag_bindings + ) + vpc_sc = ( # type: object + local.__projects_config.data_overrides.vpc_sc != null + ? local.__projects_config.data_overrides.vpc_sc + : ( + try(v.vpc_sc, null) != null + ? merge({ + perimeter_name = null + perimeter_bridges = [] + is_dry_run = false + }, v.vpc_sc) + : local.__projects_config.data_defaults.vpc_sc + ) + ) + logging_data_access = coalesce( # type: map(object({...})) + local.__projects_config.data_overrides.logging_data_access, + try(v.logging_data_access, null), + local.__projects_config.data_defaults.logging_data_access + ) + }) + } +} diff --git a/modules/secops-rules/README.md b/modules/secops-rules/README.md new file mode 100644 index 000000000..e99aa3858 --- /dev/null +++ b/modules/secops-rules/README.md @@ -0,0 +1,217 @@ +# SecOps Rules + +This module allows creation and management of [custom rules](https://cloud.google.com/chronicle/docs/detection/view-all-rules) as well as [reference lists](https://cloud.google.com/chronicle/docs/reference/reference-lists) in Google SecOps. + +- rule definition (yaral code) and reference list entries are managed as files in data folder as per the `factories_config` variable and sample code +- rule and reference list deployments can leverage both `rules_config` and `reference_lists_config` variables or YAML file still specified in the `factories_config` variable. + + +- [Examples](#examples) + - [Sample SecOps Rules and reference list deployment](#sample-secops-rules-and-reference-list-deployment) + - [SecOps Rules Factory](#secops-rules-factory) +- [Variables](#variables) + + +## Examples + +### Sample SecOps Rules and reference list deployment + +This is a sample usage of the secops-rules module for deploying a rule (network_traffic_to_specific_country) and a reference list (private_ip_ranges), definition of the rule in yaral is available in the corresponding file in the `data/rules` folder and the reference list in the `data/reference_lists` folder. Deployment configuration for both is passed as an input to the module using the `rules_config` and `reference_lists_config` variables. + +```hcl +module "secops" { + source = "./fabric/modules/secops-rules" + project_id = var.project_id + tenant_config = var.secops_tenant_config + reference_lists_config = { + "private_ip_ranges" = { + description = "Private CIDR ranges" + type = "CIDR" + } + } + rules_config = { + "network_traffic_to_specific_country" = { + enabled = true + alerting = true + archived = false + run_frequency = "LIVE" + } + } + factories_config = { + rules_defs = "./data/rules" + reference_lists_defs = "./data/reference_lists" + } +} +# tftest modules=1 resources=3 files=reference,rule inventory=basic.yaml +``` + +``` +rule network_traffic_to_specific_country { + +meta: + author = "Google Cloud Security" + description = "Identify network traffic based on target country" + type = "alert" + tags = "geoip enrichment" + data_source = "microsoft windows events" + severity = "Low" + priority = "Low" + +events: + $network.metadata.event_type = "NETWORK_CONNECTION" + //Specify a country of interest to monitor or add additional countries using an or statement + $network.target.ip_geo_artifact.location.country_or_region = "France" nocase + $network.target.ip = $ip + +match: + $ip over 30m + +outcome: + $risk_score = max(35) + $event_count = count_distinct($network.metadata.id) + + // added to populate alert graph with additional context + $principal_ip = array_distinct($network.principal.ip) + + // Commented out target.ip because it is already represented in graph as match variable. If match changes, can uncomment to add to results + //$target_ip = array_distinct($network.target.ip) + $principal_process_pid = array_distinct($network.principal.process.pid) + $principal_process_command_line = array_distinct($network.principal.process.command_line) + $principal_process_file_sha256 = array_distinct($network.principal.process.file.sha256) + $principal_process_file_full_path = array_distinct($network.principal.process.file.full_path) + $principal_process_product_specfic_process_id = array_distinct($network.principal.process.product_specific_process_id) + $principal_process_parent_process_product_specfic_process_id = array_distinct($network.principal.process.parent_process.product_specific_process_id) + $target_process_pid = array_distinct($network.target.process.pid) + $target_process_command_line = array_distinct($network.target.process.command_line) + $target_process_file_sha256 = array_distinct($network.target.process.file.sha256) + $target_process_file_full_path = array_distinct($network.target.process.file.full_path) + $target_process_product_specfic_process_id = array_distinct($network.target.process.product_specific_process_id) + $target_process_parent_process_product_specfic_process_id = array_distinct($network.target.process.parent_process.product_specific_process_id) + $principal_user_userid = array_distinct($network.principal.user.userid) + $target_user_userid = array_distinct($network.target.user.userid) + +condition: + $network +} +# tftest-file id=rule path=data/rules/network_traffic_to_specific_country.yaral +``` + +``` +10.0.0.0/8 +172.16.0.0/12 +192.168.0.0/16 +127.0.0.1/32 +::1/128 +fc00::/7 +fe80::/10 +# tftest-file id=reference path=data/reference_lists/private_ip_ranges.txt +``` + +### SecOps Rules Factory + +The module includes a secops rules and reference list factory (see [Resource Factories](../../blueprints/factories/)) for the configuration of rules and reference lists leveraging YAML configuration files. Each configuration file for rules and reference lists contains more than one rule with a structure that reflects the `rules_config` and `reference_lists_config` variables. Again rules and reference list definition is available in the corresponding yaral and txt files in the data folder. + +```hcl +module "secops" { + source = "./fabric/modules/secops-rules" + project_id = var.project_id + tenant_config = var.secops_tenant_config + factories_config = { + rules = "./secops_rules.yaml" + rules_defs = "./data/rules" + reference_lists = "./secops_reference_lists.yaml" + reference_lists_defs = "./data/reference_lists" + } +} +# tftest modules=1 resources=3 files=1,2,reference,rule inventory=factory.yaml + +``` + +```yaml +network_traffic_to_specific_country: + enabled: true + alerting: true + archived: false + run_frequency: "DAILY" +# tftest-file id=1 path=secops_rules.yaml +``` + +```yaml +private_ip_ranges: + description: "Private CIDR ranges" + type: CIDR # either CIDR, STRING, REGEX +# tftest-file id=2 path=secops_reference_lists.yaml +``` + +``` +rule network_traffic_to_specific_country { + +meta: + author = "Google Cloud Security" + description = "Identify network traffic based on target country" + type = "alert" + tags = "geoip enrichment" + data_source = "microsoft windows events" + severity = "Low" + priority = "Low" + +events: + $network.metadata.event_type = "NETWORK_CONNECTION" + //Specify a country of interest to monitor or add additional countries using an or statement + $network.target.ip_geo_artifact.location.country_or_region = "France" nocase + $network.target.ip = $ip + +match: + $ip over 30m + +outcome: + $risk_score = max(35) + $event_count = count_distinct($network.metadata.id) + + // added to populate alert graph with additional context + $principal_ip = array_distinct($network.principal.ip) + + // Commented out target.ip because it is already represented in graph as match variable. If match changes, can uncomment to add to results + //$target_ip = array_distinct($network.target.ip) + $principal_process_pid = array_distinct($network.principal.process.pid) + $principal_process_command_line = array_distinct($network.principal.process.command_line) + $principal_process_file_sha256 = array_distinct($network.principal.process.file.sha256) + $principal_process_file_full_path = array_distinct($network.principal.process.file.full_path) + $principal_process_product_specfic_process_id = array_distinct($network.principal.process.product_specific_process_id) + $principal_process_parent_process_product_specfic_process_id = array_distinct($network.principal.process.parent_process.product_specific_process_id) + $target_process_pid = array_distinct($network.target.process.pid) + $target_process_command_line = array_distinct($network.target.process.command_line) + $target_process_file_sha256 = array_distinct($network.target.process.file.sha256) + $target_process_file_full_path = array_distinct($network.target.process.file.full_path) + $target_process_product_specfic_process_id = array_distinct($network.target.process.product_specific_process_id) + $target_process_parent_process_product_specfic_process_id = array_distinct($network.target.process.parent_process.product_specific_process_id) + $principal_user_userid = array_distinct($network.principal.user.userid) + $target_user_userid = array_distinct($network.target.user.userid) + +condition: + $network +} +# tftest-file id=rule path=data/rules/network_traffic_to_specific_country.yaral +``` + +``` +10.0.0.0/8 +172.16.0.0/12 +192.168.0.0/16 +127.0.0.1/32 +::1/128 +fc00::/7 +fe80::/10 +# tftest-file id=reference path=data/reference_lists/private_ip_ranges.txt +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [project_id](variables.tf#L29) | Project used for resources. | string | ✓ | | +| [tenant_config](variables.tf#L66) | SecOps Tenant configuration. | object({…}) | ✓ | | +| [factories_config](variables.tf#L17) | Paths to YAML config expected in 'rules' and 'reference_lists'. Path to folders containing rules definitions (yaral files) and reference lists content (txt files) for the corresponding _defs keys. | object({…}) | | {} | +| [reference_lists_config](variables.tf#L34) | SecOps Reference lists configuration. | map(object({…})) | | {} | +| [rules_config](variables.tf#L49) | SecOps Detection rules configuration. | map(object({…})) | | {} | + diff --git a/modules/secops-rules/main.tf b/modules/secops-rules/main.tf new file mode 100644 index 000000000..b76191bb4 --- /dev/null +++ b/modules/secops-rules/main.tf @@ -0,0 +1,72 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + reference_lists = try(yamldecode(file(var.factories_config.reference_lists)), var.reference_lists_config) + reference_lists_entries = { + for k, v in local.reference_lists : k => split("\n", file("${var.factories_config.reference_lists_defs}/${k}.txt")) + } + reference_list_type_mapping = { + STRING = "REFERENCE_LIST_SYNTAX_TYPE_PLAIN_TEXT_STRING" + REGEX = "REFERENCE_LIST_SYNTAX_TYPE_REGEX" + CIDR = "REFERENCE_LIST_SYNTAX_TYPE_CIDR" + } + secops_rules = { + for file_name in fileset(var.factories_config.rules_defs, "*.yaral") : + replace(file_name, ".yaral", "") => file("${var.factories_config.rules_defs}/${file_name}") + } + secops_rule_deployment = try(yamldecode(file(var.factories_config.rules)), var.rules_config) +} + +resource "google_chronicle_reference_list" "default" { + for_each = local.reference_lists + project = var.project_id + location = var.tenant_config.region + instance = var.tenant_config.customer_id + reference_list_id = each.key + description = each.value.description + dynamic "entries" { + for_each = local.reference_lists_entries[each.key] + content { + value = entries.value + } + } + syntax_type = local.reference_list_type_mapping[each.value.type] +} + +resource "google_chronicle_rule" "default" { + for_each = local.secops_rule_deployment + project = var.project_id + location = var.tenant_config.region + instance = var.tenant_config.customer_id + text = local.secops_rules[each.key] + deletion_policy = "FORCE" + depends_on = [ + google_chronicle_reference_list.default + ] +} + +resource "google_chronicle_rule_deployment" "default" { + for_each = local.secops_rule_deployment + project = var.project_id + location = var.tenant_config.region + instance = var.tenant_config.customer_id + rule = google_chronicle_rule.default[each.key].rule_id + enabled = each.value.enabled + alerting = each.value.alerting + archived = each.value.archived + run_frequency = each.value.run_frequency +} diff --git a/modules/secops-rules/variables.tf b/modules/secops-rules/variables.tf new file mode 100644 index 000000000..163e29865 --- /dev/null +++ b/modules/secops-rules/variables.tf @@ -0,0 +1,72 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "factories_config" { + description = "Paths to YAML config expected in 'rules' and 'reference_lists'. Path to folders containing rules definitions (yaral files) and reference lists content (txt files) for the corresponding _defs keys." + type = object({ + rules = optional(string) + rules_defs = optional(string, "data/rules") + reference_lists = optional(string) + reference_lists_defs = optional(string, "data/reference_lists") + }) + nullable = false + default = {} +} + +variable "project_id" { + description = "Project used for resources." + type = string +} + +variable "reference_lists_config" { + description = "SecOps Reference lists configuration." + type = map(object({ + description = string + type = string + })) + default = {} + validation { + condition = alltrue([ + for config in var.reference_lists_config : contains(["CIDR", "STRING", "REGEX"], config.type) + ]) + error_message = "The 'type' attribute for each reference list must be one of: CIDR, STRING, REGEX." + } +} + +variable "rules_config" { + description = "SecOps Detection rules configuration." + type = map(object({ + enabled = optional(bool, true) + alerting = optional(bool, false) + archived = optional(bool, false) + run_frequency : optional(string) + })) + default = {} + validation { + condition = alltrue([ + for config in var.rules_config : contains(["LIVE", "HOURLY", "DAILY"], config.run_frequency) + ]) + error_message = "The 'type' attribute for each reference list must be one of: CIDR, STRING, REGEX." + } +} + +variable "tenant_config" { + description = "SecOps Tenant configuration." + type = object({ + customer_id = string + region = string + }) +} diff --git a/modules/secops-rules/versions.tf b/modules/secops-rules/versions.tf new file mode 100644 index 000000000..469fa2ddb --- /dev/null +++ b/modules/secops-rules/versions.tf @@ -0,0 +1,35 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fabric release: v38.1.0 + +terraform { + required_version = ">= 1.10.2" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.28.0, < 7.0.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.28.0, < 7.0.0" # tftest + } + } + provider_meta "google" { + module_name = "google-pso-tool/cloud-foundation-fabric/path:v38.1.0-tf" + } + provider_meta "google-beta" { + module_name = "google-pso-tool/cloud-foundation-fabric/path:v38.1.0-tf" + } +} diff --git a/modules/secops-rules/versions.tofu b/modules/secops-rules/versions.tofu new file mode 100644 index 000000000..33635f966 --- /dev/null +++ b/modules/secops-rules/versions.tofu @@ -0,0 +1,35 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fabric release: v38.1.0 + +terraform { + required_version = ">= 1.9.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.28.0, < 7.0.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.28.0, < 7.0.0" # tftest + } + } + provider_meta "google" { + module_name = "google-pso-tool/cloud-foundation-fabric/path:v38.1.0-tofu" + } + provider_meta "google-beta" { + module_name = "google-pso-tool/cloud-foundation-fabric/path:v38.1.0-tofu" + } +} diff --git a/tests/examples/variables.tf b/tests/examples/variables.tf index 80f76c234..6ba4c363f 100644 --- a/tests/examples/variables.tf +++ b/tests/examples/variables.tf @@ -61,6 +61,13 @@ variable "regions" { } } +variable "secops_tenant_config" { + default = { + customer_id = "customer-id" + region = "europe" + } +} + variable "service_account" { default = { id = "service_account_id" diff --git a/tests/modules/secops_rules/examples/basic.yaml b/tests/modules/secops_rules/examples/basic.yaml new file mode 100644 index 000000000..9c9f6d423 --- /dev/null +++ b/tests/modules/secops_rules/examples/basic.yaml @@ -0,0 +1,76 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.secops.google_chronicle_reference_list.default["private_ip_ranges"]: + description: Private CIDR ranges + entries: + - value: 10.0.0.0/8 + - value: 172.16.0.0/12 + - value: 192.168.0.0/16 + - value: 127.0.0.1/32 + - value: ::1/128 + - value: fc00::/7 + - value: fe80::/10 + - value: '# tftest-file id=reference path=data/reference_lists/private_ip_ranges.txt' + - value: '' + instance: customer-id + location: europe + project: project-id + reference_list_id: private_ip_ranges + syntax_type: REFERENCE_LIST_SYNTAX_TYPE_CIDR + timeouts: null + module.secops.google_chronicle_rule.default["network_traffic_to_specific_country"]: + deletion_policy: FORCE + instance: customer-id + location: europe + project: project-id + scope: null + text: "rule network_traffic_to_specific_country {\n\nmeta:\n author = \"Google\ + \ Cloud Security\"\n description = \"Identify network traffic based on target\ + \ country\"\n type = \"alert\"\n tags = \"geoip enrichment\"\n data_source\ + \ = \"microsoft windows events\"\n severity = \"Low\"\n priority = \"Low\"\ + \n\nevents:\n $network.metadata.event_type = \"NETWORK_CONNECTION\"\n //Specify\ + \ a country of interest to monitor or add additional countries using an or statement\n\ + \ $network.target.ip_geo_artifact.location.country_or_region = \"France\" nocase\n\ + \ $network.target.ip = $ip\n\nmatch:\n $ip over 30m\n\noutcome:\n $risk_score\ + \ = max(35)\n $event_count = count_distinct($network.metadata.id)\n\n // added\ + \ to populate alert graph with additional context\n $principal_ip = array_distinct($network.principal.ip)\n\ + \n // Commented out target.ip because it is already represented in graph as\ + \ match variable. If match changes, can uncomment to add to results\n //$target_ip\ + \ = array_distinct($network.target.ip)\n $principal_process_pid = array_distinct($network.principal.process.pid)\n\ + \ $principal_process_command_line = array_distinct($network.principal.process.command_line)\n\ + \ $principal_process_file_sha256 = array_distinct($network.principal.process.file.sha256)\n\ + \ $principal_process_file_full_path = array_distinct($network.principal.process.file.full_path)\n\ + \ $principal_process_product_specfic_process_id = array_distinct($network.principal.process.product_specific_process_id)\n\ + \ $principal_process_parent_process_product_specfic_process_id = array_distinct($network.principal.process.parent_process.product_specific_process_id)\n\ + \ $target_process_pid = array_distinct($network.target.process.pid)\n $target_process_command_line\ + \ = array_distinct($network.target.process.command_line)\n $target_process_file_sha256\ + \ = array_distinct($network.target.process.file.sha256)\n $target_process_file_full_path\ + \ = array_distinct($network.target.process.file.full_path)\n $target_process_product_specfic_process_id\ + \ = array_distinct($network.target.process.product_specific_process_id)\n $target_process_parent_process_product_specfic_process_id\ + \ = array_distinct($network.target.process.parent_process.product_specific_process_id)\n\ + \ $principal_user_userid = array_distinct($network.principal.user.userid)\n\ + \ $target_user_userid = array_distinct($network.target.user.userid)\n\ncondition:\n\ + \ $network\n}\n# tftest-file id=rule path=data/rules/network_traffic_to_specific_country.yaral\n" + timeouts: null + module.secops.google_chronicle_rule_deployment.default["network_traffic_to_specific_country"]: + alerting: true + archived: false + enabled: true + instance: customer-id + location: europe + project: project-id + run_frequency: LIVE + timeouts: null diff --git a/tests/modules/secops_rules/examples/factory.yaml b/tests/modules/secops_rules/examples/factory.yaml new file mode 100644 index 000000000..aaca9c6f0 --- /dev/null +++ b/tests/modules/secops_rules/examples/factory.yaml @@ -0,0 +1,76 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.secops.google_chronicle_reference_list.default["private_ip_ranges"]: + description: Private CIDR ranges + entries: + - value: 10.0.0.0/8 + - value: 172.16.0.0/12 + - value: 192.168.0.0/16 + - value: 127.0.0.1/32 + - value: ::1/128 + - value: fc00::/7 + - value: fe80::/10 + - value: '# tftest-file id=reference path=data/reference_lists/private_ip_ranges.txt' + - value: '' + instance: customer-id + location: europe + project: project-id + reference_list_id: private_ip_ranges + syntax_type: REFERENCE_LIST_SYNTAX_TYPE_CIDR + timeouts: null + module.secops.google_chronicle_rule.default["network_traffic_to_specific_country"]: + deletion_policy: FORCE + instance: customer-id + location: europe + project: project-id + scope: null + text: "rule network_traffic_to_specific_country {\n\nmeta:\n author = \"Google\ + \ Cloud Security\"\n description = \"Identify network traffic based on target\ + \ country\"\n type = \"alert\"\n tags = \"geoip enrichment\"\n data_source\ + \ = \"microsoft windows events\"\n severity = \"Low\"\n priority = \"Low\"\ + \n\nevents:\n $network.metadata.event_type = \"NETWORK_CONNECTION\"\n //Specify\ + \ a country of interest to monitor or add additional countries using an or statement\n\ + \ $network.target.ip_geo_artifact.location.country_or_region = \"France\" nocase\n\ + \ $network.target.ip = $ip\n\nmatch:\n $ip over 30m\n\noutcome:\n $risk_score\ + \ = max(35)\n $event_count = count_distinct($network.metadata.id)\n\n // added\ + \ to populate alert graph with additional context\n $principal_ip = array_distinct($network.principal.ip)\n\ + \n // Commented out target.ip because it is already represented in graph as\ + \ match variable. If match changes, can uncomment to add to results\n //$target_ip\ + \ = array_distinct($network.target.ip)\n $principal_process_pid = array_distinct($network.principal.process.pid)\n\ + \ $principal_process_command_line = array_distinct($network.principal.process.command_line)\n\ + \ $principal_process_file_sha256 = array_distinct($network.principal.process.file.sha256)\n\ + \ $principal_process_file_full_path = array_distinct($network.principal.process.file.full_path)\n\ + \ $principal_process_product_specfic_process_id = array_distinct($network.principal.process.product_specific_process_id)\n\ + \ $principal_process_parent_process_product_specfic_process_id = array_distinct($network.principal.process.parent_process.product_specific_process_id)\n\ + \ $target_process_pid = array_distinct($network.target.process.pid)\n $target_process_command_line\ + \ = array_distinct($network.target.process.command_line)\n $target_process_file_sha256\ + \ = array_distinct($network.target.process.file.sha256)\n $target_process_file_full_path\ + \ = array_distinct($network.target.process.file.full_path)\n $target_process_product_specfic_process_id\ + \ = array_distinct($network.target.process.product_specific_process_id)\n $target_process_parent_process_product_specfic_process_id\ + \ = array_distinct($network.target.process.parent_process.product_specific_process_id)\n\ + \ $principal_user_userid = array_distinct($network.principal.user.userid)\n\ + \ $target_user_userid = array_distinct($network.target.user.userid)\n\ncondition:\n\ + \ $network\n}\n# tftest-file id=rule path=data/rules/network_traffic_to_specific_country.yaral\n" + timeouts: null + module.secops.google_chronicle_rule_deployment.default["network_traffic_to_specific_country"]: + alerting: true + archived: false + enabled: true + instance: customer-id + location: europe + project: project-id + run_frequency: DAILY + timeouts: null diff --git a/tools/duplicate-diff.py b/tools/duplicate-diff.py index 895dcc04a..a87febfc6 100644 --- a/tools/duplicate-diff.py +++ b/tools/duplicate-diff.py @@ -19,9 +19,9 @@ import sys duplicates = [ [ - # "modules/net-vpc-factory/factory-projects-object.tf", - # data factory + "modules/net-vpc-factory/factory-projects-object.tf", "modules/project-factory/factory-projects-object.tf", + # data factory ], ]