diff --git a/CHANGELOG.md b/CHANGELOG.md
index ede4313fd..272c238c1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,10 @@ All notable changes to this project will be documented in this file.
- add support for [Private Service Connect](https://cloud.google.com/vpc/docs/private-service-connect#psc-subnets) and [Proxy-only](https://cloud.google.com/load-balancing/docs/proxy-only-subnets) subnets to `net-vpc` module
- bump Google provider versions to `>= 4.17.0`
- bump Terraform version to `>= 1.1.0`
+- add `shielded_instance_config` support for instance template on `compute-vm` module
+- add support for `gke_backup_agent_config` to GKE module addons
+- add support for subscription filters to PubSub module
+- update Hub and Spoke with VPN example
**FAST**
diff --git a/examples/cloud-operations/network-dashboard/cloud-function/main.py b/examples/cloud-operations/network-dashboard/cloud-function/main.py
index a3ec98348..ecb618ad1 100644
--- a/examples/cloud-operations/network-dashboard/cloud-function/main.py
+++ b/examples/cloud-operations/network-dashboard/cloud-function/main.py
@@ -15,32 +15,71 @@
#
from code import interact
+from distutils.command.config import config
import os
from pickletools import int4
import time
-import http
-import yaml
-from collections import defaultdict
-from google.api import metric_pb2 as ga_metric
-from google.api_core import exceptions, protobuf_helpers
from google.cloud import monitoring_v3, asset_v1
from google.protobuf import field_mask_pb2
-from googleapiclient import discovery, errors
+from googleapiclient import discovery
+from metrics import ilb_fwrules, instances, networks, metrics, limits, peerings, routes
-# Organization ID containing the projects to be monitored
-ORGANIZATION_ID = os.environ.get("ORGANIZATION_ID")
-# list of projects from which function will get quotas information
-MONITORED_PROJECTS_LIST = os.environ.get("MONITORED_PROJECTS_LIST").split(",")
-# project where the metrics and dahsboards will be created
-MONITORING_PROJECT_ID = os.environ.get("MONITORING_PROJECT_ID")
-MONITORING_PROJECT_LINK = f"projects/{MONITORING_PROJECT_ID}"
-service = discovery.build('compute', 'v1')
-# Existing GCP metrics per network
-GCE_INSTANCES_LIMIT_METRIC = "compute.googleapis.com/quota/instances_per_vpc_network/limit"
-L4_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit"
-L7_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit"
-SUBNET_RANGES_LIMIT_METRIC = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit"
+def monitoring_interval():
+ '''
+ Creates the monitoring interval of 24 hours
+
+ Returns:
+ monitoring_v3.TimeInterval: Moinitoring time interval of 24h
+ '''
+ now = time.time()
+ seconds = int(now)
+ nanos = int((now - seconds) * 10**9)
+ return monitoring_v3.TimeInterval({
+ "end_time": {
+ "seconds": seconds,
+ "nanos": nanos
+ },
+ "start_time": {
+ "seconds": (seconds - 24 * 60 * 60),
+ "nanos": nanos
+ },
+ })
+
+
+config = {
+ # Organization ID containing the projects to be monitored
+ "organization":
+ os.environ.get("ORGANIZATION_ID"),
+ # list of projects from which function will get quotas information
+ "monitored_projects":
+ os.environ.get("MONITORED_PROJECTS_LIST").split(","),
+ "monitoring_project_link":
+ os.environ.get('MONITORING_PROJECT_ID'),
+ "monitoring_project_link":
+ f"projects/{os.environ.get('MONITORING_PROJECT_ID')}",
+ "monitoring_interval":
+ monitoring_interval(),
+ "limit_names": {
+ "GCE_INSTANCES":
+ "compute.googleapis.com/quota/instances_per_vpc_network/limit",
+ "L4":
+ "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit",
+ "L7":
+ "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit",
+ "SUBNET_RANGES":
+ "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit"
+ },
+ "lb_scheme": {
+ "L7": "INTERNAL_MANAGED",
+ "L4": "INTERNAL"
+ },
+ "clients": {
+ "discovery_client": discovery.build('compute', 'v1'),
+ "asset_client": asset_v1.AssetServiceClient(),
+ "monitoring_client": monitoring_v3.MetricServiceClient()
+ }
+}
def main(event, context):
@@ -53,1167 +92,61 @@ def main(event, context):
Returns:
'Function executed successfully'
'''
- metrics_dict, limits_dict = create_metrics()
+
+ # Keep the monitoring interval up2date during each run
+ config["monitoring_interval"] = monitoring_interval()
+
+ metrics_dict, limits_dict = metrics.create_metrics(
+ config["monitoring_project_link"])
# Asset inventory queries
- gce_instance_dict = get_gce_instance_dict()
- l4_forwarding_rules_dict = get_l4_forwarding_rules_dict()
- l7_forwarding_rules_dict = get_l7_forwarding_rules_dict()
- subnet_range_dict = get_subnet_ranges_dict()
+ gce_instance_dict = instances.get_gce_instance_dict(config)
+ l4_forwarding_rules_dict = ilb_fwrules.get_forwarding_rules_dict(config, "L4")
+ l7_forwarding_rules_dict = ilb_fwrules.get_forwarding_rules_dict(config, "L7")
+ subnet_range_dict = networks.get_subnet_ranges_dict(config)
# Per Network metrics
- get_gce_instances_data(metrics_dict, gce_instance_dict,
- limits_dict['number_of_instances_limit'])
- get_l4_forwarding_rules_data(
- metrics_dict, l4_forwarding_rules_dict,
- limits_dict['internal_forwarding_rules_l4_limit'])
- get_l7_forwarding_rules_data(
- metrics_dict, l7_forwarding_rules_dict,
- limits_dict['internal_forwarding_rules_l7_limit'])
- get_vpc_peering_data(metrics_dict,
- limits_dict['number_of_vpc_peerings_limit'])
- dynamic_routes_dict = get_dynamic_routes(
- metrics_dict, limits_dict['dynamic_routes_per_network_limit'])
+ instances.get_gce_instances_data(config, metrics_dict, gce_instance_dict,
+ limits_dict['number_of_instances_limit'])
+ ilb_fwrules.get_forwarding_rules_data(
+ config, metrics_dict, l4_forwarding_rules_dict,
+ limits_dict['internal_forwarding_rules_l4_limit'], "L4")
+ ilb_fwrules.get_forwarding_rules_data(
+ config, metrics_dict, l7_forwarding_rules_dict,
+ limits_dict['internal_forwarding_rules_l7_limit'], "L7")
+ peerings.get_vpc_peering_data(config, metrics_dict,
+ limits_dict['number_of_vpc_peerings_limit'])
+ dynamic_routes_dict = routes.get_dynamic_routes(
+ config, metrics_dict, limits_dict['dynamic_routes_per_network_limit'])
# Per VPC peering group metrics
- get_pgg_data(
+ metrics.get_pgg_data(
+ config,
metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"],
- gce_instance_dict, GCE_INSTANCES_LIMIT_METRIC,
+ gce_instance_dict, config["limit_names"]["GCE_INSTANCES"],
limits_dict['number_of_instances_ppg_limit'])
-
- get_pgg_data(
- metrics_dict["metrics_per_peering_group"]
+ metrics.get_pgg_data(
+ config, metrics_dict["metrics_per_peering_group"]
["l4_forwarding_rules_per_peering_group"], l4_forwarding_rules_dict,
- L4_FORWARDING_RULES_LIMIT_METRIC,
+ config["limit_names"]["L4"],
limits_dict['internal_forwarding_rules_l4_ppg_limit'])
-
- get_pgg_data(
- metrics_dict["metrics_per_peering_group"]
+ metrics.get_pgg_data(
+ config, metrics_dict["metrics_per_peering_group"]
["l7_forwarding_rules_per_peering_group"], l7_forwarding_rules_dict,
- L7_FORWARDING_RULES_LIMIT_METRIC,
+ config["limit_names"]["L7"],
limits_dict['internal_forwarding_rules_l7_ppg_limit'])
-
- get_pgg_data(
- metrics_dict["metrics_per_peering_group"]
+ metrics.get_pgg_data(
+ config, metrics_dict["metrics_per_peering_group"]
["subnet_ranges_per_peering_group"], subnet_range_dict,
- SUBNET_RANGES_LIMIT_METRIC,
+ config["limit_names"]["SUBNET_RANGES"],
limits_dict['number_of_subnet_IP_ranges_ppg_limit'])
-
- get_dynamic_routes_ppg(
- metrics_dict["metrics_per_peering_group"]
+ routes.get_dynamic_routes_ppg(
+ config, metrics_dict["metrics_per_peering_group"]
["dynamic_routes_per_peering_group"], dynamic_routes_dict,
limits_dict['number_of_subnet_IP_ranges_ppg_limit'])
return 'Function executed successfully'
-def get_l4_forwarding_rules_dict():
- '''
- Calls the Asset Inventory API to get all L4 Forwarding Rules under the GCP organization.
-
- Parameters:
- None
- Returns:
- forwarding_rules_dict (dictionary of string: int): Keys are the network links and values are the number of Forwarding Rules per network.
- '''
- client = asset_v1.AssetServiceClient()
-
- read_mask = field_mask_pb2.FieldMask()
- read_mask.FromJsonString('name,versionedResources')
-
- forwarding_rules_dict = defaultdict(int)
-
- response = client.search_all_resources(
- request={
- "scope": f"organizations/{ORGANIZATION_ID}",
- "asset_types": ["compute.googleapis.com/ForwardingRule"],
- "read_mask": read_mask,
- })
- for resource in response:
- internal = False
- network_link = ""
- for versioned in resource.versioned_resources:
- for field_name, field_value in versioned.resource.items():
- if field_name == "loadBalancingScheme":
- internal = (field_value == "INTERNAL")
- if field_name == "network":
- network_link = field_value
- if internal:
- if network_link in forwarding_rules_dict:
- forwarding_rules_dict[network_link] += 1
- else:
- forwarding_rules_dict[network_link] = 1
-
- return forwarding_rules_dict
-
-
-def get_l7_forwarding_rules_dict():
- '''
- Calls the Asset Inventory API to get all L7 Forwarding Rules under the GCP organization.
-
- Parameters:
- None
- Returns:
- forwarding_rules_dict (dictionary of string: int): Keys are the network links and values are the number of Forwarding Rules per network.
- '''
- client = asset_v1.AssetServiceClient()
-
- read_mask = field_mask_pb2.FieldMask()
- read_mask.FromJsonString('name,versionedResources')
-
- forwarding_rules_dict = defaultdict(int)
-
- response = client.search_all_resources(
- request={
- "scope": f"organizations/{ORGANIZATION_ID}",
- "asset_types": ["compute.googleapis.com/ForwardingRule"],
- "read_mask": read_mask,
- })
- for resource in response:
- internal = False
- network_link = ""
- for versioned in resource.versioned_resources:
- for field_name, field_value in versioned.resource.items():
- if field_name == "loadBalancingScheme":
- internal = (field_value == "INTERNAL_MANAGED")
- if field_name == "network":
- network_link = field_value
- if internal:
- if network_link in forwarding_rules_dict:
- forwarding_rules_dict[network_link] += 1
- else:
- forwarding_rules_dict[network_link] = 1
-
- return forwarding_rules_dict
-
-
-def get_gce_instance_dict():
- '''
- Calls the Asset Inventory API to get all GCE instances under the GCP organization.
-
- Parameters:
- None
- Returns:
- gce_instance_dict (dictionary of string: int): Keys are the network links and values are the number of GCE Instances per network.
- '''
- client = asset_v1.AssetServiceClient()
-
- gce_instance_dict = defaultdict(int)
-
- response = client.search_all_resources(
- request={
- "scope": f"organizations/{ORGANIZATION_ID}",
- "asset_types": ["compute.googleapis.com/Instance"],
- })
- for resource in response:
- for field_name, field_value in resource.additional_attributes.items():
- if field_name == "networkInterfaceNetworks":
- for network in field_value:
- if network in gce_instance_dict:
- gce_instance_dict[network] += 1
- else:
- gce_instance_dict[network] = 1
-
- return gce_instance_dict
-
-
-def get_subnet_ranges_dict():
- '''
- Calls the Asset Inventory API to get all Subnet ranges under the GCP organization.
-
- Parameters:
- None
- Returns:
- subnet_range_dict (dictionary of string: int): Keys are the network links and values are the number of subnet ranges per network.
- '''
- client = asset_v1.AssetServiceClient()
- subnet_range_dict = defaultdict(int)
- read_mask = field_mask_pb2.FieldMask()
- read_mask.FromJsonString('name,versionedResources')
-
- response = client.search_all_resources(
- request={
- "scope": f"organizations/{ORGANIZATION_ID}",
- "asset_types": ["compute.googleapis.com/Subnetwork"],
- "read_mask": read_mask,
- })
- for resource in response:
- ranges = 0
- network_link = None
-
- for versioned in resource.versioned_resources:
- for field_name, field_value in versioned.resource.items():
- if field_name == "network":
- network_link = field_value
- ranges += 1
- if field_name == "secondaryIpRanges":
- for range in field_value:
- ranges += 1
-
- if network_link in subnet_range_dict:
- subnet_range_dict[network_link] += ranges
- else:
- subnet_range_dict[network_link] = ranges
-
- return subnet_range_dict
-
-
-def create_client():
- '''
- Creates the monitoring API client, that will be used to create, read and update custom metrics.
-
- Parameters:
- None
- Returns:
- client (monitoring_v3.MetricServiceClient): Monitoring API client
- interval (monitoring_v3.TimeInterval): Interval for the metric data points (24 hours)
- '''
- try:
- client = monitoring_v3.MetricServiceClient()
- now = time.time()
- seconds = int(now)
- nanos = int((now - seconds) * 10**9)
- interval = monitoring_v3.TimeInterval({
- "end_time": {
- "seconds": seconds,
- "nanos": nanos
- },
- "start_time": {
- "seconds": (seconds - 86400),
- "nanos": nanos
- },
- })
- return (client, interval)
- except Exception as e:
- raise Exception("Error occurred creating the client: {}".format(e))
-
-
-def create_metrics():
- '''
- Creates all Cloud Monitoring custom metrics based on the metric.yaml file
-
- Parameters:
- None
-
- Returns:
- metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions
- limits_dict (dictionary of dictionary of string: int): limits_dict[metric_name]: dict[network_name] = limit_value
- '''
- client = monitoring_v3.MetricServiceClient()
- existing_metrics = []
- for desc in client.list_metric_descriptors(name=MONITORING_PROJECT_LINK):
- existing_metrics.append(desc.type)
- limits_dict = {}
-
- with open("metrics.yaml", 'r') as stream:
- try:
- metrics_dict = yaml.safe_load(stream)
-
- for metric_list in metrics_dict.values():
- for metric in metric_list.values():
- for sub_metric_key, sub_metric in metric.items():
- metric_link = f"custom.googleapis.com/{sub_metric['name']}"
- # If the metric doesn't exist yet, then we create it
- if metric_link not in existing_metrics:
- create_metric(sub_metric["name"], sub_metric["description"])
- # Parse limits (both default values and network specific ones)
- if sub_metric_key == "limit":
- limits_dict_for_metric = {}
- for network_link, limit_value in sub_metric["values"].items():
- limits_dict_for_metric[network_link] = limit_value
- limits_dict[sub_metric["name"]] = limits_dict_for_metric
-
- return metrics_dict, limits_dict
- except yaml.YAMLError as exc:
- print(exc)
-
-
-def create_metric(metric_name, description):
- '''
- Creates a Cloud Monitoring metric based on the parameter given if the metric is not already existing
-
- Parameters:
- metric_name (string): Name of the metric to be created
- description (string): Description of the metric to be created
-
- Returns:
- None
- '''
- client = monitoring_v3.MetricServiceClient()
-
- descriptor = ga_metric.MetricDescriptor()
- descriptor.type = f"custom.googleapis.com/{metric_name}"
- descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE
- descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE
- descriptor.description = description
- descriptor = client.create_metric_descriptor(name=MONITORING_PROJECT_LINK,
- metric_descriptor=descriptor)
- print("Created {}.".format(descriptor.name))
-
-
-def get_gce_instances_data(metrics_dict, gce_instance_dict, limit_dict):
- '''
- Gets the data for GCE instances per VPC Network and writes it to the metric defined in instance_metric.
-
- Parameters:
- metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions
- gce_instance_dict (dictionary of string: int): Keys are the network links and values are the number of GCE Instances per network.
- limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
- Returns:
- gce_instance_dict
- '''
- # Existing GCP Monitoring metrics for GCE instances
- metric_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit"
-
- for project in MONITORED_PROJECTS_LIST:
- network_dict = get_networks(project)
-
- current_quota_limit = get_quota_current_limit(f"projects/{project}",
- metric_instances_limit)
- if current_quota_limit is None:
- print(
- f"Could not write number of instances for projects/{project} due to missing quotas"
- )
-
- current_quota_limit_view = customize_quota_view(current_quota_limit)
-
- for net in network_dict:
- set_limits(net, current_quota_limit_view, limit_dict)
-
- usage = 0
- if net['self_link'] in gce_instance_dict:
- usage = gce_instance_dict[net['self_link']]
-
- write_data_to_metric(
- project, usage, metrics_dict["metrics_per_network"]
- ["instance_per_network"]["usage"]["name"], net['network_name'])
- write_data_to_metric(
- project, net['limit'], metrics_dict["metrics_per_network"]
- ["instance_per_network"]["limit"]["name"], net['network_name'])
- write_data_to_metric(
- project, usage / net['limit'], metrics_dict["metrics_per_network"]
- ["instance_per_network"]["utilization"]["name"], net['network_name'])
-
- print(f"Wrote number of instances to metric for projects/{project}")
-
-
-def get_vpc_peering_data(metrics_dict, limit_dict):
- '''
- Gets the data for VPC peerings (active or not) and writes it to the metric defined (vpc_peering_active_metric and vpc_peering_metric).
-
- Parameters:
- metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions
- limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
- Returns:
- None
- '''
- for project in MONITORED_PROJECTS_LIST:
- active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(
- project, limit_dict)
- for peering in active_vpc_peerings:
- write_data_to_metric(
- project, peering['active_peerings'],
- metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]
- ["usage"]["name"], peering['network_name'])
- write_data_to_metric(
- project, peering['network_limit'], metrics_dict["metrics_per_network"]
- ["vpc_peering_active_per_network"]["limit"]["name"],
- peering['network_name'])
- write_data_to_metric(
- project, peering['active_peerings'] / peering['network_limit'],
- metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]
- ["utilization"]["name"], peering['network_name'])
- print("Wrote number of active VPC peerings to custom metric for project:",
- project)
-
- for peering in vpc_peerings:
- write_data_to_metric(
- project, peering['peerings'], metrics_dict["metrics_per_network"]
- ["vpc_peering_per_network"]["usage"]["name"], peering['network_name'])
- write_data_to_metric(
- project, peering['network_limit'], metrics_dict["metrics_per_network"]
- ["vpc_peering_per_network"]["limit"]["name"], peering['network_name'])
- write_data_to_metric(
- project, peering['peerings'] / peering['network_limit'],
- metrics_dict["metrics_per_network"]["vpc_peering_per_network"]
- ["utilization"]["name"], peering['network_name'])
- print("Wrote number of VPC peerings to custom metric for project:", project)
-
-
-def gather_vpc_peerings_data(project_id, limit_dict):
- '''
- Gets the data for all VPC peerings (active or not) in project_id and writes it to the metric defined in vpc_peering_active_metric and vpc_peering_metric.
-
- Parameters:
- project_id (string): We will take all VPCs in that project_id and look for all peerings to these VPCs.
- limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
- Returns:
- active_peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each active VPC peering.
- peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each VPC peering.
- '''
- active_peerings_dict = []
- peerings_dict = []
- request = service.networks().list(project=project_id)
- response = request.execute()
- if 'items' in response:
- for network in response['items']:
- if 'peerings' in network:
- STATE = network['peerings'][0]['state']
- if STATE == "ACTIVE":
- active_peerings_count = len(network['peerings'])
- else:
- active_peerings_count = 0
-
- peerings_count = len(network['peerings'])
- else:
- peerings_count = 0
- active_peerings_count = 0
-
- network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network['name']}"
- network_limit = get_limit_ppg(network_link, limit_dict)
-
- active_d = {
- 'project_id': project_id,
- 'network_name': network['name'],
- 'active_peerings': active_peerings_count,
- 'network_limit': network_limit
- }
- active_peerings_dict.append(active_d)
- d = {
- 'project_id': project_id,
- 'network_name': network['name'],
- 'peerings': peerings_count,
- 'network_limit': network_limit
- }
- peerings_dict.append(d)
-
- return active_peerings_dict, peerings_dict
-
-
-def get_limit_ppg(network_link, limit_dict):
- '''
- Checks if this network has a specific limit for a metric, if so, returns that limit, if not, returns the default limit.
-
- Parameters:
- network_link (string): VPC network link.
- limit_list (list of string): Used to get the limit per VPC or the default limit.
- Returns:
- limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
- '''
- if network_link in limit_dict:
- return limit_dict[network_link]
- else:
- if 'default_value' in limit_dict:
- return limit_dict['default_value']
- else:
- print(f"Error: limit not found for {network_link}")
- return 0
-
-
-def get_l4_forwarding_rules_data(metrics_dict, forwarding_rules_dict,
- limit_dict):
- '''
- Gets the data for L4 Internal Forwarding Rules per VPC Network and writes it to the metric defined in forwarding_rules_metric.
-
- Parameters:
- metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions.
- forwarding_rules_dict (dictionary of string: int): Keys are the network links and values are the number of Forwarding Rules per network.
- limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value.
- Returns:
- None
- '''
- for project in MONITORED_PROJECTS_LIST:
- network_dict = get_networks(project)
-
- current_quota_limit = get_quota_current_limit(
- f"projects/{project}", L4_FORWARDING_RULES_LIMIT_METRIC)
-
- if current_quota_limit is None:
- print(
- f"Could not write L4 forwarding rules to metric for projects/{project} due to missing quotas"
- )
- continue
-
- current_quota_limit_view = customize_quota_view(current_quota_limit)
-
- for net in network_dict:
- set_limits(net, current_quota_limit_view, limit_dict)
-
- usage = 0
- if net['self_link'] in forwarding_rules_dict:
- usage = forwarding_rules_dict[net['self_link']]
-
- write_data_to_metric(
- project, usage, metrics_dict["metrics_per_network"]
- ["l4_forwarding_rules_per_network"]["usage"]["name"],
- net['network_name'])
- write_data_to_metric(
- project, net['limit'], metrics_dict["metrics_per_network"]
- ["l4_forwarding_rules_per_network"]["limit"]["name"],
- net['network_name'])
- write_data_to_metric(
- project, usage / net['limit'], metrics_dict["metrics_per_network"]
- ["l4_forwarding_rules_per_network"]["utilization"]["name"],
- net['network_name'])
-
- print(
- f"Wrote number of L4 forwarding rules to metric for projects/{project}")
-
-
-def get_l7_forwarding_rules_data(metrics_dict, forwarding_rules_dict,
- limit_dict):
- '''
- Gets the data for L7 Internal Forwarding Rules per VPC Network and writes it to the metric defined in forwarding_rules_metric.
-
- Parameters:
- metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions.
- forwarding_rules_dict (dictionary of string: int): Keys are the network links and values are the number of Forwarding Rules per network.
- limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value.
- Returns:
- None
- '''
- for project in MONITORED_PROJECTS_LIST:
- network_dict = get_networks(project)
-
- current_quota_limit = get_quota_current_limit(
- f"projects/{project}", L7_FORWARDING_RULES_LIMIT_METRIC)
- if current_quota_limit is None:
- print(
- f"Could not write number of L7 forwarding rules to metric for projects/{project} due to missing quotas"
- )
- continue
-
- current_quota_limit_view = customize_quota_view(current_quota_limit)
-
- for net in network_dict:
- set_limits(net, current_quota_limit_view, limit_dict)
-
- usage = 0
- if net['self_link'] in forwarding_rules_dict:
- usage = forwarding_rules_dict[net['self_link']]
-
- write_data_to_metric(
- project, usage, metrics_dict["metrics_per_network"]
- ["l7_forwarding_rules_per_network"]["usage"]["name"],
- net['network_name'])
- write_data_to_metric(
- project, net['limit'], metrics_dict["metrics_per_network"]
- ["l7_forwarding_rules_per_network"]["limit"]["name"],
- net['network_name'])
- write_data_to_metric(
- project, usage / net['limit'], metrics_dict["metrics_per_network"]
- ["l7_forwarding_rules_per_network"]["utilization"]["name"],
- net['network_name'])
-
- print(
- f"Wrote number of L7 forwarding rules to metric for projects/{project}")
-
-
-def get_pgg_data(metric_dict, usage_dict, limit_metric, limit_dict):
- '''
- This function gets the usage, limit and utilization per VPC peering group for a specific metric for all projects to be monitored.
-
- Parameters:
- metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics
- usage_dict (dictionnary of string:int): Dictionary with the network link as key and the number of resources as value
- limit_metric (string): Name of the existing GCP metric for limit per VPC network
- limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
- Returns:
- None
- '''
- for project in MONITORED_PROJECTS_LIST:
- network_dict_list = gather_peering_data(project)
- # Network dict list is a list of dictionary (one for each network)
- # For each network, this dictionary contains:
- # project_id, network_name, network_id, usage, limit, peerings (list of peered networks)
- # peerings is a list of dictionary (one for each peered network) and contains:
- # project_id, network_name, network_id
- current_quota_limit = get_quota_current_limit(f"projects/{project}",
- limit_metric)
- if current_quota_limit is None:
- print(
- f"Could not write number of L7 forwarding rules to metric for projects/{project} due to missing quotas"
- )
- continue
-
- current_quota_limit_view = customize_quota_view(current_quota_limit)
-
- # For each network in this GCP project
- for network_dict in network_dict_list:
- if network_dict['network_id'] == 0:
- print(
- f"Could not write {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project} due to missing permissions."
- )
- continue
- network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{network_dict['network_name']}"
-
- limit = get_limit_network(network_dict, network_link,
- current_quota_limit_view, limit_dict)
-
- usage = 0
- if network_link in usage_dict:
- usage = usage_dict[network_link]
-
- # Here we add usage and limit to the network dictionary
- network_dict["usage"] = usage
- network_dict["limit"] = limit
-
- # For every peered network, get usage and limits
- for peered_network_dict in network_dict['peerings']:
- peered_network_link = f"https://www.googleapis.com/compute/v1/projects/{peered_network_dict['project_id']}/global/networks/{peered_network_dict['network_name']}"
- peered_usage = 0
- if peered_network_link in usage_dict:
- peered_usage = usage_dict[peered_network_link]
-
- current_peered_quota_limit = get_quota_current_limit(
- f"projects/{peered_network_dict['project_id']}", limit_metric)
- if current_peered_quota_limit is None:
- print(
- f"Could not write metrics for peering to projects/{peered_network_dict['project_id']} due to missing quotas"
- )
- continue
-
- peering_project_limit = customize_quota_view(current_peered_quota_limit)
-
- peered_limit = get_limit_network(peered_network_dict,
- peered_network_link,
- peering_project_limit, limit_dict)
- # Here we add usage and limit to the peered network dictionary
- peered_network_dict["usage"] = peered_usage
- peered_network_dict["limit"] = peered_limit
-
- count_effective_limit(project, network_dict, metric_dict["usage"]["name"],
- metric_dict["limit"]["name"],
- metric_dict["utilization"]["name"], limit_dict)
- print(
- f"Wrote {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project}"
- )
-
-
-def get_dynamic_routes_ppg(metric_dict, usage_dict, limit_dict):
- '''
- This function gets the usage, limit and utilization for the dynamic routes per VPC peering group.
-
- Parameters:
- metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics
- usage_dict (dictionnary of string:int): Dictionary with the network link as key and the number of resources as value
- limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
- Returns:
- None
- '''
- for project in MONITORED_PROJECTS_LIST:
- network_dict_list = gather_peering_data(project)
-
- for network_dict in network_dict_list:
- network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{network_dict['network_name']}"
-
- limit = get_limit_ppg(network_link, limit_dict)
-
- usage = 0
- if network_link in usage_dict:
- usage = usage_dict[network_link]
-
- # Here we add usage and limit to the network dictionary
- network_dict["usage"] = usage
- network_dict["limit"] = limit
-
- # For every peered network, get usage and limits
- for peered_network_dict in network_dict['peerings']:
- peered_network_link = f"https://www.googleapis.com/compute/v1/projects/{peered_network_dict['project_id']}/global/networks/{peered_network_dict['network_name']}"
- peered_usage = 0
- if peered_network_link in usage_dict:
- peered_usage = usage_dict[peered_network_link]
-
- peered_limit = get_limit_ppg(peered_network_link, limit_dict)
-
- # Here we add usage and limit to the peered network dictionary
- peered_network_dict["usage"] = peered_usage
- peered_network_dict["limit"] = peered_limit
-
- count_effective_limit(project, network_dict, metric_dict["usage"]["name"],
- metric_dict["limit"]["name"],
- metric_dict["utilization"]["name"], limit_dict)
- print(
- f"Wrote {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project}"
- )
-
-
-def count_effective_limit(project_id, network_dict, usage_metric_name,
- limit_metric_name, utilization_metric_name,
- limit_dict):
- '''
- Calculates the effective limits (using algorithm in the link below) for peering groups and writes data (usage, limit, utilization) to the custom metrics.
- Source: https://cloud.google.com/vpc/docs/quota#vpc-peering-effective-limit
-
- Parameters:
- project_id (string): Project ID for the project to be analyzed.
- network_dict (dictionary of string: string): Contains all required information about the network to get the usage, limit and utilization.
- usage_metric_name (string): Name of the custom metric to be populated for usage per VPC peering group.
- limit_metric_name (string): Name of the custom metric to be populated for limit per VPC peering group.
- utilization_metric_name (string): Name of the custom metric to be populated for utilization per VPC peering group.
- limit_dict (dictionary of string:int): Dictionary containing the limit per peering group (either VPC specific or default limit).
- Returns:
- None
- '''
-
- if network_dict['peerings'] == []:
- return
-
- # Get usage: Sums usage for current network + all peered networks
- peering_group_usage = network_dict['usage']
- for peered_network in network_dict['peerings']:
- if 'usage' not in peered_network:
- print(
- f"Can not add metrics for peered network in projects/{project_id} as no usage metrics exist due to missing permissions"
- )
- continue
- peering_group_usage += peered_network['usage']
-
- network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network_dict['network_name']}"
-
- # Calculates effective limit: Step 1: max(per network limit, per network_peering_group limit)
- limit_step1 = max(network_dict['limit'],
- get_limit_ppg(network_link, limit_dict))
-
- # Calculates effective limit: Step 2: List of max(per network limit, per network_peering_group limit) for each peered network
- limit_step2 = []
- for peered_network in network_dict['peerings']:
- peered_network_link = f"https://www.googleapis.com/compute/v1/projects/{peered_network['project_id']}/global/networks/{peered_network['network_name']}"
-
- if 'limit' in peered_network:
- limit_step2.append(
- max(peered_network['limit'],
- get_limit_ppg(peered_network_link, limit_dict)))
- else:
- print(
- f"Ignoring projects/{peered_network['project_id']} for limits in peering group of project {project_id} as no limits are available."
- +
- "This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project"
- )
-
- # Calculates effective limit: Step 3: Find minimum from the list created by Step 2
- limit_step3 = 0
- if len(limit_step2) > 0:
- limit_step3 = min(limit_step2)
-
- # Calculates effective limit: Step 4: Find maximum from step 1 and step 3
- effective_limit = max(limit_step1, limit_step3)
- utilization = peering_group_usage / effective_limit
-
- write_data_to_metric(project_id, peering_group_usage, usage_metric_name,
- network_dict['network_name'])
- write_data_to_metric(project_id, effective_limit, limit_metric_name,
- network_dict['network_name'])
- write_data_to_metric(project_id, utilization, utilization_metric_name,
- network_dict['network_name'])
-
-
-def get_networks(project_id):
- '''
- Returns a dictionary of all networks in a project.
-
- Parameters:
- project_id (string): Project ID for the project containing the networks.
- Returns:
- network_dict (dictionary of string: string): Contains the project_id, network_name(s) and network_id(s)
- '''
- request = service.networks().list(project=project_id)
- response = request.execute()
- network_dict = []
- if 'items' in response:
- for network in response['items']:
- network_name = network['name']
- network_id = network['id']
- self_link = network['selfLink']
- d = {
- 'project_id': project_id,
- 'network_name': network_name,
- 'network_id': network_id,
- 'self_link': self_link
- }
- network_dict.append(d)
- return network_dict
-
-
-def get_dynamic_routes(metrics_dict, limits_dict):
- '''
- Writes all dynamic routes per VPC to custom metrics.
-
- Parameters:
- metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions.
- limits_dict (dictionary of string: int): key is network link (or 'default_value') and value is the limit for that network
- Returns:
- dynamic_routes_dict (dictionary of string: int): key is network link and value is the number of dynamic routes for that network
- '''
- routers_dict = get_routers()
- dynamic_routes_dict = defaultdict(int)
-
- for project_id in MONITORED_PROJECTS_LIST:
- network_dict = get_networks(project_id)
-
- for network in network_dict:
- sum_routes = get_routes_for_network(network['self_link'], project_id,
- routers_dict)
- dynamic_routes_dict[network['self_link']] = sum_routes
-
- if network['self_link'] in limits_dict:
- limit = limits_dict[network['self_link']]
- else:
- if 'default_value' in limits_dict:
- limit = limits_dict['default_value']
- else:
- print("Error: couldn't find limit for dynamic routes.")
- break
-
- utilization = sum_routes / limit
-
- write_data_to_metric(
- project_id, sum_routes, metrics_dict["metrics_per_network"]
- ["dynamic_routes_per_network"]["usage"]["name"],
- network['network_name'])
- write_data_to_metric(
- project_id, limit, metrics_dict["metrics_per_network"]
- ["dynamic_routes_per_network"]["limit"]["name"],
- network['network_name'])
- write_data_to_metric(
- project_id, utilization, metrics_dict["metrics_per_network"]
- ["dynamic_routes_per_network"]["utilization"]["name"],
- network['network_name'])
-
- print("Wrote metrics for dynamic routes for VPCs in project", project_id)
-
- return dynamic_routes_dict
-
-
-def get_routes_for_network(network_link, project_id, routers_dict):
- '''
- Returns a the number of dynamic routes for a given network
-
- Parameters:
- network_link (string): Network self link.
- project_id (string): Project ID containing the network.
- routers_dict (dictionary of string: list of string): Dictionary with key as network link and value as list of router links.
- Returns:
- sum_routes (int): Number of routes in that network.
- '''
- sum_routes = 0
-
- if network_link in routers_dict:
- for router_link in routers_dict[network_link]:
- # Router link is using the following format:
- # 'https://www.googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/routers/ROUTER_NAME'
- start = router_link.find("/regions/") + len("/regions/")
- end = router_link.find("/routers/")
- router_region = router_link[start:end]
- router_name = router_link.split('/routers/')[1]
- routes = get_routes_for_router(project_id, router_region, router_name)
-
- sum_routes += routes
-
- return sum_routes
-
-
-def get_routes_for_router(project_id, router_region, router_name):
- '''
- Returns the same of dynamic routes learned by a specific Cloud Router instance
-
- Parameters:
- project_id (string): Project ID for the project containing the Cloud Router.
- router_region (string): GCP region for the Cloud Router.
- router_name (string): Cloud Router name.
- Returns:
- sum_routes (int): Number of dynamic routes learned by the Cloud Router.
- '''
- request = service.routers().getRouterStatus(project=project_id,
- region=router_region,
- router=router_name)
- response = request.execute()
-
- sum_routes = 0
-
- if 'result' in response:
- if 'bgpPeerStatus' in response['result']:
- for peer in response['result']['bgpPeerStatus']:
- sum_routes += peer['numLearnedRoutes']
-
- return sum_routes
-
-
-def get_routers():
- '''
- Returns a dictionary of all Cloud Routers in the GCP organization.
-
- Parameters:
- None
- Returns:
- routers_dict (dictionary of string: list of string): Key is the network link and value is a list of router links.
- '''
- client = asset_v1.AssetServiceClient()
-
- read_mask = field_mask_pb2.FieldMask()
- read_mask.FromJsonString('name,versionedResources')
-
- routers_dict = {}
-
- response = client.search_all_resources(
- request={
- "scope": f"organizations/{ORGANIZATION_ID}",
- "asset_types": ["compute.googleapis.com/Router"],
- "read_mask": read_mask,
- })
- for resource in response:
- network_link = None
- router_link = None
- for versioned in resource.versioned_resources:
- for field_name, field_value in versioned.resource.items():
- if field_name == "network":
- network_link = field_value
- if field_name == "selfLink":
- router_link = field_value
-
- if network_link in routers_dict:
- routers_dict[network_link].append(router_link)
- else:
- routers_dict[network_link] = [router_link]
-
- return routers_dict
-
-
-def gather_peering_data(project_id):
- '''
- Returns a dictionary of all peerings for all networks in a project.
-
- Parameters:
- project_id (string): Project ID for the project containing the networks.
- Returns:
- network_list (dictionary of string: string): Contains the project_id, network_name(s) and network_id(s) of peered networks.
- '''
- request = service.networks().list(project=project_id)
- response = request.execute()
-
- network_list = []
- if 'items' in response:
- for network in response['items']:
- net = {
- 'project_id': project_id,
- 'network_name': network['name'],
- 'network_id': network['id'],
- 'peerings': []
- }
- if 'peerings' in network:
- STATE = network['peerings'][0]['state']
- if STATE == "ACTIVE":
- for peered_network in network[
- 'peerings']: # "projects/{project_name}/global/networks/{network_name}"
- start = peered_network['network'].find("projects/") + len(
- 'projects/')
- end = peered_network['network'].find("/global")
- peered_project = peered_network['network'][start:end]
- peered_network_name = peered_network['network'].split(
- "networks/")[1]
- peered_net = {
- 'project_id':
- peered_project,
- 'network_name':
- peered_network_name,
- 'network_id':
- get_network_id(peered_project, peered_network_name)
- }
- net["peerings"].append(peered_net)
- network_list.append(net)
- return network_list
-
-
-def get_network_id(project_id, network_name):
- '''
- Returns the network_id for a specific project / network name.
-
- Parameters:
- project_id (string): Project ID for the project containing the networks.
- network_name (string): Name of the network
- Returns:
- network_id (int): Network ID.
- '''
- request = service.networks().list(project=project_id)
- try:
- response = request.execute()
- except errors.HttpError as err:
- # TODO: log proper warning
- if err.resp.status == http.HTTPStatus.FORBIDDEN:
- print(
- f"Warning: error reading networks for {project_id}. " +
- f"This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project"
- )
- else:
- print(f"Warning: error reading networks for {project_id}: {err}")
- return 0
-
- network_id = 0
-
- if 'items' in response:
- for network in response['items']:
- if network['name'] == network_name:
- network_id = network['id']
- break
-
- if network_id == 0:
- print(f"Error: network_id not found for {network_name} in {project_id}")
-
- return network_id
-
-
-def get_quota_current_limit(project_link, metric_name):
- '''
- Retrieves limit for a specific metric.
-
- Parameters:
- project_link (string): Project link.
- metric_name (string): Name of the metric.
- Returns:
- results_list (list of string): Current limit.
- '''
- client, interval = create_client()
-
- try:
- results = client.list_time_series(
- request={
- "name": project_link,
- "filter": f'metric.type = "{metric_name}"',
- "interval": interval,
- "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL
- })
- results_list = list(results)
- return results_list
- except exceptions.PermissionDenied as err:
- print(
- f"Warning: error reading quotas for {project_link}. " +
- f"This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project"
- )
- return None
-
-
-def customize_quota_view(quota_results):
- '''
- Customize the quota output for an easier parsable output.
-
- Parameters:
- quota_results (string): Input from get_quota_current_usage or get_quota_current_limit. Contains the Current usage or limit for all networks in that project.
- Returns:
- quotaViewList (list of dictionaries of string: string): Current quota usage or limit.
- '''
- quotaViewList = []
- for result in quota_results:
- quotaViewJson = {}
- quotaViewJson.update(dict(result.resource.labels))
- quotaViewJson.update(dict(result.metric.labels))
- for val in result.points:
- quotaViewJson.update({'value': val.value.int64_value})
- quotaViewList.append(quotaViewJson)
- return quotaViewList
-
-
-def set_limits(network_dict, quota_limit, limit_dict):
- '''
- Updates the network dictionary with quota limit values.
-
- Parameters:
- network_dict (dictionary of string: string): Contains network information.
- quota_limit (list of dictionaries of string: string): Current quota limit.
- limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
- Returns:
- None
- '''
-
- network_dict['limit'] = None
-
- if quota_limit:
- for net in quota_limit:
- if net['network_id'] == network_dict['network_id']:
- network_dict['limit'] = net['value']
- return
-
- network_link = f"https://www.googleapis.com/compute/v1/projects/{network_dict['project_id']}/global/networks/{network_dict['network_name']}"
-
- if network_link in limit_dict:
- network_dict['limit'] = limit_dict[network_link]
- else:
- if 'default_value' in limit_dict:
- network_dict['limit'] = limit_dict['default_value']
- else:
- print(f"Error: Couldn't find limit for {network_link}")
- network_dict['limit'] = 0
-
-
-def get_limit_network(network_dict, network_link, quota_limit, limit_dict):
- '''
- Returns limit for a specific network and metric, using the GCP quota metrics or the values in the yaml file if not found.
-
- Parameters:
- network_dict (dictionary of string: string): Contains network information.
- network_link (string): Contains network link
- quota_limit (list of dictionaries of string: string): Current quota limit for all networks in that project.
- limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
- Returns:
- limit (int): Current limit for that network.
- '''
- if quota_limit:
- for net in quota_limit:
- if net['network_id'] == network_dict['network_id']:
- return net['value']
-
- if network_link in limit_dict:
- return limit_dict[network_link]
- else:
- if 'default_value' in limit_dict:
- return limit_dict['default_value']
- else:
- print(f"Error: Couldn't find limit for {network_link}")
-
- return 0
-
-
-def write_data_to_metric(monitored_project_id, value, metric_name,
- network_name):
- '''
- Writes data to Cloud Monitoring custom metrics.
-
- Parameters:
- monitored_project_id: ID of the project where the resource lives (will be added as a label)
- value (int): Value for the data point of the metric.
- metric_name (string): Name of the metric
- network_name (string): Name of the network (will be added as a label)
- Returns:
- usage (int): Current usage for that network.
- limit (int): Current usage for that network.
- '''
- client = monitoring_v3.MetricServiceClient()
-
- series = monitoring_v3.TimeSeries()
- series.metric.type = f"custom.googleapis.com/{metric_name}"
- series.resource.type = "global"
- series.metric.labels["network_name"] = network_name
- series.metric.labels["project"] = monitored_project_id
-
- now = time.time()
- seconds = int(now)
- nanos = int((now - seconds) * 10**9)
- interval = monitoring_v3.TimeInterval(
- {"end_time": {
- "seconds": seconds,
- "nanos": nanos
- }})
- point = monitoring_v3.Point({
- "interval": interval,
- "value": {
- "double_value": value
- }
- })
- series.points = [point]
-
- # TODO: sometimes this cashes with 'DeadlineExceeded: 504 Deadline expired before operation could complete' error
- # Implement exponential backoff retries?
- try:
- client.create_time_series(name=MONITORING_PROJECT_LINK,
- time_series=[series])
- except Exception as e:
- print(e)
+if __name__ == "__main__":
+ main(None, None)
diff --git a/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml b/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml
index c8f045cdc..2e5621d7f 100644
--- a/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml
+++ b/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml
@@ -60,7 +60,7 @@ metrics_per_network:
name: internal_forwarding_rules_l4_limit
description: Number of Internal Forwarding Rules for Internal L4 Load Balancers - limit.
values:
- default_value: 75
+ default_value: 300
utilization:
name: internal_forwarding_rules_l4_utilization
description: Number of Internal Forwarding Rules for Internal L4 Load Balancers - utilization.
@@ -97,7 +97,7 @@ metrics_per_peering_group:
name: internal_forwarding_rules_l4_ppg_limit
description: Number of Internal Forwarding Rules for Internal L4 Load Balancers per VPC peering group - effective limit.
values:
- default_value: 175
+ default_value: 300
utilization:
name: internal_forwarding_rules_l4_ppg_utilization
description: Number of Internal Forwarding Rules for Internal L4 Load Balancers per VPC peering group - utilization.
@@ -148,4 +148,4 @@ metrics_per_peering_group:
default_value: 300
utilization:
name: dynamic_routes_per_peering_group_utilization
- description: Number of Dynamic routes per peering group - utilization.
\ No newline at end of file
+ description: Number of Dynamic routes per peering group - utilization.
diff --git a/examples/cloud-operations/network-dashboard/cloud-function/metrics/ilb_fwrules.py b/examples/cloud-operations/network-dashboard/cloud-function/metrics/ilb_fwrules.py
new file mode 100644
index 000000000..55490544f
--- /dev/null
+++ b/examples/cloud-operations/network-dashboard/cloud-function/metrics/ilb_fwrules.py
@@ -0,0 +1,114 @@
+#
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from collections import defaultdict
+from google.protobuf import field_mask_pb2
+from . import metrics, networks, limits
+
+
+def get_forwarding_rules_dict(config, layer: str):
+ '''
+ Calls the Asset Inventory API to get all L4 Forwarding Rules under the GCP organization.
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+ layer (string): the Layer to get Forwarding rules (L4/L7)
+ Returns:
+ forwarding_rules_dict (dictionary of string: int): Keys are the network links and values are the number of Forwarding Rules per network.
+ '''
+
+ read_mask = field_mask_pb2.FieldMask()
+ read_mask.FromJsonString('name,versionedResources')
+
+ forwarding_rules_dict = defaultdict(int)
+
+ # TODO: Paging?
+ response = config["clients"]["asset_client"].search_all_resources(
+ request={
+ "scope": f"organizations/{config['organization']}",
+ "asset_types": ["compute.googleapis.com/ForwardingRule"],
+ "read_mask": read_mask,
+ })
+
+ for resource in response:
+ internal = False
+ network_link = ""
+ for versioned in resource.versioned_resources:
+ for field_name, field_value in versioned.resource.items():
+ if field_name == "loadBalancingScheme":
+ internal = (field_value == config["lb_scheme"][layer])
+ if field_name == "network":
+ network_link = field_value
+ if internal:
+ if network_link in forwarding_rules_dict:
+ forwarding_rules_dict[network_link] += 1
+ else:
+ forwarding_rules_dict[network_link] = 1
+
+ return forwarding_rules_dict
+
+
+def get_forwarding_rules_data(config, metrics_dict, forwarding_rules_dict,
+ limit_dict, layer):
+ '''
+ Gets the data for L4 Internal Forwarding Rules per VPC Network and writes it to the metric defined in forwarding_rules_metric.
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+ metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions.
+ forwarding_rules_dict (dictionary of string: int): Keys are the network links and values are the number of Forwarding Rules per network.
+ limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value.
+ layer (string): the Layer to get Forwarding rules (L4/L7)
+ Returns:
+ None
+ '''
+ for project in config["monitored_projects"]:
+ network_dict = networks.get_networks(config, project)
+
+ current_quota_limit = limits.get_quota_current_limit(
+ config, f"projects/{project}", config["limit_names"][layer])
+
+ if current_quota_limit is None:
+ print(
+ f"Could not write {layer} forwarding rules to metric for projects/{project} due to missing quotas"
+ )
+ continue
+
+ current_quota_limit_view = metrics.customize_quota_view(current_quota_limit)
+
+ for net in network_dict:
+ limits.set_limits(net, current_quota_limit_view, limit_dict)
+
+ usage = 0
+ if net['self_link'] in forwarding_rules_dict:
+ usage = forwarding_rules_dict[net['self_link']]
+ metrics.write_data_to_metric(
+ config, project, usage, metrics_dict["metrics_per_network"]
+ [f"{layer.lower()}_forwarding_rules_per_network"]["usage"]["name"],
+ net['network_name'])
+ metrics.write_data_to_metric(
+ config, project, net['limit'], metrics_dict["metrics_per_network"]
+ [f"{layer.lower()}_forwarding_rules_per_network"]["limit"]["name"],
+ net['network_name'])
+ metrics.write_data_to_metric(
+ config, project, usage / net['limit'],
+ metrics_dict["metrics_per_network"]
+ [f"{layer.lower()}_forwarding_rules_per_network"]["utilization"]
+ ["name"], net['network_name'])
+
+ print(
+ f"Wrote number of {layer} forwarding rules to metric for projects/{project}"
+ )
diff --git a/examples/cloud-operations/network-dashboard/cloud-function/metrics/instances.py b/examples/cloud-operations/network-dashboard/cloud-function/metrics/instances.py
new file mode 100644
index 000000000..ef78e67d0
--- /dev/null
+++ b/examples/cloud-operations/network-dashboard/cloud-function/metrics/instances.py
@@ -0,0 +1,95 @@
+#
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from code import interact
+from collections import defaultdict
+from . import metrics, networks, limits
+
+
+def get_gce_instance_dict(config: dict):
+ '''
+ Calls the Asset Inventory API to get all GCE instances under the GCP organization.
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+
+ Returns:
+ gce_instance_dict (dictionary of string: int): Keys are the network links and values are the number of GCE Instances per network.
+ '''
+
+ gce_instance_dict = defaultdict(int)
+
+ response = config["clients"]["asset_client"].search_all_resources(
+ request={
+ "scope": f"organizations/{config['organization']}",
+ "asset_types": ["compute.googleapis.com/Instance"],
+ })
+ for resource in response:
+ for field_name, field_value in resource.additional_attributes.items():
+ if field_name == "networkInterfaceNetworks":
+ for network in field_value:
+ if network in gce_instance_dict:
+ gce_instance_dict[network] += 1
+ else:
+ gce_instance_dict[network] = 1
+
+ return gce_instance_dict
+
+
+def get_gce_instances_data(config, metrics_dict, gce_instance_dict, limit_dict):
+ '''
+ Gets the data for GCE instances per VPC Network and writes it to the metric defined in instance_metric.
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+ metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions
+ gce_instance_dict (dictionary of string: int): Keys are the network links and values are the number of GCE Instances per network.
+ limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
+ Returns:
+ gce_instance_dict
+ '''
+
+ for project in config["monitored_projects"]:
+ network_dict = networks.get_networks(config, project)
+
+ current_quota_limit = limits.get_quota_current_limit(
+ config, f"projects/{project}", config["limit_names"]["GCE_INSTANCES"])
+ if current_quota_limit is None:
+ print(
+ f"Could not write number of instances for projects/{project} due to missing quotas"
+ )
+
+ current_quota_limit_view = metrics.customize_quota_view(current_quota_limit)
+
+ for net in network_dict:
+ limits.set_limits(net, current_quota_limit_view, limit_dict)
+
+ usage = 0
+ if net['self_link'] in gce_instance_dict:
+ usage = gce_instance_dict[net['self_link']]
+
+ metrics.write_data_to_metric(
+ config, project, usage, metrics_dict["metrics_per_network"]
+ ["instance_per_network"]["usage"]["name"], net['network_name'])
+ metrics.write_data_to_metric(
+ config, project, net['limit'], metrics_dict["metrics_per_network"]
+ ["instance_per_network"]["limit"]["name"], net['network_name'])
+ metrics.write_data_to_metric(
+ config, project, usage / net['limit'],
+ metrics_dict["metrics_per_network"]["instance_per_network"]
+ ["utilization"]["name"], net['network_name'])
+
+ print(f"Wrote number of instances to metric for projects/{project}")
diff --git a/examples/cloud-operations/network-dashboard/cloud-function/metrics/limits.py b/examples/cloud-operations/network-dashboard/cloud-function/metrics/limits.py
new file mode 100644
index 000000000..8b3d5ae97
--- /dev/null
+++ b/examples/cloud-operations/network-dashboard/cloud-function/metrics/limits.py
@@ -0,0 +1,171 @@
+#
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from google.api_core import exceptions
+from google.cloud import monitoring_v3
+from . import metrics
+
+
+def get_ppg(network_link, limit_dict):
+ '''
+ Checks if this network has a specific limit for a metric, if so, returns that limit, if not, returns the default limit.
+
+ Parameters:
+ network_link (string): VPC network link.
+ limit_list (list of string): Used to get the limit per VPC or the default limit.
+ Returns:
+ limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
+ '''
+ if network_link in limit_dict:
+ return limit_dict[network_link]
+ else:
+ if 'default_value' in limit_dict:
+ return limit_dict['default_value']
+ else:
+ print(f"Error: limit not found for {network_link}")
+ return 0
+
+
+def set_limits(network_dict, quota_limit, limit_dict):
+ '''
+ Updates the network dictionary with quota limit values.
+
+ Parameters:
+ network_dict (dictionary of string: string): Contains network information.
+ quota_limit (list of dictionaries of string: string): Current quota limit.
+ limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
+ Returns:
+ None
+ '''
+
+ network_dict['limit'] = None
+
+ if quota_limit:
+ for net in quota_limit:
+ if net['network_id'] == network_dict['network_id']:
+ network_dict['limit'] = net['value']
+ return
+
+ network_link = f"https://www.googleapis.com/compute/v1/projects/{network_dict['project_id']}/global/networks/{network_dict['network_name']}"
+
+ if network_link in limit_dict:
+ network_dict['limit'] = limit_dict[network_link]
+ else:
+ if 'default_value' in limit_dict:
+ network_dict['limit'] = limit_dict['default_value']
+ else:
+ print(f"Error: Couldn't find limit for {network_link}")
+ network_dict['limit'] = 0
+
+
+def get_quota_current_limit(config, project_link, metric_name):
+ '''
+ Retrieves limit for a specific metric.
+
+ Parameters:
+ project_link (string): Project link.
+ metric_name (string): Name of the metric.
+ Returns:
+ results_list (list of string): Current limit.
+ '''
+
+ try:
+ results = config["clients"]["monitoring_client"].list_time_series(
+ request={
+ "name": project_link,
+ "filter": f'metric.type = "{metric_name}"',
+ "interval": config["monitoring_interval"],
+ "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL
+ })
+ results_list = list(results)
+ return results_list
+ except exceptions.PermissionDenied as err:
+ print(
+ f"Warning: error reading quotas for {project_link}. " +
+ f"This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project"
+ )
+ return None
+
+
+def count_effective_limit(config, project_id, network_dict, usage_metric_name,
+ limit_metric_name, utilization_metric_name,
+ limit_dict):
+ '''
+ Calculates the effective limits (using algorithm in the link below) for peering groups and writes data (usage, limit, utilization) to the custom metrics.
+ Source: https://cloud.google.com/vpc/docs/quota#vpc-peering-effective-limit
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+ project_id (string): Project ID for the project to be analyzed.
+ network_dict (dictionary of string: string): Contains all required information about the network to get the usage, limit and utilization.
+ usage_metric_name (string): Name of the custom metric to be populated for usage per VPC peering group.
+ limit_metric_name (string): Name of the custom metric to be populated for limit per VPC peering group.
+ utilization_metric_name (string): Name of the custom metric to be populated for utilization per VPC peering group.
+ limit_dict (dictionary of string:int): Dictionary containing the limit per peering group (either VPC specific or default limit).
+ Returns:
+ None
+ '''
+
+ if network_dict['peerings'] == []:
+ return
+
+ # Get usage: Sums usage for current network + all peered networks
+ peering_group_usage = network_dict['usage']
+ for peered_network in network_dict['peerings']:
+ if 'usage' not in peered_network:
+ print(
+ f"Can not add metrics for peered network in projects/{project_id} as no usage metrics exist due to missing permissions"
+ )
+ continue
+ peering_group_usage += peered_network['usage']
+
+ network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network_dict['network_name']}"
+
+ # Calculates effective limit: Step 1: max(per network limit, per network_peering_group limit)
+ limit_step1 = max(network_dict['limit'], get_ppg(network_link, limit_dict))
+
+ # Calculates effective limit: Step 2: List of max(per network limit, per network_peering_group limit) for each peered network
+ limit_step2 = []
+ for peered_network in network_dict['peerings']:
+ peered_network_link = f"https://www.googleapis.com/compute/v1/projects/{peered_network['project_id']}/global/networks/{peered_network['network_name']}"
+
+ if 'limit' in peered_network:
+ limit_step2.append(
+ max(peered_network['limit'], get_ppg(peered_network_link,
+ limit_dict)))
+ else:
+ print(
+ f"Ignoring projects/{peered_network['project_id']} for limits in peering group of project {project_id} as no limits are available."
+ +
+ "This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project"
+ )
+
+ # Calculates effective limit: Step 3: Find minimum from the list created by Step 2
+ limit_step3 = 0
+ if len(limit_step2) > 0:
+ limit_step3 = min(limit_step2)
+
+ # Calculates effective limit: Step 4: Find maximum from step 1 and step 3
+ effective_limit = max(limit_step1, limit_step3)
+ utilization = peering_group_usage / effective_limit
+
+ metrics.write_data_to_metric(config, project_id, peering_group_usage,
+ usage_metric_name, network_dict['network_name'])
+ metrics.write_data_to_metric(config, project_id, effective_limit,
+ limit_metric_name, network_dict['network_name'])
+ metrics.write_data_to_metric(config, project_id, utilization,
+ utilization_metric_name,
+ network_dict['network_name'])
diff --git a/examples/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py b/examples/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py
new file mode 100644
index 000000000..14183f157
--- /dev/null
+++ b/examples/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py
@@ -0,0 +1,238 @@
+#
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import yaml
+from google.api import metric_pb2 as ga_metric
+from google.cloud import monitoring_v3
+from . import peerings, limits, networks
+
+
+def create_metrics(monitoring_project):
+ '''
+ Creates all Cloud Monitoring custom metrics based on the metric.yaml file
+
+ Parameters:
+ monitoring_project (string): the project where the metrics are written to
+ Returns:
+ metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions
+ limits_dict (dictionary of dictionary of string: int): limits_dict[metric_name]: dict[network_name] = limit_value
+ '''
+ client = monitoring_v3.MetricServiceClient()
+ existing_metrics = []
+ for desc in client.list_metric_descriptors(name=monitoring_project):
+ existing_metrics.append(desc.type)
+ limits_dict = {}
+
+ with open("metrics.yaml", 'r') as stream:
+ try:
+ metrics_dict = yaml.safe_load(stream)
+
+ for metric_list in metrics_dict.values():
+ for metric in metric_list.values():
+ for sub_metric_key, sub_metric in metric.items():
+ metric_link = f"custom.googleapis.com/{sub_metric['name']}"
+ # If the metric doesn't exist yet, then we create it
+ if metric_link not in existing_metrics:
+ create_metric(sub_metric["name"], sub_metric["description"],
+ monitoring_project)
+ # Parse limits (both default values and network specific ones)
+ if sub_metric_key == "limit":
+ limits_dict_for_metric = {}
+ for network_link, limit_value in sub_metric["values"].items():
+ limits_dict_for_metric[network_link] = limit_value
+ limits_dict[sub_metric["name"]] = limits_dict_for_metric
+
+ return metrics_dict, limits_dict
+ except yaml.YAMLError as exc:
+ print(exc)
+
+
+def create_metric(metric_name, description, monitoring_project):
+ '''
+ Creates a Cloud Monitoring metric based on the parameter given if the metric is not already existing
+
+ Parameters:
+ metric_name (string): Name of the metric to be created
+ description (string): Description of the metric to be created
+ monitoring_project (string): the project where the metrics are written to
+ Returns:
+ None
+ '''
+ client = monitoring_v3.MetricServiceClient()
+
+ descriptor = ga_metric.MetricDescriptor()
+ descriptor.type = f"custom.googleapis.com/{metric_name}"
+ descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE
+ descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE
+ descriptor.description = description
+ descriptor = client.create_metric_descriptor(name=monitoring_project,
+ metric_descriptor=descriptor)
+ print("Created {}.".format(descriptor.name))
+
+
+def write_data_to_metric(config, monitored_project_id, value, metric_name,
+ network_name):
+ '''
+ Writes data to Cloud Monitoring custom metrics.
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+ monitored_project_id: ID of the project where the resource lives (will be added as a label)
+ value (int): Value for the data point of the metric.
+ metric_name (string): Name of the metric
+ network_name (string): Name of the network (will be added as a label)
+ Returns:
+ usage (int): Current usage for that network.
+ limit (int): Current usage for that network.
+ '''
+ client = monitoring_v3.MetricServiceClient()
+
+ series = monitoring_v3.TimeSeries()
+ series.metric.type = f"custom.googleapis.com/{metric_name}"
+ series.resource.type = "global"
+ series.metric.labels["network_name"] = network_name
+ series.metric.labels["project"] = monitored_project_id
+
+ now = time.time()
+ seconds = int(now)
+ nanos = int((now - seconds) * 10**9)
+ interval = monitoring_v3.TimeInterval(
+ {"end_time": {
+ "seconds": seconds,
+ "nanos": nanos
+ }})
+ point = monitoring_v3.Point({
+ "interval": interval,
+ "value": {
+ "double_value": value
+ }
+ })
+ series.points = [point]
+
+ # TODO: sometimes this cashes with 'DeadlineExceeded: 504 Deadline expired before operation could complete' error
+ # Implement exponential backoff retries?
+ try:
+ client.create_time_series(name=config["monitoring_project_link"],
+ time_series=[series])
+ except Exception as e:
+ print(e)
+
+
+def get_pgg_data(config, metric_dict, usage_dict, limit_metric, limit_dict):
+ '''
+ This function gets the usage, limit and utilization per VPC peering group for a specific metric for all projects to be monitored.
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+ metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics
+ usage_dict (dictionnary of string:int): Dictionary with the network link as key and the number of resources as value
+ limit_metric (string): Name of the existing GCP metric for limit per VPC network
+ limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
+ Returns:
+ None
+ '''
+ for project in config["monitored_projects"]:
+ network_dict_list = peerings.gather_peering_data(config, project)
+ # Network dict list is a list of dictionary (one for each network)
+ # For each network, this dictionary contains:
+ # project_id, network_name, network_id, usage, limit, peerings (list of peered networks)
+ # peerings is a list of dictionary (one for each peered network) and contains:
+ # project_id, network_name, network_id
+ current_quota_limit = limits.get_quota_current_limit(
+ config, f"projects/{project}", limit_metric)
+ if current_quota_limit is None:
+ print(
+ f"Could not write number of L7 forwarding rules to metric for projects/{project} due to missing quotas"
+ )
+ continue
+
+ current_quota_limit_view = customize_quota_view(current_quota_limit)
+
+ # For each network in this GCP project
+ for network_dict in network_dict_list:
+ if network_dict['network_id'] == 0:
+ print(
+ f"Could not write {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project} due to missing permissions."
+ )
+ continue
+ network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{network_dict['network_name']}"
+
+ limit = networks.get_limit_network(network_dict, network_link,
+ current_quota_limit_view, limit_dict)
+
+ usage = 0
+ if network_link in usage_dict:
+ usage = usage_dict[network_link]
+
+ # Here we add usage and limit to the network dictionary
+ network_dict["usage"] = usage
+ network_dict["limit"] = limit
+
+ # For every peered network, get usage and limits
+ for peered_network_dict in network_dict['peerings']:
+ peered_network_link = f"https://www.googleapis.com/compute/v1/projects/{peered_network_dict['project_id']}/global/networks/{peered_network_dict['network_name']}"
+ peered_usage = 0
+ if peered_network_link in usage_dict:
+ peered_usage = usage_dict[peered_network_link]
+
+ current_peered_quota_limit = limits.get_quota_current_limit(
+ config, f"projects/{peered_network_dict['project_id']}",
+ limit_metric)
+ if current_peered_quota_limit is None:
+ print(
+ f"Could not write metrics for peering to projects/{peered_network_dict['project_id']} due to missing quotas"
+ )
+ continue
+
+ peering_project_limit = customize_quota_view(current_peered_quota_limit)
+
+ peered_limit = networks.get_limit_network(peered_network_dict,
+ peered_network_link,
+ peering_project_limit,
+ limit_dict)
+ # Here we add usage and limit to the peered network dictionary
+ peered_network_dict["usage"] = peered_usage
+ peered_network_dict["limit"] = peered_limit
+
+ limits.count_effective_limit(config, project, network_dict,
+ metric_dict["usage"]["name"],
+ metric_dict["limit"]["name"],
+ metric_dict["utilization"]["name"],
+ limit_dict)
+ print(
+ f"Wrote {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project}"
+ )
+
+
+def customize_quota_view(quota_results):
+ '''
+ Customize the quota output for an easier parsable output.
+
+ Parameters:
+ quota_results (string): Input from get_quota_current_usage or get_quota_current_limit. Contains the Current usage or limit for all networks in that project.
+ Returns:
+ quotaViewList (list of dictionaries of string: string): Current quota usage or limit.
+ '''
+ quotaViewList = []
+ for result in quota_results:
+ quotaViewJson = {}
+ quotaViewJson.update(dict(result.resource.labels))
+ quotaViewJson.update(dict(result.metric.labels))
+ for val in result.points:
+ quotaViewJson.update({'value': val.value.int64_value})
+ quotaViewList.append(quotaViewJson)
+ return quotaViewList
diff --git a/examples/cloud-operations/network-dashboard/cloud-function/metrics/networks.py b/examples/cloud-operations/network-dashboard/cloud-function/metrics/networks.py
new file mode 100644
index 000000000..6caeff173
--- /dev/null
+++ b/examples/cloud-operations/network-dashboard/cloud-function/metrics/networks.py
@@ -0,0 +1,159 @@
+#
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from code import interact
+from collections import defaultdict
+from google.protobuf import field_mask_pb2
+from googleapiclient import errors
+import http
+
+
+def get_subnet_ranges_dict(config: dict):
+ '''
+ Calls the Asset Inventory API to get all Subnet ranges under the GCP organization.
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+ Returns:
+ subnet_range_dict (dictionary of string: int): Keys are the network links and values are the number of subnet ranges per network.
+ '''
+
+ subnet_range_dict = defaultdict(int)
+ read_mask = field_mask_pb2.FieldMask()
+ read_mask.FromJsonString('name,versionedResources')
+
+ response = config["clients"]["asset_client"].search_all_resources(
+ request={
+ "scope": f"organizations/{config['organization']}",
+ "asset_types": ["compute.googleapis.com/Subnetwork"],
+ "read_mask": read_mask,
+ })
+ for resource in response:
+ ranges = 0
+ network_link = None
+
+ for versioned in resource.versioned_resources:
+ for field_name, field_value in versioned.resource.items():
+ if field_name == "network":
+ network_link = field_value
+ ranges += 1
+ if field_name == "secondaryIpRanges":
+ for range in field_value:
+ ranges += 1
+
+ if network_link in subnet_range_dict:
+ subnet_range_dict[network_link] += ranges
+ else:
+ subnet_range_dict[network_link] = ranges
+
+ return subnet_range_dict
+
+
+def get_networks(config, project_id):
+ '''
+ Returns a dictionary of all networks in a project.
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+ project_id (string): Project ID for the project containing the networks.
+ Returns:
+ network_dict (dictionary of string: string): Contains the project_id, network_name(s) and network_id(s)
+ '''
+ request = config["clients"]["discovery_client"].networks().list(
+ project=project_id)
+ response = request.execute()
+ network_dict = []
+ if 'items' in response:
+ for network in response['items']:
+ network_name = network['name']
+ network_id = network['id']
+ self_link = network['selfLink']
+ d = {
+ 'project_id': project_id,
+ 'network_name': network_name,
+ 'network_id': network_id,
+ 'self_link': self_link
+ }
+ network_dict.append(d)
+ return network_dict
+
+
+def get_network_id(config, project_id, network_name):
+ '''
+ Returns the network_id for a specific project / network name.
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+ project_id (string): Project ID for the project containing the networks.
+ network_name (string): Name of the network
+ Returns:
+ network_id (int): Network ID.
+ '''
+ request = config["clients"]["discovery_client"].networks().list(
+ project=project_id)
+ try:
+ response = request.execute()
+ except errors.HttpError as err:
+ # TODO: log proper warning
+ if err.resp.status == http.HTTPStatus.FORBIDDEN:
+ print(
+ f"Warning: error reading networks for {project_id}. " +
+ f"This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project"
+ )
+ else:
+ print(f"Warning: error reading networks for {project_id}: {err}")
+ return 0
+
+ network_id = 0
+
+ if 'items' in response:
+ for network in response['items']:
+ if network['name'] == network_name:
+ network_id = network['id']
+ break
+
+ if network_id == 0:
+ print(f"Error: network_id not found for {network_name} in {project_id}")
+
+ return network_id
+
+
+def get_limit_network(network_dict, network_link, quota_limit, limit_dict):
+ '''
+ Returns limit for a specific network and metric, using the GCP quota metrics or the values in the yaml file if not found.
+
+ Parameters:
+ network_dict (dictionary of string: string): Contains network information.
+ network_link (string): Contains network link
+ quota_limit (list of dictionaries of string: string): Current quota limit for all networks in that project.
+ limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
+ Returns:
+ limit (int): Current limit for that network.
+ '''
+ if quota_limit:
+ for net in quota_limit:
+ if net['network_id'] == network_dict['network_id']:
+ return net['value']
+
+ if network_link in limit_dict:
+ return limit_dict[network_link]
+ else:
+ if 'default_value' in limit_dict:
+ return limit_dict['default_value']
+ else:
+ print(f"Error: Couldn't find limit for {network_link}")
+
+ return 0
diff --git a/examples/cloud-operations/network-dashboard/cloud-function/metrics/peerings.py b/examples/cloud-operations/network-dashboard/cloud-function/metrics/peerings.py
new file mode 100644
index 000000000..141364d1f
--- /dev/null
+++ b/examples/cloud-operations/network-dashboard/cloud-function/metrics/peerings.py
@@ -0,0 +1,164 @@
+#
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from . import metrics, networks, limits
+
+
+def get_vpc_peering_data(config, metrics_dict, limit_dict):
+ '''
+ Gets the data for VPC peerings (active or not) and writes it to the metric defined (vpc_peering_active_metric and vpc_peering_metric).
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+ metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions
+ limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
+ Returns:
+ None
+ '''
+ for project in config["monitored_projects"]:
+ active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(
+ config, project, limit_dict)
+ for peering in active_vpc_peerings:
+ metrics.write_data_to_metric(
+ config, project, peering['active_peerings'],
+ metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]
+ ["usage"]["name"], peering['network_name'])
+ metrics.write_data_to_metric(
+ config, project, peering['network_limit'],
+ metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]
+ ["limit"]["name"], peering['network_name'])
+ metrics.write_data_to_metric(
+ config, project,
+ peering['active_peerings'] / peering['network_limit'],
+ metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]
+ ["utilization"]["name"], peering['network_name'])
+ print("Wrote number of active VPC peerings to custom metric for project:",
+ project)
+
+ for peering in vpc_peerings:
+ metrics.write_data_to_metric(
+ config, project, peering['peerings'],
+ metrics_dict["metrics_per_network"]["vpc_peering_per_network"]
+ ["usage"]["name"], peering['network_name'])
+ metrics.write_data_to_metric(
+ config, project, peering['network_limit'],
+ metrics_dict["metrics_per_network"]["vpc_peering_per_network"]
+ ["limit"]["name"], peering['network_name'])
+ metrics.write_data_to_metric(
+ config, project, peering['peerings'] / peering['network_limit'],
+ metrics_dict["metrics_per_network"]["vpc_peering_per_network"]
+ ["utilization"]["name"], peering['network_name'])
+ print("Wrote number of VPC peerings to custom metric for project:", project)
+
+
+def gather_peering_data(config, project_id):
+ '''
+ Returns a dictionary of all peerings for all networks in a project.
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+ project_id (string): Project ID for the project containing the networks.
+ Returns:
+ network_list (dictionary of string: string): Contains the project_id, network_name(s) and network_id(s) of peered networks.
+ '''
+ request = config["clients"]["discovery_client"].networks().list(
+ project=project_id)
+ response = request.execute()
+
+ network_list = []
+ if 'items' in response:
+ for network in response['items']:
+ net = {
+ 'project_id': project_id,
+ 'network_name': network['name'],
+ 'network_id': network['id'],
+ 'peerings': []
+ }
+ if 'peerings' in network:
+ STATE = network['peerings'][0]['state']
+ if STATE == "ACTIVE":
+ for peered_network in network[
+ 'peerings']: # "projects/{project_name}/global/networks/{network_name}"
+ start = peered_network['network'].find("projects/") + len(
+ 'projects/')
+ end = peered_network['network'].find("/global")
+ peered_project = peered_network['network'][start:end]
+ peered_network_name = peered_network['network'].split(
+ "networks/")[1]
+ peered_net = {
+ 'project_id':
+ peered_project,
+ 'network_name':
+ peered_network_name,
+ 'network_id':
+ networks.get_network_id(config, peered_project,
+ peered_network_name)
+ }
+ net["peerings"].append(peered_net)
+ network_list.append(net)
+ return network_list
+
+
+def gather_vpc_peerings_data(config, project_id, limit_dict):
+ '''
+ Gets the data for all VPC peerings (active or not) in project_id and writes it to the metric defined in vpc_peering_active_metric and vpc_peering_metric.
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+ project_id (string): We will take all VPCs in that project_id and look for all peerings to these VPCs.
+ limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
+ Returns:
+ active_peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each active VPC peering.
+ peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each VPC peering.
+ '''
+ active_peerings_dict = []
+ peerings_dict = []
+ request = config["clients"]["discovery_client"].networks().list(
+ project=project_id)
+ response = request.execute()
+ if 'items' in response:
+ for network in response['items']:
+ if 'peerings' in network:
+ STATE = network['peerings'][0]['state']
+ if STATE == "ACTIVE":
+ active_peerings_count = len(network['peerings'])
+ else:
+ active_peerings_count = 0
+
+ peerings_count = len(network['peerings'])
+ else:
+ peerings_count = 0
+ active_peerings_count = 0
+
+ network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network['name']}"
+ network_limit = limits.get_ppg(network_link, limit_dict)
+
+ active_d = {
+ 'project_id': project_id,
+ 'network_name': network['name'],
+ 'active_peerings': active_peerings_count,
+ 'network_limit': network_limit
+ }
+ active_peerings_dict.append(active_d)
+ d = {
+ 'project_id': project_id,
+ 'network_name': network['name'],
+ 'peerings': peerings_count,
+ 'network_limit': network_limit
+ }
+ peerings_dict.append(d)
+
+ return active_peerings_dict, peerings_dict
diff --git a/examples/cloud-operations/network-dashboard/cloud-function/metrics/routers.py b/examples/cloud-operations/network-dashboard/cloud-function/metrics/routers.py
new file mode 100644
index 000000000..d20a76c4b
--- /dev/null
+++ b/examples/cloud-operations/network-dashboard/cloud-function/metrics/routers.py
@@ -0,0 +1,56 @@
+#
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from google.protobuf import field_mask_pb2
+
+
+def get_routers(config):
+ '''
+ Returns a dictionary of all Cloud Routers in the GCP organization.
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+ Returns:
+ routers_dict (dictionary of string: list of string): Key is the network link and value is a list of router links.
+ '''
+
+ read_mask = field_mask_pb2.FieldMask()
+ read_mask.FromJsonString('name,versionedResources')
+
+ routers_dict = {}
+
+ response = config["clients"]["asset_client"].search_all_resources(
+ request={
+ "scope": f"organizations/{config['organization']}",
+ "asset_types": ["compute.googleapis.com/Router"],
+ "read_mask": read_mask,
+ })
+ for resource in response:
+ network_link = None
+ router_link = None
+ for versioned in resource.versioned_resources:
+ for field_name, field_value in versioned.resource.items():
+ if field_name == "network":
+ network_link = field_value
+ if field_name == "selfLink":
+ router_link = field_value
+
+ if network_link in routers_dict:
+ routers_dict[network_link].append(router_link)
+ else:
+ routers_dict[network_link] = [router_link]
+
+ return routers_dict
diff --git a/examples/cloud-operations/network-dashboard/cloud-function/metrics/routes.py b/examples/cloud-operations/network-dashboard/cloud-function/metrics/routes.py
new file mode 100644
index 000000000..fbe9ff3be
--- /dev/null
+++ b/examples/cloud-operations/network-dashboard/cloud-function/metrics/routes.py
@@ -0,0 +1,176 @@
+#
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from collections import defaultdict
+from . import metrics, networks, limits, peerings, routers
+
+
+def get_routes_for_router(config, project_id, router_region, router_name):
+ '''
+ Returns the same of dynamic routes learned by a specific Cloud Router instance
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+ project_id (string): Project ID for the project containing the Cloud Router.
+ router_region (string): GCP region for the Cloud Router.
+ router_name (string): Cloud Router name.
+ Returns:
+ sum_routes (int): Number of dynamic routes learned by the Cloud Router.
+ '''
+ request = config["clients"]["discovery_client"].routers().getRouterStatus(
+ project=project_id, region=router_region, router=router_name)
+ response = request.execute()
+
+ sum_routes = 0
+
+ if 'result' in response:
+ if 'bgpPeerStatus' in response['result']:
+ for peer in response['result']['bgpPeerStatus']:
+ sum_routes += peer['numLearnedRoutes']
+
+ return sum_routes
+
+
+def get_routes_for_network(config, network_link, project_id, routers_dict):
+ '''
+ Returns a the number of dynamic routes for a given network
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+ network_link (string): Network self link.
+ project_id (string): Project ID containing the network.
+ routers_dict (dictionary of string: list of string): Dictionary with key as network link and value as list of router links.
+ Returns:
+ sum_routes (int): Number of routes in that network.
+ '''
+ sum_routes = 0
+
+ if network_link in routers_dict:
+ for router_link in routers_dict[network_link]:
+ # Router link is using the following format:
+ # 'https://www.googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/routers/ROUTER_NAME'
+ start = router_link.find("/regions/") + len("/regions/")
+ end = router_link.find("/routers/")
+ router_region = router_link[start:end]
+ router_name = router_link.split('/routers/')[1]
+ routes = get_routes_for_router(config, project_id, router_region,
+ router_name)
+
+ sum_routes += routes
+
+ return sum_routes
+
+
+def get_dynamic_routes(config, metrics_dict, limits_dict):
+ '''
+ Writes all dynamic routes per VPC to custom metrics.
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+ metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions.
+ limits_dict (dictionary of string: int): key is network link (or 'default_value') and value is the limit for that network
+ Returns:
+ dynamic_routes_dict (dictionary of string: int): key is network link and value is the number of dynamic routes for that network
+ '''
+ routers_dict = routers.get_routers(config)
+ dynamic_routes_dict = defaultdict(int)
+
+ for project_id in config["monitored_projects"]:
+ network_dict = networks.get_networks(config, project_id)
+
+ for network in network_dict:
+ sum_routes = get_routes_for_network(config, network['self_link'],
+ project_id, routers_dict)
+ dynamic_routes_dict[network['self_link']] = sum_routes
+
+ if network['self_link'] in limits_dict:
+ limit = limits_dict[network['self_link']]
+ else:
+ if 'default_value' in limits_dict:
+ limit = limits_dict['default_value']
+ else:
+ print("Error: couldn't find limit for dynamic routes.")
+ break
+
+ utilization = sum_routes / limit
+
+ metrics.write_data_to_metric(
+ config, project_id, sum_routes, metrics_dict["metrics_per_network"]
+ ["dynamic_routes_per_network"]["usage"]["name"],
+ network['network_name'])
+ metrics.write_data_to_metric(
+ config, project_id, limit, metrics_dict["metrics_per_network"]
+ ["dynamic_routes_per_network"]["limit"]["name"],
+ network['network_name'])
+ metrics.write_data_to_metric(
+ config, project_id, utilization, metrics_dict["metrics_per_network"]
+ ["dynamic_routes_per_network"]["utilization"]["name"],
+ network['network_name'])
+
+ print("Wrote metrics for dynamic routes for VPCs in project", project_id)
+
+ return dynamic_routes_dict
+
+
+def get_dynamic_routes_ppg(config, metric_dict, usage_dict, limit_dict):
+ '''
+ This function gets the usage, limit and utilization for the dynamic routes per VPC peering group.
+
+ Parameters:
+ config (dict): The dict containing config like clients and limits
+ metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics
+ usage_dict (dictionnary of string:int): Dictionary with the network link as key and the number of resources as value
+ limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
+ Returns:
+ None
+ '''
+ for project in config["monitored_projects"]:
+ network_dict_list = peerings.gather_peering_data(config, project)
+
+ for network_dict in network_dict_list:
+ network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{network_dict['network_name']}"
+
+ limit = limits.get_ppg(network_link, limit_dict)
+
+ usage = 0
+ if network_link in usage_dict:
+ usage = usage_dict[network_link]
+
+ # Here we add usage and limit to the network dictionary
+ network_dict["usage"] = usage
+ network_dict["limit"] = limit
+
+ # For every peered network, get usage and limits
+ for peered_network_dict in network_dict['peerings']:
+ peered_network_link = f"https://www.googleapis.com/compute/v1/projects/{peered_network_dict['project_id']}/global/networks/{peered_network_dict['network_name']}"
+ peered_usage = 0
+ if peered_network_link in usage_dict:
+ peered_usage = usage_dict[peered_network_link]
+
+ peered_limit = limits.get_ppg(peered_network_link, limit_dict)
+
+ # Here we add usage and limit to the peered network dictionary
+ peered_network_dict["usage"] = peered_usage
+ peered_network_dict["limit"] = peered_limit
+
+ limits.count_effective_limit(config, project, network_dict,
+ metric_dict["usage"]["name"],
+ metric_dict["limit"]["name"],
+ metric_dict["utilization"]["name"],
+ limit_dict)
+ print(
+ f"Wrote {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project}"
+ )
diff --git a/examples/data-solutions/data-platform-foundations/01-landing.tf b/examples/data-solutions/data-platform-foundations/01-dropoff.tf
similarity index 70%
rename from examples/data-solutions/data-platform-foundations/01-landing.tf
rename to examples/data-solutions/data-platform-foundations/01-dropoff.tf
index 49f32aff9..9a07b5171 100644
--- a/examples/data-solutions/data-platform-foundations/01-landing.tf
+++ b/examples/data-solutions/data-platform-foundations/01-dropoff.tf
@@ -12,20 +12,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# tfdoc:file:description land project and resources.
+# tfdoc:file:description drop off project and resources.
locals {
- land_orch_service_accounts = [
+ drop_orch_service_accounts = [
module.load-sa-df-0.iam_email, module.orch-sa-cmp-0.iam_email
]
}
-module "land-project" {
+module "drop-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
- name = "lnd${local.project_suffix}"
+ name = "drp${local.project_suffix}"
group_iam = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor",
@@ -34,14 +34,14 @@ module "land-project" {
]
}
iam = {
- "roles/bigquery.dataEditor" = [module.land-sa-bq-0.iam_email]
+ "roles/bigquery.dataEditor" = [module.drop-sa-bq-0.iam_email]
"roles/bigquery.user" = [module.load-sa-df-0.iam_email]
- "roles/pubsub.publisher" = [module.land-sa-ps-0.iam_email]
+ "roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email]
"roles/pubsub.subscriber" = concat(
- local.land_orch_service_accounts, [module.load-sa-df-0.iam_email]
+ local.drop_orch_service_accounts, [module.load-sa-df-0.iam_email]
)
"roles/storage.objectAdmin" = [module.load-sa-df-0.iam_email]
- "roles/storage.objectCreator" = [module.land-sa-cs-0.iam_email]
+ "roles/storage.objectCreator" = [module.drop-sa-cs-0.iam_email]
"roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email]
"roles/storage.admin" = [module.load-sa-df-0.iam_email]
}
@@ -63,12 +63,12 @@ module "land-project" {
# Cloud Storage
-module "land-sa-cs-0" {
+module "drop-sa-cs-0" {
source = "../../../modules/iam-service-account"
- project_id = module.land-project.project_id
+ project_id = module.drop-project.project_id
prefix = var.prefix
- name = "lnd-cs-0"
- display_name = "Data platform GCS landing service account."
+ name = "drp-cs-0"
+ display_name = "Data platform GCS drop off service account."
iam = {
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers
@@ -76,11 +76,11 @@ module "land-sa-cs-0" {
}
}
-module "land-cs-0" {
+module "drop-cs-0" {
source = "../../../modules/gcs"
- project_id = module.land-project.project_id
+ project_id = module.drop-project.project_id
prefix = var.prefix
- name = "lnd-cs-0"
+ name = "drp-cs-0"
location = var.location
storage_class = "MULTI_REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
@@ -93,12 +93,12 @@ module "land-cs-0" {
# PubSub
-module "land-sa-ps-0" {
+module "drop-sa-ps-0" {
source = "../../../modules/iam-service-account"
- project_id = module.land-project.project_id
+ project_id = module.drop-project.project_id
prefix = var.prefix
- name = "lnd-ps-0"
- display_name = "Data platform PubSub landing service account"
+ name = "drp-ps-0"
+ display_name = "Data platform PubSub drop off service account"
iam = {
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers
@@ -106,30 +106,30 @@ module "land-sa-ps-0" {
}
}
-module "land-ps-0" {
+module "drop-ps-0" {
source = "../../../modules/pubsub"
- project_id = module.land-project.project_id
- name = "${var.prefix}-lnd-ps-0"
+ project_id = module.drop-project.project_id
+ name = "${var.prefix}-drp-ps-0"
kms_key = try(local.service_encryption_keys.pubsub, null)
}
# BigQuery
-module "land-sa-bq-0" {
+module "drop-sa-bq-0" {
source = "../../../modules/iam-service-account"
- project_id = module.land-project.project_id
+ project_id = module.drop-project.project_id
prefix = var.prefix
- name = "lnd-bq-0"
- display_name = "Data platform BigQuery landing service account"
+ name = "drp-bq-0"
+ display_name = "Data platform BigQuery drop off service account"
iam = {
"roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers]
}
}
-module "land-bq-0" {
+module "drop-bq-0" {
source = "../../../modules/bigquery-dataset"
- project_id = module.land-project.project_id
- id = "${replace(var.prefix, "-", "_")}lnd_bq_0"
+ project_id = module.drop-project.project_id
+ id = "${replace(var.prefix, "-", "_")}drp_bq_0"
location = var.location
encryption_key = try(local.service_encryption_keys.bq, null)
}
diff --git a/examples/data-solutions/data-platform-foundations/03-composer.tf b/examples/data-solutions/data-platform-foundations/03-composer.tf
index fac47ec57..2622ffa20 100644
--- a/examples/data-solutions/data-platform-foundations/03-composer.tf
+++ b/examples/data-solutions/data-platform-foundations/03-composer.tf
@@ -66,39 +66,39 @@ resource "google_composer_environment" "orch-cmp-0" {
image_version = var.composer_config.airflow_version
env_variables = merge(
var.composer_config.env_variables, {
- BQ_LOCATION = var.location
- DATA_CAT_TAGS = try(jsonencode(module.common-datacatalog.tags), "{}")
- DF_KMS_KEY = try(var.service_encryption_keys.dataflow, "")
- DTL_L0_PRJ = module.lake-0-project.project_id
- DTL_L0_BQ_DATASET = module.lake-0-bq-0.dataset_id
- DTL_L0_GCS = module.lake-0-cs-0.url
- DTL_L1_PRJ = module.lake-1-project.project_id
- DTL_L1_BQ_DATASET = module.lake-1-bq-0.dataset_id
- DTL_L1_GCS = module.lake-1-cs-0.url
- DTL_L2_PRJ = module.lake-2-project.project_id
- DTL_L2_BQ_DATASET = module.lake-2-bq-0.dataset_id
- DTL_L2_GCS = module.lake-2-cs-0.url
- DTL_PLG_PRJ = module.lake-plg-project.project_id
- DTL_PLG_BQ_DATASET = module.lake-plg-bq-0.dataset_id
- DTL_PLG_GCS = module.lake-plg-cs-0.url
- GCP_REGION = var.region
- LND_PRJ = module.land-project.project_id
- LND_BQ = module.land-bq-0.dataset_id
- LND_GCS = module.land-cs-0.url
- LND_PS = module.land-ps-0.id
- LOD_PRJ = module.load-project.project_id
- LOD_GCS_STAGING = module.load-cs-df-0.url
- LOD_NET_VPC = local.load_vpc
- LOD_NET_SUBNET = local.load_subnet
- LOD_SA_DF = module.load-sa-df-0.email
- ORC_PRJ = module.orch-project.project_id
- ORC_GCS = module.orch-cs-0.url
- TRF_PRJ = module.transf-project.project_id
- TRF_GCS_STAGING = module.transf-cs-df-0.url
- TRF_NET_VPC = local.transf_vpc
- TRF_NET_SUBNET = local.transf_subnet
- TRF_SA_DF = module.transf-sa-df-0.email
- TRF_SA_BQ = module.transf-sa-bq-0.email
+ BQ_LOCATION = var.location
+ DATA_CAT_TAGS = try(jsonencode(module.common-datacatalog.tags), "{}")
+ DF_KMS_KEY = try(var.service_encryption_keys.dataflow, "")
+ DRP_PRJ = module.drop-project.project_id
+ DRP_BQ = module.drop-bq-0.dataset_id
+ DRP_GCS = module.drop-cs-0.url
+ DRP_PS = module.drop-ps-0.id
+ DWH_LAND_PRJ = module.dwh-lnd-project.project_id
+ DWH_LAND_BQ_DATASET = module.dwh-lnd-bq-0.dataset_id
+ DWH_LAND_GCS = module.dwh-lnd-cs-0.url
+ DWH_CURATED_PRJ = module.dwh-cur-project.project_id
+ DWH_CURATED_BQ_DATASET = module.dwh-cur-bq-0.dataset_id
+ DWH_CURATED_GCS = module.dwh-cur-cs-0.url
+ DWH_CONFIDENTIAL_PRJ = module.dwh-conf-project.project_id
+ DWH_CONFIDENTIAL_BQ_DATASET = module.dwh-conf-bq-0.dataset_id
+ DWH_CONFIDENTIAL_GCS = module.dwh-conf-cs-0.url
+ DWH_PLG_PRJ = module.dwh-plg-project.project_id
+ DWH_PLG_BQ_DATASET = module.dwh-plg-bq-0.dataset_id
+ DWH_PLG_GCS = module.dwh-plg-cs-0.url
+ GCP_REGION = var.region
+ LOD_PRJ = module.load-project.project_id
+ LOD_GCS_STAGING = module.load-cs-df-0.url
+ LOD_NET_VPC = local.load_vpc
+ LOD_NET_SUBNET = local.load_subnet
+ LOD_SA_DF = module.load-sa-df-0.email
+ ORC_PRJ = module.orch-project.project_id
+ ORC_GCS = module.orch-cs-0.url
+ TRF_PRJ = module.transf-project.project_id
+ TRF_GCS_STAGING = module.transf-cs-df-0.url
+ TRF_NET_VPC = local.transf_vpc
+ TRF_NET_SUBNET = local.transf_subnet
+ TRF_SA_DF = module.transf-sa-df-0.email
+ TRF_SA_BQ = module.transf-sa-bq-0.email
}
)
}
diff --git a/examples/data-solutions/data-platform-foundations/05-datalake.tf b/examples/data-solutions/data-platform-foundations/05-datawarehouse.tf
similarity index 74%
rename from examples/data-solutions/data-platform-foundations/05-datalake.tf
rename to examples/data-solutions/data-platform-foundations/05-datawarehouse.tf
index b163f9e56..879a0e0b1 100644
--- a/examples/data-solutions/data-platform-foundations/05-datalake.tf
+++ b/examples/data-solutions/data-platform-foundations/05-datawarehouse.tf
@@ -12,10 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# tfdoc:file:description Datalake projects.
+# tfdoc:file:description Data Warehouse projects.
locals {
- lake_group_iam = {
+ dwh_group_iam = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor",
"roles/storage.admin",
@@ -30,7 +30,7 @@ locals {
"roles/storage.objectViewer",
]
}
- lake_plg_group_iam = {
+ dwh_plg_group_iam = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor",
"roles/storage.admin",
@@ -45,7 +45,7 @@ locals {
"roles/storage.objectAdmin",
]
}
- lake_0_iam = {
+ dwh_lnd_iam = {
"roles/bigquery.dataOwner" = [
module.load-sa-df-0.iam_email,
module.transf-sa-df-0.iam_email,
@@ -61,7 +61,7 @@ locals {
module.load-sa-df-0.iam_email,
]
}
- lake_iam = {
+ dwh_iam = {
"roles/bigquery.dataOwner" = [
module.transf-sa-df-0.iam_email,
module.transf-sa-bq-0.iam_email,
@@ -79,7 +79,7 @@ locals {
module.transf-sa-df-0.iam_email,
]
}
- lake_services = concat(var.project_services, [
+ dwh_services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
"bigquerystorage.googleapis.com",
@@ -95,60 +95,60 @@ locals {
# Project
-module "lake-0-project" {
+module "dwh-lnd-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
- name = "dtl-0${local.project_suffix}"
- group_iam = local.lake_group_iam
- iam = local.lake_0_iam
- services = local.lake_services
+ name = "dwh-lnd${local.project_suffix}"
+ group_iam = local.dwh_group_iam
+ iam = local.dwh_lnd_iam
+ services = local.dwh_services
service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)]
}
}
-module "lake-1-project" {
+module "dwh-cur-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
- name = "dtl-1${local.project_suffix}"
- group_iam = local.lake_group_iam
- iam = local.lake_iam
- services = local.lake_services
+ name = "dwh-cur${local.project_suffix}"
+ group_iam = local.dwh_group_iam
+ iam = local.dwh_iam
+ services = local.dwh_services
service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)]
}
}
-module "lake-2-project" {
+module "dwh-conf-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
- name = "dtl-2${local.project_suffix}"
- group_iam = local.lake_group_iam
- iam = local.lake_iam
- services = local.lake_services
+ name = "dwh-conf${local.project_suffix}"
+ group_iam = local.dwh_group_iam
+ iam = local.dwh_iam
+ services = local.dwh_services
service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)]
}
}
-module "lake-plg-project" {
+module "dwh-plg-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
- name = "dtl-plg${local.project_suffix}"
- group_iam = local.lake_plg_group_iam
+ name = "dwh-plg${local.project_suffix}"
+ group_iam = local.dwh_plg_group_iam
iam = {}
- services = local.lake_services
+ services = local.dwh_services
service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)]
@@ -157,78 +157,78 @@ module "lake-plg-project" {
# Bigquery
-module "lake-0-bq-0" {
+module "dwh-lnd-bq-0" {
source = "../../../modules/bigquery-dataset"
- project_id = module.lake-0-project.project_id
- id = "${replace(var.prefix, "-", "_")}_dtl_0_bq_0"
+ project_id = module.dwh-lnd-project.project_id
+ id = "${replace(var.prefix, "-", "_")}_dwh_lnd_bq_0"
location = var.location
encryption_key = try(local.service_encryption_keys.bq, null)
}
-module "lake-1-bq-0" {
+module "dwh-cur-bq-0" {
source = "../../../modules/bigquery-dataset"
- project_id = module.lake-1-project.project_id
- id = "${replace(var.prefix, "-", "_")}_dtl_1_bq_0"
+ project_id = module.dwh-cur-project.project_id
+ id = "${replace(var.prefix, "-", "_")}_dwh_lnd_bq_0"
location = var.location
encryption_key = try(local.service_encryption_keys.bq, null)
}
-module "lake-2-bq-0" {
+module "dwh-conf-bq-0" {
source = "../../../modules/bigquery-dataset"
- project_id = module.lake-2-project.project_id
- id = "${replace(var.prefix, "-", "_")}_dtl_2_bq_0"
+ project_id = module.dwh-conf-project.project_id
+ id = "${replace(var.prefix, "-", "_")}_dwh_conf_bq_0"
location = var.location
encryption_key = try(local.service_encryption_keys.bq, null)
}
-module "lake-plg-bq-0" {
+module "dwh-plg-bq-0" {
source = "../../../modules/bigquery-dataset"
- project_id = module.lake-plg-project.project_id
- id = "${replace(var.prefix, "-", "_")}_dtl_plg_bq_0"
+ project_id = module.dwh-plg-project.project_id
+ id = "${replace(var.prefix, "-", "_")}_dwh_plg_bq_0"
location = var.location
encryption_key = try(local.service_encryption_keys.bq, null)
}
# Cloud storage
-module "lake-0-cs-0" {
+module "dwh-lnd-cs-0" {
source = "../../../modules/gcs"
- project_id = module.lake-0-project.project_id
+ project_id = module.dwh-lnd-project.project_id
prefix = var.prefix
- name = "dtl-0-cs-0"
+ name = "dwh-lnd-cs-0"
location = var.location
storage_class = "MULTI_REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
force_destroy = var.data_force_destroy
}
-module "lake-1-cs-0" {
+module "dwh-cur-cs-0" {
source = "../../../modules/gcs"
- project_id = module.lake-1-project.project_id
+ project_id = module.dwh-cur-project.project_id
prefix = var.prefix
- name = "dtl-1-cs-0"
+ name = "dwh-cur-cs-0"
location = var.location
storage_class = "MULTI_REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
force_destroy = var.data_force_destroy
}
-module "lake-2-cs-0" {
+module "dwh-conf-cs-0" {
source = "../../../modules/gcs"
- project_id = module.lake-2-project.project_id
+ project_id = module.dwh-conf-project.project_id
prefix = var.prefix
- name = "dtl-2-cs-0"
+ name = "dwh-conf-cs-0"
location = var.location
storage_class = "MULTI_REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
force_destroy = var.data_force_destroy
}
-module "lake-plg-cs-0" {
+module "dwh-plg-cs-0" {
source = "../../../modules/gcs"
- project_id = module.lake-plg-project.project_id
+ project_id = module.dwh-plg-project.project_id
prefix = var.prefix
- name = "dtl-plg-cs-0"
+ name = "dwh-plg-cs-0"
location = var.location
storage_class = "MULTI_REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
diff --git a/examples/data-solutions/data-platform-foundations/IAM.md b/examples/data-solutions/data-platform-foundations/IAM.md
index d6ccbecbc..54d35939b 100644
--- a/examples/data-solutions/data-platform-foundations/IAM.md
+++ b/examples/data-solutions/data-platform-foundations/IAM.md
@@ -13,7 +13,40 @@ Legend: + additive, • conditional.
|trf-bq-0
serviceAccount|[roles/datacatalog.categoryFineGrainedReader](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryFineGrainedReader)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) |
|trf-df-0
serviceAccount|[roles/datacatalog.categoryFineGrainedReader](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryFineGrainedReader)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/dlp.user](https://cloud.google.com/iam/docs/understanding-roles#dlp.user) |
-## Project dtl-0
+## Project drp
+
+| members | roles |
+|---|---|
+|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/pubsub.editor](https://cloud.google.com/iam/docs/understanding-roles#pubsub.editor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
+|drp-bq-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) |
+|drp-cs-0
serviceAccount|[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) |
+|drp-ps-0
serviceAccount|[roles/pubsub.publisher](https://cloud.google.com/iam/docs/understanding-roles#pubsub.publisher) |
+|load-df-0
serviceAccount|[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
+|orc-cmp-0
serviceAccount|[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
+
+## Project dwh-conf
+
+| members | roles |
+|---|---|
+|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
+|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
+|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +|
+|load-df-0
serviceAccount|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
+|trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) |
+|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
+
+## Project dwh-cur
+
+| members | roles |
+|---|---|
+|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
+|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
+|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +|
+|load-df-0
serviceAccount|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
+|trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) |
+|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
+
+## Project dwh-lnd
| members | roles |
|---|---|
@@ -24,29 +57,7 @@ Legend: + additive, • conditional.
|trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) |
-## Project dtl-1
-
-| members | roles |
-|---|---|
-|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
-|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
-|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +|
-|load-df-0
serviceAccount|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
-|trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) |
-|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
-
-## Project dtl-2
-
-| members | roles |
-|---|---|
-|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
-|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
-|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +|
-|load-df-0
serviceAccount|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
-|trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) |
-|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
-
-## Project dtl-plg
+## Project dwh-plg
| members | roles |
|---|---|
@@ -54,17 +65,6 @@ Legend: + additive, • conditional.
|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +|
-## Project lnd
-
-| members | roles |
-|---|---|
-|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/pubsub.editor](https://cloud.google.com/iam/docs/understanding-roles#pubsub.editor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
-|lnd-bq-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) |
-|lnd-cs-0
serviceAccount|[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) |
-|lnd-ps-0
serviceAccount|[roles/pubsub.publisher](https://cloud.google.com/iam/docs/understanding-roles#pubsub.publisher) |
-|load-df-0
serviceAccount|[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
-|orc-cmp-0
serviceAccount|[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
-
## Project lod
| members | roles |
diff --git a/examples/data-solutions/data-platform-foundations/README.md b/examples/data-solutions/data-platform-foundations/README.md
index 09f8e63a5..2ac1a9871 100644
--- a/examples/data-solutions/data-platform-foundations/README.md
+++ b/examples/data-solutions/data-platform-foundations/README.md
@@ -27,9 +27,9 @@ The code in this example doesn't address Organization-level configurations (Orga
The Data Platform is designed to rely on several projects, one project per data stage. The stages identified are:
-- landing
+- drop off
- load
-- data lake
+- data warehouse
- orchestration
- transformation
- exposure
@@ -38,15 +38,15 @@ This separation into projects allows adhering to the least-privilege principle b
The script will create the following projects:
-- **Landing** Used to store temporary data. Data is pushed to Cloud Storage, BigQuery, or Cloud PubSub. Resources are configured with a customizable lifecycle policy.
-- **Load** Used to load data from landing to data lake. The load is made with minimal to zero transformation logic (mainly `cast`). Anonymization or tokenization of Personally Identifiable Information (PII) can be implemented here or in the transformation stage, depending on your requirements. The use of [Cloud Dataflow templates](https://cloud.google.com/dataflow/docs/concepts/dataflow-templates) is recommended.
-- **Data Lake** Several projects distributed across 3 separate layers, to host progressively processed and refined data:
- - **L0 - Raw data** Structured Data, stored in relevant formats: structured data stored in BigQuery, unstructured data stored on Cloud Storage with additional metadata stored in BigQuery (for example pictures stored in Cloud Storage and analysis of the images for Cloud Vision API stored in BigQuery).
- - **L1 - Cleansed, aggregated and standardized data**
- - **L2 - Curated layer**
- - **Playground** Temporary tables that Data Analyst may use to perform R&D on data available in other Data Lake layers.
+- **Drop off** Used to store temporary data. Data is pushed to Cloud Storage, BigQuery, or Cloud PubSub. Resources are configured with a customizable lifecycle policy.
+- **Load** Used to load data from the drop off zone to the data warehouse. The load is made with minimal to zero transformation logic (mainly `cast`). Anonymization or tokenization of Personally Identifiable Information (PII) can be implemented here or in the transformation stage, depending on your requirements. The use of [Cloud Dataflow templates](https://cloud.google.com/dataflow/docs/concepts/dataflow-templates) is recommended.
+- **Data Warehouse** Several projects distributed across 3 separate layers, to host progressively processed and refined data:
+ - **Landing - Raw data** Structured Data, stored in relevant formats: structured data stored in BigQuery, unstructured data stored on Cloud Storage with additional metadata stored in BigQuery (for example pictures stored in Cloud Storage and analysis of the images for Cloud Vision API stored in BigQuery).
+ - **Curated - Cleansed, aggregated and curated data**
+ - **Confidential - Curated and unencrypted layer**
+ - **Playground** Temporary tables that Data Analyst may use to perform R&D on data available in other Data Warehouse layers.
- **Orchestration** Used to host Cloud Composer, which orchestrates all tasks that move data across layers.
-- **Transformation** Used to move data between Data Lake layers. We strongly suggest relying on BigQuery Engine to perform the transformations. If BigQuery doesn't have the features needed to perform your transformations, you can use Cloud Dataflow with [Cloud Dataflow templates](https://cloud.google.com/dataflow/docs/concepts/dataflow-templates). This stage can also optionally anonymize or tokenize PII.
+- **Transformation** Used to move data between Data Warehouse layers. We strongly suggest relying on BigQuery Engine to perform the transformations. If BigQuery doesn't have the features needed to perform your transformations, you can use Cloud Dataflow with [Cloud Dataflow templates](https://cloud.google.com/dataflow/docs/concepts/dataflow-templates). This stage can also optionally anonymize or tokenize PII.
- **Exposure** Used to host resources that share processed data with external systems. Depending on the access pattern, data can be presented via Cloud SQL, BigQuery, or Bigtable. For BigQuery data, we strongly suggest relying on [Authorized views](https://cloud.google.com/bigquery/docs/authorized-views).
### Roles
@@ -57,9 +57,9 @@ We assign roles on resources at the project level, granting the appropriate role
Service account creation follows the least privilege principle, performing a single task which requires access to a defined set of resources. The table below shows a high level overview of roles for each service account on each data layer, using `READ` or `WRITE` access patterns for simplicity. For detailed roles please refer to the code.
-|Service Account|Landing|DataLake L0|DataLake L1|DataLake L2|
+|Service Account|Drop off|DWH Landing|DWH Curated|DWH Confidential|
|-|:-:|:-:|:-:|:-:|
-|`landing-sa`|`WRITE`|-|-|-|
+|`drop-sa`|`WRITE`|-|-|-|
|`load-sa`|`READ`|`READ`/`WRITE`|-|-|
|`transformation-sa`|-|`READ`/`WRITE`|`READ`/`WRITE`|`READ`/`WRITE`|
|`orchestration-sa`|-|-|-|-|
@@ -75,12 +75,12 @@ User groups provide a stable frame of reference that allows decoupling the final
We use three groups to control access to resources:
- *Data Engineers* They handle and run the Data Hub, with read access to all resources in order to troubleshoot possible issues with pipelines. This team can also impersonate any service account.
-- *Data Analysts*. They perform analysis on datasets, with read access to the data lake L2 project, and BigQuery READ/WRITE access to the playground project.
+- *Data Analysts*. They perform analysis on datasets, with read access to the Data Warehouse Confidential project, and BigQuery READ/WRITE access to the playground project.
- *Data Security*:. They handle security configurations related to the Data Hub. This team has admin access to the common project to configure Cloud DLP templates or Data Catalog policy tags.
The table below shows a high level overview of roles for each group on each project, using `READ`, `WRITE` and `ADMIN` access patterns for simplicity. For detailed roles please refer to the code.
-|Group|Landing|Load|Transformation|Data Lake L0|Data Lake L1|Data Lake L2|Data Lake Playground|Orchestration|Common|
+|Group|Drop off|Load|Transformation|DHW Landing|DWH Curated|DWH Confidential|DWH Playground|Orchestration|Common|
|-|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|
|Data Engineers|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|
|Data Analysts|-|-|-|-|-|`READ`|`READ`/`WRITE`|-|-|
@@ -215,12 +215,12 @@ To create Cloud Key Management keys in the Data Platform you can uncomment the C
### Assign roles at BQ Dataset level
-To handle multiple groups of `data-analysts` accessing the same Data Lake layer projects but only to the dataset belonging to a specific group, you may want to assign roles at BigQuery dataset level instead of at project-level.
+To handle multiple groups of `data-analysts` accessing the same Data Warehouse layer projects but only to the dataset belonging to a specific group, you may want to assign roles at BigQuery dataset level instead of at project-level.
To do this, you need to remove IAM binging at project-level for the `data-analysts` group and give roles at BigQuery dataset level using the `iam` variable on `bigquery-dataset` modules.
## Demo pipeline
-The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `landing` area to the `DataLake L2` dataset suing different features.
+The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `drop off` area to the `Data Warehouse Confidential` dataset suing different features.
You can find examples in the `[demo](./demo)` folder.
diff --git a/examples/data-solutions/data-platform-foundations/demo/README.md b/examples/data-solutions/data-platform-foundations/demo/README.md
index 5347b2cff..13f750a31 100644
--- a/examples/data-solutions/data-platform-foundations/demo/README.md
+++ b/examples/data-solutions/data-platform-foundations/demo/README.md
@@ -8,7 +8,7 @@ The example is not intended to be a production-ready code.
The demo imports purchase data generated by a store.
## Input files
-Data are uploaded to the `landing` GCS bucket. File structure:
+Data are uploaded to the `drop off` GCS bucket. File structure:
- `customers.csv`: Comma separate value with customer information in the following format: Customer ID, Name, Surname, Registration Timestamp
- `purchases.csv`: Comma separate value with customer information in the following format: Item ID, Customer ID, Item, Item price, Purchase Timestamp
@@ -16,14 +16,14 @@ Data are uploaded to the `landing` GCS bucket. File structure:
Different data pipelines are provided to highlight different features and patterns. For the purpose of the example, a single pipeline handle all data lifecycles. When adapting them to your real use case, you may want to evaluate the option to handle each functional step on a separate pipeline or a dedicated tool. For example, you may want to use `Dataform` to handle data schemas lifecycle.
Below you can find a description of each example:
- - Simple import data: [`datapipeline.py`](./datapipeline.py) is a simple pipeline to import provided data from the `landing` Google Cloud Storage bucket to the Data Hub L2 layer joining `customers` and `purchases` tables into `customerpurchase` table.
- - Import data with Policy Tags: [`datapipeline_dc_tags.py`](./datapipeline.py) imports provided data from `landing` bucket to the Data Hub L2 layer protecting sensitive data using Data Catalog policy Tags.
+ - Simple import data: [`datapipeline.py`](./datapipeline.py) is a simple pipeline to import provided data from the `drop off` Google Cloud Storage bucket to the Data Hub Confidential layer joining `customers` and `purchases` tables into `customerpurchase` table.
+ - Import data with Policy Tags: [`datapipeline_dc_tags.py`](./datapipeline.py) imports provided data from `drop off` bucket to the Data Hub Confidential layer protecting sensitive data using Data Catalog policy Tags.
- Delete tables: [`delete_table.py`](./delete_table.py) deletes BigQuery tables created by import pipelines.
## Runnin the demo
To run demo examples, please follow the following steps:
-- 01: copy sample data to the `landing` Cloud Storage bucket impersonating the `load` service account.
+- 01: copy sample data to the `drop off` Cloud Storage bucket impersonating the `load` service account.
- 02: copy sample data structure definition in the `orchestration` Cloud Storage bucket impersonating the `orchestration` service account.
- 03: copy the Cloud Composer DAG to the Cloud Composer Storage bucket impersonating the `orchestration` service account.
- 04: Open the Cloud Composer Airflow UI and run the imported DAG.
diff --git a/examples/data-solutions/data-platform-foundations/demo/datapipeline.py b/examples/data-solutions/data-platform-foundations/demo/datapipeline.py
index 1f748c088..a682d346a 100644
--- a/examples/data-solutions/data-platform-foundations/demo/datapipeline.py
+++ b/examples/data-solutions/data-platform-foundations/demo/datapipeline.py
@@ -34,23 +34,23 @@ from airflow.utils.task_group import TaskGroup
# --------------------------------------------------------------------------------
BQ_LOCATION = os.environ.get("BQ_LOCATION")
DATA_CAT_TAGS = json.loads(os.environ.get("DATA_CAT_TAGS"))
-DTL_L0_PRJ = os.environ.get("DTL_L0_PRJ")
-DTL_L0_BQ_DATASET = os.environ.get("DTL_L0_BQ_DATASET")
-DTL_L0_GCS = os.environ.get("DTL_L0_GCS")
-DTL_L1_PRJ = os.environ.get("DTL_L1_PRJ")
-DTL_L1_BQ_DATASET = os.environ.get("DTL_L1_BQ_DATASET")
-DTL_L1_GCS = os.environ.get("DTL_L1_GCS")
-DTL_L2_PRJ = os.environ.get("DTL_L2_PRJ")
-DTL_L2_BQ_DATASET = os.environ.get("DTL_L2_BQ_DATASET")
-DTL_L2_GCS = os.environ.get("DTL_L2_GCS")
-DTL_PLG_PRJ = os.environ.get("DTL_PLG_PRJ")
-DTL_PLG_BQ_DATASET = os.environ.get("DTL_PLG_BQ_DATASET")
-DTL_PLG_GCS = os.environ.get("DTL_PLG_GCS")
+DWH_LAND_PRJ = os.environ.get("DWH_LAND_PRJ")
+DWH_LAND_BQ_DATASET = os.environ.get("DWH_LAND_BQ_DATASET")
+DWH_LAND_GCS = os.environ.get("DWH_LAND_GCS")
+DWH_CURATED_PRJ = os.environ.get("DWH_CURATED_PRJ")
+DWH_CURATED_BQ_DATASET = os.environ.get("DWH_CURATED_BQ_DATASET")
+DWH_CURATED_GCS = os.environ.get("DWH_CURATED_GCS")
+DWH_CONFIDENTIAL_PRJ = os.environ.get("DWH_CONFIDENTIAL_PRJ")
+DWH_CONFIDENTIAL_BQ_DATASET = os.environ.get("DWH_CONFIDENTIAL_BQ_DATASET")
+DWH_CONFIDENTIAL_GCS = os.environ.get("DWH_CONFIDENTIAL_GCS")
+DWH_PLG_PRJ = os.environ.get("DWH_PLG_PRJ")
+DWH_PLG_BQ_DATASET = os.environ.get("DWH_PLG_BQ_DATASET")
+DWH_PLG_GCS = os.environ.get("DWH_PLG_GCS")
GCP_REGION = os.environ.get("GCP_REGION")
-LND_PRJ = os.environ.get("LND_PRJ")
-LND_BQ = os.environ.get("LND_BQ")
-LND_GCS = os.environ.get("LND_GCS")
-LND_PS = os.environ.get("LND_PS")
+DRP_PRJ = os.environ.get("DRP_PRJ")
+DRP_BQ = os.environ.get("DRP_BQ")
+DRP_GCS = os.environ.get("DRP_GCS")
+DRP_PS = os.environ.get("DRP_PS")
LOD_PRJ = os.environ.get("LOD_PRJ")
LOD_GCS_STAGING = os.environ.get("LOD_GCS_STAGING")
LOD_NET_VPC = os.environ.get("LOD_NET_VPC")
@@ -127,8 +127,8 @@ with models.DAG(
"javascriptTextTransformFunctionName": "transform",
"JSONPath": ORC_GCS + "/customers_schema.json",
"javascriptTextTransformGcsPath": ORC_GCS + "/customers_udf.js",
- "inputFilePattern": LND_GCS + "/customers.csv",
- "outputTable": DTL_L0_PRJ + ":"+DTL_L0_BQ_DATASET+".customers",
+ "inputFilePattern": DRP_GCS + "/customers.csv",
+ "outputTable": DWH_LAND_PRJ + ":" + DWH_LAND_BQ_DATASET + ".customers",
"bigQueryLoadingTemporaryDirectory": LOD_GCS_STAGING + "/tmp/bq/",
},
)
@@ -142,8 +142,8 @@ with models.DAG(
"javascriptTextTransformFunctionName": "transform",
"JSONPath": ORC_GCS + "/purchases_schema.json",
"javascriptTextTransformGcsPath": ORC_GCS + "/purchases_udf.js",
- "inputFilePattern": LND_GCS + "/purchases.csv",
- "outputTable": DTL_L0_PRJ + ":"+DTL_L0_BQ_DATASET+".purchases",
+ "inputFilePattern": DRP_GCS + "/purchases.csv",
+ "outputTable": DWH_LAND_PRJ + ":" + DWH_LAND_BQ_DATASET + ".purchases",
"bigQueryLoadingTemporaryDirectory": LOD_GCS_STAGING + "/tmp/bq/",
},
)
@@ -159,17 +159,15 @@ with models.DAG(
'query':"""SELECT
c.id as customer_id,
p.id as purchase_id,
- c.name as name,
- c.surname as surname,
p.item as item,
p.price as price,
p.timestamp as timestamp
- FROM `{dtl_0_prj}.{dtl_0_dataset}.customers` c
- JOIN `{dtl_0_prj}.{dtl_0_dataset}.purchases` p ON c.id = p.customer_id
- """.format(dtl_0_prj=DTL_L0_PRJ, dtl_0_dataset=DTL_L0_BQ_DATASET, ),
+ FROM `{dwh_0_prj}.{dwh_0_dataset}.customers` c
+ JOIN `{dwh_0_prj}.{dwh_0_dataset}.purchases` p ON c.id = p.customer_id
+ """.format(dwh_0_prj=DWH_LAND_PRJ, dwh_0_dataset=DWH_LAND_BQ_DATASET, ),
'destinationTable':{
- 'projectId': DTL_L1_PRJ,
- 'datasetId': DTL_L1_BQ_DATASET,
+ 'projectId': DWH_CURATED_PRJ,
+ 'datasetId': DWH_CURATED_BQ_DATASET,
'tableId': 'customer_purchase'
},
'writeDisposition':'WRITE_TRUNCATE',
@@ -179,8 +177,8 @@ with models.DAG(
impersonation_chain=[TRF_SA_BQ]
)
- l2_customer_purchase = BigQueryInsertJobOperator(
- task_id='bq_l2_customer_purchase',
+ confidential_customer_purchase = BigQueryInsertJobOperator(
+ task_id='bq_confidential_customer_purchase',
gcp_conn_id='bigquery_default',
project_id=TRF_PRJ,
location=BQ_LOCATION,
@@ -188,18 +186,19 @@ with models.DAG(
'jobType':'QUERY',
'query':{
'query':"""SELECT
- customer_id,
- purchase_id,
- name,
- surname,
- item,
- price,
- timestamp
- FROM `{dtl_1_prj}.{dtl_1_dataset}.customer_purchase`
- """.format(dtl_1_prj=DTL_L1_PRJ, dtl_1_dataset=DTL_L1_BQ_DATASET, ),
+ c.id as customer_id,
+ p.id as purchase_id,
+ c.name as name,
+ c.surname as surname,
+ p.item as item,
+ p.price as price,
+ p.timestamp as timestamp
+ FROM `{dwh_0_prj}.{dwh_0_dataset}.customers` c
+ JOIN `{dwh_0_prj}.{dwh_0_dataset}.purchases` p ON c.id = p.customer_id
+ """.format(dwh_0_prj=DWH_LAND_PRJ, dwh_0_dataset=DWH_LAND_BQ_DATASET, ),
'destinationTable':{
- 'projectId': DTL_L2_PRJ,
- 'datasetId': DTL_L2_BQ_DATASET,
+ 'projectId': DWH_CONFIDENTIAL_PRJ,
+ 'datasetId': DWH_CONFIDENTIAL_BQ_DATASET,
'tableId': 'customer_purchase'
},
'writeDisposition':'WRITE_TRUNCATE',
@@ -209,4 +208,4 @@ with models.DAG(
impersonation_chain=[TRF_SA_BQ]
)
- start >> [customers_import, purchases_import] >> join_customer_purchase >> l2_customer_purchase >> end
\ No newline at end of file
+ start >> [customers_import, purchases_import] >> join_customer_purchase >> confidential_customer_purchase >> end
\ No newline at end of file
diff --git a/examples/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py b/examples/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py
index 2fb88c9e5..4b15eaaba 100644
--- a/examples/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py
+++ b/examples/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py
@@ -34,23 +34,23 @@ from airflow.utils.task_group import TaskGroup
# --------------------------------------------------------------------------------
BQ_LOCATION = os.environ.get("BQ_LOCATION")
DATA_CAT_TAGS = json.loads(os.environ.get("DATA_CAT_TAGS"))
-DTL_L0_PRJ = os.environ.get("DTL_L0_PRJ")
-DTL_L0_BQ_DATASET = os.environ.get("DTL_L0_BQ_DATASET")
-DTL_L0_GCS = os.environ.get("DTL_L0_GCS")
-DTL_L1_PRJ = os.environ.get("DTL_L1_PRJ")
-DTL_L1_BQ_DATASET = os.environ.get("DTL_L1_BQ_DATASET")
-DTL_L1_GCS = os.environ.get("DTL_L1_GCS")
-DTL_L2_PRJ = os.environ.get("DTL_L2_PRJ")
-DTL_L2_BQ_DATASET = os.environ.get("DTL_L2_BQ_DATASET")
-DTL_L2_GCS = os.environ.get("DTL_L2_GCS")
-DTL_PLG_PRJ = os.environ.get("DTL_PLG_PRJ")
-DTL_PLG_BQ_DATASET = os.environ.get("DTL_PLG_BQ_DATASET")
-DTL_PLG_GCS = os.environ.get("DTL_PLG_GCS")
+DWH_LAND_PRJ = os.environ.get("DWH_LAND_PRJ")
+DWH_LAND_BQ_DATASET = os.environ.get("DWH_LAND_BQ_DATASET")
+DWH_LAND_GCS = os.environ.get("DWH_LAND_GCS")
+DWH_CURATED_PRJ = os.environ.get("DWH_CURATED_PRJ")
+DWH_CURATED_BQ_DATASET = os.environ.get("DWH_CURATED_BQ_DATASET")
+DWH_CURATED_GCS = os.environ.get("DWH_CURATED_GCS")
+DWH_CONFIDENTIAL_PRJ = os.environ.get("DWH_CONFIDENTIAL_PRJ")
+DWH_CONFIDENTIAL_BQ_DATASET = os.environ.get("DWH_CONFIDENTIAL_BQ_DATASET")
+DWH_CONFIDENTIAL_GCS = os.environ.get("DWH_CONFIDENTIAL_GCS")
+DWH_PLG_PRJ = os.environ.get("DWH_PLG_PRJ")
+DWH_PLG_BQ_DATASET = os.environ.get("DWH_PLG_BQ_DATASET")
+DWH_PLG_GCS = os.environ.get("DWH_PLG_GCS")
GCP_REGION = os.environ.get("GCP_REGION")
-LND_PRJ = os.environ.get("LND_PRJ")
-LND_BQ = os.environ.get("LND_BQ")
-LND_GCS = os.environ.get("LND_GCS")
-LND_PS = os.environ.get("LND_PS")
+DRP_PRJ = os.environ.get("DRP_PRJ")
+DRP_BQ = os.environ.get("DRP_BQ")
+DRP_GCS = os.environ.get("DRP_GCS")
+DRP_PS = os.environ.get("DRP_PS")
LOD_PRJ = os.environ.get("LOD_PRJ")
LOD_GCS_STAGING = os.environ.get("LOD_GCS_STAGING")
LOD_NET_VPC = os.environ.get("LOD_NET_VPC")
@@ -121,8 +121,8 @@ with models.DAG(
with TaskGroup('upsert_table') as upsert_table:
upsert_table_customers = BigQueryUpsertTableOperator(
task_id="upsert_table_customers",
- project_id=DTL_L0_PRJ,
- dataset_id=DTL_L0_BQ_DATASET,
+ project_id=DWH_LAND_PRJ,
+ dataset_id=DWH_LAND_BQ_DATASET,
impersonation_chain=[TRF_SA_DF],
table_resource={
"tableReference": {"tableId": "customers"},
@@ -131,28 +131,28 @@ with models.DAG(
upsert_table_purchases = BigQueryUpsertTableOperator(
task_id="upsert_table_purchases",
- project_id=DTL_L0_PRJ,
- dataset_id=DTL_L0_BQ_DATASET,
+ project_id=DWH_LAND_PRJ,
+ dataset_id=DWH_LAND_BQ_DATASET,
impersonation_chain=[TRF_SA_BQ],
table_resource={
"tableReference": {"tableId": "purchases"}
},
)
- upsert_table_customer_purchase_l1 = BigQueryUpsertTableOperator(
- task_id="upsert_table_customer_purchase_l1",
- project_id=DTL_L1_PRJ,
- dataset_id=DTL_L1_BQ_DATASET,
+ upsert_table_customer_purchase_curated = BigQueryUpsertTableOperator(
+ task_id="upsert_table_customer_purchase_curated",
+ project_id=DWH_CURATED_PRJ,
+ dataset_id=DWH_CURATED_BQ_DATASET,
impersonation_chain=[TRF_SA_BQ],
table_resource={
"tableReference": {"tableId": "customer_purchase"}
},
)
- upsert_table_customer_purchase_l2 = BigQueryUpsertTableOperator(
- task_id="upsert_table_customer_purchase_l2",
- project_id=DTL_L2_PRJ,
- dataset_id=DTL_L2_BQ_DATASET,
+ upsert_table_customer_purchase_confidential = BigQueryUpsertTableOperator(
+ task_id="upsert_table_customer_purchase_confidential",
+ project_id=DWH_CONFIDENTIAL_PRJ,
+ dataset_id=DWH_CONFIDENTIAL_BQ_DATASET,
impersonation_chain=[TRF_SA_BQ],
table_resource={
"tableReference": {"tableId": "customer_purchase"}
@@ -164,8 +164,8 @@ with models.DAG(
with TaskGroup('update_schema_table') as update_schema_table:
update_table_schema_customers = BigQueryUpdateTableSchemaOperator(
task_id="update_table_schema_customers",
- project_id=DTL_L0_PRJ,
- dataset_id=DTL_L0_BQ_DATASET,
+ project_id=DWH_LAND_PRJ,
+ dataset_id=DWH_LAND_BQ_DATASET,
table_id="customers",
impersonation_chain=[TRF_SA_BQ],
include_policy_tags=True,
@@ -179,8 +179,8 @@ with models.DAG(
update_table_schema_customers = BigQueryUpdateTableSchemaOperator(
task_id="update_table_schema_purchases",
- project_id=DTL_L0_PRJ,
- dataset_id=DTL_L0_BQ_DATASET,
+ project_id=DWH_LAND_PRJ,
+ dataset_id=DWH_LAND_BQ_DATASET,
table_id="purchases",
impersonation_chain=[TRF_SA_BQ],
include_policy_tags=True,
@@ -193,10 +193,10 @@ with models.DAG(
]
)
- update_table_schema_customer_purchase_l1 = BigQueryUpdateTableSchemaOperator(
- task_id="update_table_schema_customer_purchase_l1",
- project_id=DTL_L1_PRJ,
- dataset_id=DTL_L1_BQ_DATASET,
+ update_table_schema_customer_purchase_curated = BigQueryUpdateTableSchemaOperator(
+ task_id="update_table_schema_customer_purchase_curated",
+ project_id=DWH_CURATED_PRJ,
+ dataset_id=DWH_CURATED_BQ_DATASET,
table_id="customer_purchase",
impersonation_chain=[TRF_SA_BQ],
include_policy_tags=True,
@@ -211,10 +211,10 @@ with models.DAG(
]
)
- update_table_schema_customer_purchase_l2 = BigQueryUpdateTableSchemaOperator(
- task_id="update_table_schema_customer_purchase_l2",
- project_id=DTL_L2_PRJ,
- dataset_id=DTL_L2_BQ_DATASET,
+ update_table_schema_customer_purchase_confidential = BigQueryUpdateTableSchemaOperator(
+ task_id="update_table_schema_customer_purchase_confidential",
+ project_id=DWH_CONFIDENTIAL_PRJ,
+ dataset_id=DWH_CONFIDENTIAL_BQ_DATASET,
table_id="customer_purchase",
impersonation_chain=[TRF_SA_BQ],
include_policy_tags=True,
@@ -238,8 +238,8 @@ with models.DAG(
"javascriptTextTransformFunctionName": "transform",
"JSONPath": ORC_GCS + "/customers_schema.json",
"javascriptTextTransformGcsPath": ORC_GCS + "/customers_udf.js",
- "inputFilePattern": LND_GCS + "/customers.csv",
- "outputTable": DTL_L0_PRJ + ":"+DTL_L0_BQ_DATASET+".customers",
+ "inputFilePattern": DRP_GCS + "/customers.csv",
+ "outputTable": DWH_LAND_PRJ + ":" + DWH_LAND_BQ_DATASET + ".customers",
"bigQueryLoadingTemporaryDirectory": LOD_GCS_STAGING + "/tmp/bq/",
},
)
@@ -253,8 +253,8 @@ with models.DAG(
"javascriptTextTransformFunctionName": "transform",
"JSONPath": ORC_GCS + "/purchases_schema.json",
"javascriptTextTransformGcsPath": ORC_GCS + "/purchases_udf.js",
- "inputFilePattern": LND_GCS + "/purchases.csv",
- "outputTable": DTL_L0_PRJ + ":"+DTL_L0_BQ_DATASET+".purchases",
+ "inputFilePattern": DRP_GCS + "/purchases.csv",
+ "outputTable": DWH_LAND_PRJ + ":" + DWH_LAND_BQ_DATASET + ".purchases",
"bigQueryLoadingTemporaryDirectory": LOD_GCS_STAGING + "/tmp/bq/",
},
)
@@ -275,12 +275,12 @@ with models.DAG(
p.item as item,
p.price as price,
p.timestamp as timestamp
- FROM `{dtl_0_prj}.{dtl_0_dataset}.customers` c
- JOIN `{dtl_0_prj}.{dtl_0_dataset}.purchases` p ON c.id = p.customer_id
- """.format(dtl_0_prj=DTL_L0_PRJ, dtl_0_dataset=DTL_L0_BQ_DATASET, ),
+ FROM `{dwh_0_prj}.{dwh_0_dataset}.customers` c
+ JOIN `{dwh_0_prj}.{dwh_0_dataset}.purchases` p ON c.id = p.customer_id
+ """.format(dwh_0_prj=DWH_LAND_PRJ, dwh_0_dataset=DWH_LAND_BQ_DATASET, ),
'destinationTable':{
- 'projectId': DTL_L1_PRJ,
- 'datasetId': DTL_L1_BQ_DATASET,
+ 'projectId': DWH_CURATED_PRJ,
+ 'datasetId': DWH_CURATED_BQ_DATASET,
'tableId': 'customer_purchase'
},
'writeDisposition':'WRITE_TRUNCATE',
@@ -290,8 +290,8 @@ with models.DAG(
impersonation_chain=[TRF_SA_BQ]
)
- l2_customer_purchase = BigQueryInsertJobOperator(
- task_id='bq_l2_customer_purchase',
+ confidential_customer_purchase = BigQueryInsertJobOperator(
+ task_id='bq_confidential_customer_purchase',
gcp_conn_id='bigquery_default',
project_id=TRF_PRJ,
location=BQ_LOCATION,
@@ -306,11 +306,11 @@ with models.DAG(
item,
price,
timestamp
- FROM `{dtl_1_prj}.{dtl_1_dataset}.customer_purchase`
- """.format(dtl_1_prj=DTL_L1_PRJ, dtl_1_dataset=DTL_L1_BQ_DATASET, ),
+ FROM `{dwh_cur_prj}.{dwh_cur_dataset}.customer_purchase`
+ """.format(dwh_cur_prj=DWH_CURATED_PRJ, dwh_cur_dataset=DWH_CURATED_BQ_DATASET, ),
'destinationTable':{
- 'projectId': DTL_L2_PRJ,
- 'datasetId': DTL_L2_BQ_DATASET,
+ 'projectId': DWH_CONFIDENTIAL_PRJ,
+ 'datasetId': DWH_CONFIDENTIAL_BQ_DATASET,
'tableId': 'customer_purchase'
},
'writeDisposition':'WRITE_TRUNCATE',
@@ -319,4 +319,4 @@ with models.DAG(
},
impersonation_chain=[TRF_SA_BQ]
)
- start >> upsert_table >> update_schema_table >> [customers_import, purchases_import] >> join_customer_purchase >> l2_customer_purchase >> end
+ start >> upsert_table >> update_schema_table >> [customers_import, purchases_import] >> join_customer_purchase >> confidential_customer_purchase >> end
diff --git a/examples/data-solutions/data-platform-foundations/demo/delete_table.py b/examples/data-solutions/data-platform-foundations/demo/delete_table.py
index a2585a68b..dc0c954b1 100644
--- a/examples/data-solutions/data-platform-foundations/demo/delete_table.py
+++ b/examples/data-solutions/data-platform-foundations/demo/delete_table.py
@@ -34,23 +34,23 @@ from airflow.utils.task_group import TaskGroup
# --------------------------------------------------------------------------------
BQ_LOCATION = os.environ.get("BQ_LOCATION")
DATA_CAT_TAGS = json.loads(os.environ.get("DATA_CAT_TAGS"))
-DTL_L0_PRJ = os.environ.get("DTL_L0_PRJ")
-DTL_L0_BQ_DATASET = os.environ.get("DTL_L0_BQ_DATASET")
-DTL_L0_GCS = os.environ.get("DTL_L0_GCS")
-DTL_L1_PRJ = os.environ.get("DTL_L1_PRJ")
-DTL_L1_BQ_DATASET = os.environ.get("DTL_L1_BQ_DATASET")
-DTL_L1_GCS = os.environ.get("DTL_L1_GCS")
-DTL_L2_PRJ = os.environ.get("DTL_L2_PRJ")
-DTL_L2_BQ_DATASET = os.environ.get("DTL_L2_BQ_DATASET")
-DTL_L2_GCS = os.environ.get("DTL_L2_GCS")
-DTL_PLG_PRJ = os.environ.get("DTL_PLG_PRJ")
-DTL_PLG_BQ_DATASET = os.environ.get("DTL_PLG_BQ_DATASET")
-DTL_PLG_GCS = os.environ.get("DTL_PLG_GCS")
+DWH_LAND_PRJ = os.environ.get("DWH_LAND_PRJ")
+DWH_LAND_BQ_DATASET = os.environ.get("DWH_LAND_BQ_DATASET")
+DWH_LAND_GCS = os.environ.get("DWH_LAND_GCS")
+DWH_CURATED_PRJ = os.environ.get("DWH_CURATED_PRJ")
+DWH_CURATED_BQ_DATASET = os.environ.get("DWH_CURATED_BQ_DATASET")
+DWH_CURATED_GCS = os.environ.get("DWH_CURATED_GCS")
+DWH_CONFIDENTIAL_PRJ = os.environ.get("DWH_CONFIDENTIAL_PRJ")
+DWH_CONFIDENTIAL_BQ_DATASET = os.environ.get("DWH_CONFIDENTIAL_BQ_DATASET")
+DWH_CONFIDENTIAL_GCS = os.environ.get("DWH_CONFIDENTIAL_GCS")
+DWH_PLG_PRJ = os.environ.get("DWH_PLG_PRJ")
+DWH_PLG_BQ_DATASET = os.environ.get("DWH_PLG_BQ_DATASET")
+DWH_PLG_GCS = os.environ.get("DWH_PLG_GCS")
GCP_REGION = os.environ.get("GCP_REGION")
-LND_PRJ = os.environ.get("LND_PRJ")
-LND_BQ = os.environ.get("LND_BQ")
-LND_GCS = os.environ.get("LND_GCS")
-LND_PS = os.environ.get("LND_PS")
+DRP_PRJ = os.environ.get("DRP_PRJ")
+DRP_BQ = os.environ.get("DRP_BQ")
+DRP_GCS = os.environ.get("DRP_GCS")
+DRP_PS = os.environ.get("DRP_PS")
LOD_PRJ = os.environ.get("LOD_PRJ")
LOD_GCS_STAGING = os.environ.get("LOD_GCS_STAGING")
LOD_NET_VPC = os.environ.get("LOD_NET_VPC")
@@ -121,25 +121,25 @@ with models.DAG(
with TaskGroup('delete_table') as delte_table:
delete_table_customers = BigQueryDeleteTableOperator(
task_id="delete_table_customers",
- deletion_dataset_table=DTL_L0_PRJ+"."+DTL_L0_BQ_DATASET+".customers",
+ deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".customers",
impersonation_chain=[TRF_SA_DF]
)
delete_table_purchases = BigQueryDeleteTableOperator(
task_id="delete_table_purchases",
- deletion_dataset_table=DTL_L0_PRJ+"."+DTL_L0_BQ_DATASET+".purchases",
+ deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".purchases",
impersonation_chain=[TRF_SA_DF]
)
- delete_table_customer_purchase_l1 = BigQueryDeleteTableOperator(
- task_id="delete_table_customer_purchase_l1",
- deletion_dataset_table=DTL_L1_PRJ+"."+DTL_L1_BQ_DATASET+".customer_purchase",
+ delete_table_customer_purchase_curated = BigQueryDeleteTableOperator(
+ task_id="delete_table_customer_purchase_curated",
+ deletion_dataset_table=DWH_CURATED_PRJ+"."+DWH_CURATED_BQ_DATASET+".customer_purchase",
impersonation_chain=[TRF_SA_DF]
)
- delete_table_customer_purchase_l2 = BigQueryDeleteTableOperator(
- task_id="delete_table_customer_purchase_l2",
- deletion_dataset_table=DTL_L2_PRJ+"."+DTL_L2_BQ_DATASET+".customer_purchase",
+ delete_table_customer_purchase_confidential = BigQueryDeleteTableOperator(
+ task_id="delete_table_customer_purchase_confidential",
+ deletion_dataset_table=DWH_CONFIDENTIAL_PRJ+"."+DWH_CONFIDENTIAL_BQ_DATASET+".customer_purchase",
impersonation_chain=[TRF_SA_DF]
)
diff --git a/examples/data-solutions/data-platform-foundations/images/overview_diagram.png b/examples/data-solutions/data-platform-foundations/images/overview_diagram.png
index b1a0f7887..642c81c2f 100644
Binary files a/examples/data-solutions/data-platform-foundations/images/overview_diagram.png and b/examples/data-solutions/data-platform-foundations/images/overview_diagram.png differ
diff --git a/examples/data-solutions/data-platform-foundations/outputs.tf b/examples/data-solutions/data-platform-foundations/outputs.tf
index 32e98fc60..3fd81a0ea 100644
--- a/examples/data-solutions/data-platform-foundations/outputs.tf
+++ b/examples/data-solutions/data-platform-foundations/outputs.tf
@@ -17,25 +17,25 @@
output "bigquery-datasets" {
description = "BigQuery datasets."
value = {
- land-bq-0 = module.land-bq-0.dataset_id,
- lake-0-bq-0 = module.lake-0-bq-0.dataset_id,
- lake-1-bq-0 = module.lake-1-bq-0.dataset_id,
- lake-2-bq-0 = module.lake-2-bq-0.dataset_id,
- lake-plg-bq-0 = module.lake-plg-bq-0.dataset_id,
+ drop-bq-0 = module.drop-bq-0.dataset_id,
+ dwh-landing-bq-0 = module.dwh-lnd-bq-0.dataset_id,
+ dwh-curated-bq-0 = module.dwh-cur-bq-0.dataset_id,
+ dwh-confidential-bq-0 = module.dwh-conf-bq-0.dataset_id,
+ dwh-plg-bq-0 = module.dwh-plg-bq-0.dataset_id,
}
}
output "gcs-buckets" {
description = "GCS buckets."
value = {
- lake-0-cs-0 = module.lake-0-cs-0.name,
- lake-1-cs-0 = module.lake-1-cs-0.name,
- lake-2-cs-0 = module.lake-2-cs-0.name,
- lake-plg-cs-0 = module.lake-plg-cs-0.name,
- land-cs-0 = module.land-cs-0.name,
- lod-cs-df = module.load-cs-df-0.name,
- orch-cs-0 = module.orch-cs-0.name,
- transf-cs-df = module.transf-cs-df-0.name,
+ dwh-landing-cs-0 = module.dwh-lnd-cs-0.name,
+ dwh-curated-cs-0 = module.dwh-cur-cs-0.name,
+ dwh-confidential-cs-0 = module.dwh-conf-cs-0.name,
+ dwh-plg-cs-0 = module.dwh-plg-cs-0.name,
+ drop-cs-0 = module.drop-cs-0.name,
+ lod-cs-df = module.load-cs-df-0.name,
+ orch-cs-0 = module.orch-cs-0.name,
+ transf-cs-df = module.transf-cs-df-0.name,
}
}
@@ -48,26 +48,26 @@ output "projects" {
description = "GCP Projects informations."
value = {
project_number = {
- lake-0 = module.lake-0-project.number,
- lake-1 = module.lake-1-project.number,
- lake-2 = module.lake-2-project.number,
- lake-plg = module.lake-plg-project.number,
- exposure = module.exp-project.number,
- landing = module.land-project.number,
- load = module.load-project.number,
- orchestration = module.orch-project.number,
- transformation = module.transf-project.number,
+ dwh-landing = module.dwh-lnd-project.number,
+ dwh-curated = module.dwh-cur-project.number,
+ dwh-confidential = module.dwh-conf-project.number,
+ dwh-plg = module.dwh-plg-project.number,
+ exposure = module.exp-project.number,
+ dropoff = module.drop-project.number,
+ load = module.load-project.number,
+ orchestration = module.orch-project.number,
+ transformation = module.transf-project.number,
}
project_id = {
- lake-0 = module.lake-0-project.project_id,
- lake-1 = module.lake-1-project.project_id,
- lake-2 = module.lake-2-project.project_id,
- lake-plg = module.lake-plg-project.project_id,
- exposure = module.exp-project.project_id,
- landing = module.land-project.project_id,
- load = module.load-project.project_id,
- orchestration = module.orch-project.project_id,
- transformation = module.transf-project.project_id,
+ dwh-landing = module.dwh-lnd-project.project_id,
+ dwh-curated = module.dwh-cur-project.project_id,
+ dwh-confidential = module.dwh-conf-project.project_id,
+ dwh-plg = module.dwh-plg-project.project_id,
+ exposure = module.exp-project.project_id,
+ dropoff = module.drop-project.project_id,
+ load = module.load-project.project_id,
+ orchestration = module.orch-project.project_id,
+ transformation = module.transf-project.project_id,
}
}
}
@@ -93,12 +93,12 @@ output "vpc_subnet" {
output "demo_commands" {
description = "Demo commands."
value = {
- 01 = "gsutil -i ${module.land-sa-cs-0.email} cp demo/data/*.csv gs://${module.land-cs-0.name}"
+ 01 = "gsutil -i ${module.drop-sa-cs-0.email} cp demo/data/*.csv gs://${module.drop-cs-0.name}"
02 = "gsutil -i ${module.orch-sa-cmp-0.email} cp demo/data/*.j* gs://${module.orch-cs-0.name}"
03 = "gsutil -i ${module.orch-sa-cmp-0.email} cp demo/*.py ${google_composer_environment.orch-cmp-0.config[0].dag_gcs_prefix}/"
04 = "Open ${google_composer_environment.orch-cmp-0.config.0.airflow_uri} and run uploaded DAG."
05 = <"
-gcloud projects create $MY_PROJECT_ID
-gcloud alpha billing projects link --billing-account=XXXXXX-XXXXXX-XXXXXX $MY_PROJECT_ID
-gcloud services enable --project=$MY_PROJECT_ID {compute,dns}.googleapis.com
+The provided project needs a valid billing account, the Compute and DNS APIs are enabled by the example.
+
+You can easily create such a project by commenting turning on project creation in the project module contained in `main.tf`, as shown in this snippet:
+
+```hcl
+module "project" {
+ source = "../../..//modules/project"
+ name = var.project_id
+ # comment or remove this line to enable project creation
+ # project_create = false
+ # add the following line with your billing account id value
+ billing_account = "12345-ABCD-12345"
+ services = [
+ "compute.googleapis.com",
+ "dns.googleapis.com"
+ ]
+ service_config = {
+ disable_on_destroy = false
+ disable_dependent_services = false
+ }
+}
```
-The example does not account for HA, but the VPN gateways can be easily upgraded to use HA VPN via the [net-vpn-ha module](../../../modules/net-vpn-ha).
+## Testing
-If a single router and VPN gateway are used in the hub to manage all tunnels, particular care must be taken in announcing ranges from hub to spokes, as Cloud Router does not explicitly support transitivity and overlapping routes received from both sides create unintended side effects. The simple workaround is to announce a single aggregated route from hub to spokes so that it does not overlap with any of the ranges advertised by each spoke to the hub.
+Once the example is up, you can quickly test features by logging in to one of the test VMs:
+
+```bash
+gcloud compute ssh hs-ha-lnd-test-r1
+# test DNS resolution of the landing zone
+ping test-r1.example.com
+# test DNS resolution of the prod zone, and prod reachability
+ping test-r1.prod.example.com
+# test DNS resolution of the dev zone, and dev reachability via global routing
+ping test-r2.dev.example.com
+```
+
+
+## Files
+
+| name | description | modules |
+|---|---|---|
+| [main.tf](./main.tf) | Module-level locals and resources. | compute-vm · project |
+| [net-dev.tf](./net-dev.tf) | Development spoke VPC. | dns · net-vpc · net-vpc-firewall |
+| [net-landing.tf](./net-landing.tf) | Landing hub VPC. | dns · net-vpc · net-vpc-firewall |
+| [net-prod.tf](./net-prod.tf) | Production spoke VPC. | dns · net-vpc · net-vpc-firewall |
+| [outputs.tf](./outputs.tf) | Module outputs. | |
+| [variables.tf](./variables.tf) | Module variables. | |
+| [versions.tf](./versions.tf) | Version pins. | |
+| [vpn-dev-r1.tf](./vpn-dev-r1.tf) | Landing to Development VPN for region 1. | net-vpn-ha |
+| [vpn-prod-r1.tf](./vpn-prod-r1.tf) | Landing to Production VPN for region 1. | net-vpn-ha |
+
## Variables
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
-| [project_id](variables.tf#L56) | Project id for all resources. | string | ✓ | |
-| [bgp_asn](variables.tf#L15) | BGP ASNs. | map(number) | | {…} |
-| [bgp_custom_advertisements](variables.tf#L25) | BGP custom advertisement IP CIDR ranges. | map(string) | | {…} |
-| [bgp_interface_ranges](variables.tf#L34) | BGP interface IP CIDR ranges. | map(string) | | {…} |
-| [ip_ranges](variables.tf#L43) | IP CIDR ranges. | map(string) | | {…} |
-| [regions](variables.tf#L61) | VPC regions. | map(string) | | {…} |
+| [project_id](variables.tf#L49) | Project id for all resources. | string | ✓ | |
+| [ip_ranges](variables.tf#L15) | Subnet IP CIDR ranges. | map(string) | | {…} |
+| [ip_secondary_ranges](variables.tf#L28) | Subnet secondary ranges. | map(map(string)) | | {} |
+| [prefix](variables.tf#L34) | Prefix used in resource names. | string | | null |
+| [project_create_config](variables.tf#L40) | Populate with billing account id to trigger project creation. | object({…}) | | null |
+| [regions](variables.tf#L54) | VPC regions. | map(string) | | {…} |
+| [vpn_configs](variables.tf#L63) | VPN configurations. | map(object({…})) | | {…} |
## Outputs
| name | description | sensitive |
|---|---|:---:|
-| [vms](outputs.tf#L15) | GCE VMs. | |
+| [subnets](outputs.tf#L15) | Subnet details. | |
+| [vms](outputs.tf#L39) | GCE VMs. | |
diff --git a/examples/networking/hub-and-spoke-vpn/diagram.png b/examples/networking/hub-and-spoke-vpn/diagram.png
index 6d5dc5ff2..6bddab62e 100644
Binary files a/examples/networking/hub-and-spoke-vpn/diagram.png and b/examples/networking/hub-and-spoke-vpn/diagram.png differ
diff --git a/examples/networking/hub-and-spoke-vpn/main.tf b/examples/networking/hub-and-spoke-vpn/main.tf
index 3eec0ae6e..f448ca798 100644
--- a/examples/networking/hub-and-spoke-vpn/main.tf
+++ b/examples/networking/hub-and-spoke-vpn/main.tf
@@ -13,297 +13,71 @@
# limitations under the License.
locals {
- vm-startup-script = join("\n", [
- "#! /bin/bash",
- "apt-get update && apt-get install -y dnsutils"
- ])
+ prefix = var.prefix == null ? "" : "${var.prefix}-"
}
-################################################################################
-# Hub networking #
-################################################################################
+# enable services in the project used
-module "vpc-hub" {
- source = "../../../modules/net-vpc"
- project_id = var.project_id
- name = "hub"
- subnets = [
- {
- ip_cidr_range = var.ip_ranges.hub-a
- name = "hub-a"
- region = var.regions.a
- secondary_ip_range = {}
- },
- {
- ip_cidr_range = var.ip_ranges.hub-b
- name = "hub-b"
- region = var.regions.b
- secondary_ip_range = {}
- }
+module "project" {
+ source = "../../..//modules/project"
+ name = var.project_id
+ parent = try(var.project_create_config.parent, null)
+ billing_account = try(var.project_create_config.billing_account_id, null)
+ project_create = try(var.project_create_config.billing_account_id, null) != null
+ services = [
+ "compute.googleapis.com",
+ "dns.googleapis.com"
]
-}
-
-module "vpc-hub-firewall" {
- source = "../../../modules/net-vpc-firewall"
- project_id = var.project_id
- network = module.vpc-hub.name
- admin_ranges = values(var.ip_ranges)
-}
-
-module "vpn-hub-a" {
- source = "../../../modules/net-vpn-dynamic"
- project_id = var.project_id
- region = var.regions.a
- network = module.vpc-hub.name
- name = "hub-a"
- router_asn = var.bgp_asn.hub
- tunnels = {
- spoke-1 = {
- bgp_peer = {
- address = cidrhost(var.bgp_interface_ranges.spoke-1, 2)
- asn = var.bgp_asn.spoke-1
- }
- bgp_peer_options = {
- advertise_groups = ["ALL_SUBNETS"]
- advertise_ip_ranges = {
- (var.bgp_custom_advertisements.hub-to-spoke-1) = "spoke-2"
- }
- advertise_mode = "CUSTOM"
- route_priority = 1000
- }
- bgp_session_range = "${cidrhost(var.bgp_interface_ranges.spoke-1, 1)}/30"
- ike_version = 2
- peer_ip = module.vpn-spoke-1.address
- router = null
- shared_secret = ""
- }
+ service_config = {
+ disable_on_destroy = false
+ disable_dependent_services = false
}
}
-module "vpn-hub-b" {
- source = "../../../modules/net-vpn-dynamic"
- project_id = var.project_id
- region = var.regions.b
- network = module.vpc-hub.name
- name = "hub-b"
- router_asn = var.bgp_asn.hub
- tunnels = {
- spoke-2 = {
- bgp_peer = {
- address = cidrhost(var.bgp_interface_ranges.spoke-2, 2)
- asn = var.bgp_asn.spoke-2
- }
- bgp_peer_options = {
- advertise_groups = ["ALL_SUBNETS"]
- advertise_ip_ranges = {
- (var.bgp_custom_advertisements.hub-to-spoke-2) = "spoke-1"
- }
- advertise_mode = "CUSTOM"
- route_priority = 1000
- }
- bgp_session_range = "${cidrhost(var.bgp_interface_ranges.spoke-2, 1)}/30"
- ike_version = 2
- peer_ip = module.vpn-spoke-2.address
- router = null
- shared_secret = ""
- }
- }
-}
+# test VM in landing region 1
-################################################################################
-# Spoke 1 networking #
-################################################################################
-
-module "vpc-spoke-1" {
- source = "../../../modules/net-vpc"
- project_id = var.project_id
- name = "spoke-1"
- subnets = [
- {
- ip_cidr_range = var.ip_ranges.spoke-1-a
- name = "spoke-1-a"
- region = var.regions.a
- secondary_ip_range = {}
- },
- {
- ip_cidr_range = var.ip_ranges.spoke-1-b
- name = "spoke-1-b"
- region = var.regions.b
- secondary_ip_range = {}
- }
- ]
-}
-
-module "vpc-spoke-1-firewall" {
- source = "../../../modules/net-vpc-firewall"
- project_id = var.project_id
- network = module.vpc-spoke-1.name
- admin_ranges = values(var.ip_ranges)
-}
-
-module "vpn-spoke-1" {
- source = "../../../modules/net-vpn-dynamic"
- project_id = var.project_id
- region = var.regions.a
- network = module.vpc-spoke-1.name
- name = "spoke-1"
- router_asn = var.bgp_asn.spoke-1
- tunnels = {
- hub = {
- bgp_peer = {
- address = cidrhost(var.bgp_interface_ranges.spoke-1, 1)
- asn = var.bgp_asn.hub
- }
- bgp_peer_options = null
- bgp_session_range = "${cidrhost(var.bgp_interface_ranges.spoke-1, 2)}/30"
- ike_version = 2
- peer_ip = module.vpn-hub-a.address
- router = null
- shared_secret = module.vpn-hub-a.random_secret
- }
- }
-}
-
-module "nat-spoke-1" {
- source = "../../../modules/net-cloudnat"
- project_id = var.project_id
- region = var.regions.a
- name = "spoke-1"
- router_create = false
- router_name = module.vpn-spoke-1.router_name
-}
-
-################################################################################
-# Spoke 2 networking #
-################################################################################
-
-module "vpc-spoke-2" {
- source = "../../../modules/net-vpc"
- project_id = var.project_id
- name = "spoke-2"
- subnets = [
- {
- ip_cidr_range = var.ip_ranges.spoke-2-a
- name = "spoke-2-a"
- region = var.regions.a
- secondary_ip_range = {}
- },
- {
- ip_cidr_range = var.ip_ranges.spoke-2-b
- name = "spoke-2-b"
- region = var.regions.b
- secondary_ip_range = {}
- }
- ]
-}
-
-module "vpc-spoke-2-firewall" {
- source = "../../../modules/net-vpc-firewall"
- project_id = var.project_id
- network = module.vpc-spoke-2.name
- admin_ranges = values(var.ip_ranges)
-}
-
-module "vpn-spoke-2" {
- source = "../../../modules/net-vpn-dynamic"
- project_id = var.project_id
- region = var.regions.a
- network = module.vpc-spoke-2.name
- name = "spoke-2"
- router_asn = var.bgp_asn.spoke-2
- tunnels = {
- hub = {
- bgp_peer = {
- address = cidrhost(var.bgp_interface_ranges.spoke-2, 1)
- asn = var.bgp_asn.hub
- }
- bgp_peer_options = null
- bgp_session_range = "${cidrhost(var.bgp_interface_ranges.spoke-2, 2)}/30"
- ike_version = 2
- peer_ip = module.vpn-hub-b.address
- router = null
- shared_secret = module.vpn-hub-b.random_secret
- }
- }
-}
-
-module "nat-spoke-2" {
- source = "../../../modules/net-cloudnat"
- project_id = var.project_id
- region = var.regions.a
- name = "spoke-2"
- router_create = false
- router_name = module.vpn-spoke-2.router_name
-}
-
-################################################################################
-# Test VMs #
-################################################################################
-
-module "vm-spoke-1" {
+module "landing-r1-vm" {
source = "../../../modules/compute-vm"
project_id = var.project_id
- zone = "${var.regions.b}-b"
- name = "spoke-1-test"
+ name = "${local.prefix}lnd-test-r1"
+ zone = "${var.regions.r1}-b"
network_interfaces = [{
- network = module.vpc-spoke-1.self_link
- subnetwork = module.vpc-spoke-1.subnet_self_links["${var.regions.b}/spoke-1-b"]
+ network = module.landing-vpc.self_link
+ subnetwork = module.landing-vpc.subnet_self_links["${var.regions.r1}/${local.prefix}lnd-0"]
nat = false
addresses = null
}]
- tags = ["ssh"]
- metadata = { startup-script = local.vm-startup-script }
+ tags = ["ssh"]
}
-module "vm-spoke-2" {
+# test VM in prod region 1
+
+module "prod-r1-vm" {
source = "../../../modules/compute-vm"
project_id = var.project_id
- zone = "${var.regions.b}-b"
- name = "spoke-2-test"
+ name = "${local.prefix}prd-test-r1"
+ zone = "${var.regions.r1}-b"
network_interfaces = [{
- network = module.vpc-spoke-2.self_link
- subnetwork = module.vpc-spoke-2.subnet_self_links["${var.regions.b}/spoke-2-b"]
+ network = module.prod-vpc.self_link
+ subnetwork = module.prod-vpc.subnet_self_links["${var.regions.r1}/${local.prefix}prd-0"]
nat = false
addresses = null
}]
- tags = ["ssh"]
- metadata = { startup-script = local.vm-startup-script }
+ tags = ["ssh"]
}
-################################################################################
-# DNS zones #
-################################################################################
+# test VM in dev region 1
-module "dns-host" {
- source = "../../../modules/dns"
- project_id = var.project_id
- type = "private"
- name = "example"
- domain = "example.com."
- client_networks = [module.vpc-hub.self_link]
- recordsets = {
- "A localhost" = { ttl = 300, records = ["127.0.0.1"] }
- "A spoke-1-test" = { ttl = 300, records = [module.vm-spoke-1.internal_ip] }
- "A spoke-2-test" = { ttl = 300, records = [module.vm-spoke-2.internal_ip] }
- }
-}
-
-module "dns-spoke-1" {
- source = "../../../modules/dns"
- project_id = var.project_id
- type = "peering"
- name = "spoke-1"
- domain = "example.com."
- client_networks = [module.vpc-spoke-1.self_link]
- peer_network = module.vpc-hub.self_link
-}
-
-module "dns-spoke-2" {
- source = "../../../modules/dns"
- project_id = var.project_id
- type = "peering"
- name = "spoke-2"
- domain = "example.com."
- client_networks = [module.vpc-spoke-2.self_link]
- peer_network = module.vpc-hub.self_link
+module "dev-r2-vm" {
+ source = "../../../modules/compute-vm"
+ project_id = var.project_id
+ name = "${local.prefix}dev-test-r2"
+ zone = "${var.regions.r2}-b"
+ network_interfaces = [{
+ network = module.dev-vpc.self_link
+ subnetwork = module.dev-vpc.subnet_self_links["${var.regions.r2}/${local.prefix}dev-0"]
+ nat = false
+ addresses = null
+ }]
+ tags = ["ssh"]
}
diff --git a/examples/networking/hub-and-spoke-vpn/net-dev.tf b/examples/networking/hub-and-spoke-vpn/net-dev.tf
new file mode 100644
index 000000000..eedca0f75
--- /dev/null
+++ b/examples/networking/hub-and-spoke-vpn/net-dev.tf
@@ -0,0 +1,69 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# tfdoc:file:description Development spoke VPC.
+
+module "dev-vpc" {
+ source = "../../../modules/net-vpc"
+ project_id = var.project_id
+ name = "${local.prefix}dev"
+ subnets = [
+ {
+ ip_cidr_range = var.ip_ranges.dev-0-r1
+ name = "${local.prefix}dev-0"
+ region = var.regions.r1
+ secondary_ip_range = try(
+ var.ip_secondary_ranges.dev-0-r1, {}
+ )
+ },
+ {
+ ip_cidr_range = var.ip_ranges.dev-0-r2
+ name = "${local.prefix}dev-0"
+ region = var.regions.r2
+ secondary_ip_range = try(
+ var.ip_secondary_ranges.dev-0-r2, {}
+ )
+ }
+ ]
+}
+
+module "dev-firewall" {
+ source = "../../../modules/net-vpc-firewall"
+ project_id = var.project_id
+ network = module.dev-vpc.name
+ admin_ranges = values(var.ip_ranges)
+}
+
+module "dev-dns-peering" {
+ source = "../../../modules/dns"
+ project_id = var.project_id
+ type = "peering"
+ name = "${local.prefix}example-com-dev-peering"
+ domain = "example.com."
+ client_networks = [module.dev-vpc.self_link]
+ peer_network = module.landing-vpc.self_link
+}
+
+module "dev-dns-zone" {
+ source = "../../../modules/dns"
+ project_id = var.project_id
+ type = "private"
+ name = "${local.prefix}dev-example-com"
+ domain = "dev.example.com."
+ client_networks = [module.landing-vpc.self_link]
+ recordsets = {
+ "A localhost" = { ttl = 300, records = ["127.0.0.1"] }
+ "A test-r2" = { ttl = 300, records = [module.dev-r2-vm.internal_ip] }
+ }
+}
diff --git a/examples/networking/hub-and-spoke-vpn/net-landing.tf b/examples/networking/hub-and-spoke-vpn/net-landing.tf
new file mode 100644
index 000000000..baaea4bc6
--- /dev/null
+++ b/examples/networking/hub-and-spoke-vpn/net-landing.tf
@@ -0,0 +1,59 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# tfdoc:file:description Landing hub VPC.
+
+module "landing-vpc" {
+ source = "../../../modules/net-vpc"
+ project_id = var.project_id
+ name = "${local.prefix}lnd"
+ subnets = [
+ {
+ ip_cidr_range = var.ip_ranges.land-0-r1
+ name = "${local.prefix}lnd-0"
+ region = var.regions.r1
+ secondary_ip_range = try(
+ var.ip_secondary_ranges.land-0-r1, {}
+ )
+ },
+ {
+ ip_cidr_range = var.ip_ranges.land-0-r2
+ name = "${local.prefix}lnd-0"
+ region = var.regions.r2
+ secondary_ip_range = try(
+ var.ip_secondary_ranges.land-0-r2, {}
+ )
+ }
+ ]
+}
+
+module "landing-firewall" {
+ source = "../../../modules/net-vpc-firewall"
+ project_id = var.project_id
+ network = module.landing-vpc.name
+ admin_ranges = values(var.ip_ranges)
+}
+
+module "landing-dns-zone" {
+ source = "../../../modules/dns"
+ project_id = var.project_id
+ type = "private"
+ name = "${local.prefix}example-com"
+ domain = "example.com."
+ client_networks = [module.landing-vpc.self_link]
+ recordsets = {
+ "A localhost" = { ttl = 300, records = ["127.0.0.1"] }
+ "A test-r1" = { ttl = 300, records = [module.landing-r1-vm.internal_ip] }
+ }
+}
diff --git a/examples/networking/hub-and-spoke-vpn/net-prod.tf b/examples/networking/hub-and-spoke-vpn/net-prod.tf
new file mode 100644
index 000000000..c058537d2
--- /dev/null
+++ b/examples/networking/hub-and-spoke-vpn/net-prod.tf
@@ -0,0 +1,69 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# tfdoc:file:description Production spoke VPC.
+
+module "prod-vpc" {
+ source = "../../../modules/net-vpc"
+ project_id = var.project_id
+ name = "${local.prefix}prd"
+ subnets = [
+ {
+ ip_cidr_range = var.ip_ranges.prod-0-r1
+ name = "${local.prefix}prd-0"
+ region = var.regions.r1
+ secondary_ip_range = try(
+ var.ip_secondary_ranges.prod-0-r1, {}
+ )
+ },
+ {
+ ip_cidr_range = var.ip_ranges.prod-0-r2
+ name = "${local.prefix}prd-0"
+ region = var.regions.r2
+ secondary_ip_range = try(
+ var.ip_secondary_ranges.prod-0-r2, {}
+ )
+ }
+ ]
+}
+
+module "prod-firewall" {
+ source = "../../../modules/net-vpc-firewall"
+ project_id = var.project_id
+ network = module.prod-vpc.name
+ admin_ranges = values(var.ip_ranges)
+}
+
+module "prod-dns-peering" {
+ source = "../../../modules/dns"
+ project_id = var.project_id
+ type = "peering"
+ name = "${local.prefix}example-com-prd-peering"
+ domain = "example.com."
+ client_networks = [module.prod-vpc.self_link]
+ peer_network = module.landing-vpc.self_link
+}
+
+module "prod-dns-zone" {
+ source = "../../../modules/dns"
+ project_id = var.project_id
+ type = "private"
+ name = "${local.prefix}prd-example-com"
+ domain = "prd.example.com."
+ client_networks = [module.landing-vpc.self_link]
+ recordsets = {
+ "A localhost" = { ttl = 300, records = ["127.0.0.1"] }
+ "A test-r1" = { ttl = 300, records = [module.prod-r1-vm.internal_ip] }
+ }
+}
diff --git a/examples/networking/hub-and-spoke-vpn/outputs.tf b/examples/networking/hub-and-spoke-vpn/outputs.tf
index f69ca1096..befd20ff8 100644
--- a/examples/networking/hub-and-spoke-vpn/outputs.tf
+++ b/examples/networking/hub-and-spoke-vpn/outputs.tf
@@ -12,10 +12,34 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+output "subnets" {
+ description = "Subnet details."
+ value = {
+ dev = {
+ for k, v in module.dev-vpc.subnets : k => {
+ id = v.id
+ ip_cidr_range = v.ip_cidr_range
+ }
+ }
+ landing = {
+ for k, v in module.landing-vpc.subnets : k => {
+ id = v.id
+ ip_cidr_range = v.ip_cidr_range
+ }
+ }
+ prod = {
+ for k, v in module.prod-vpc.subnets : k => {
+ id = v.id
+ ip_cidr_range = v.ip_cidr_range
+ }
+ }
+ }
+}
+
output "vms" {
description = "GCE VMs."
value = {
- for instance in [module.vm-spoke-1.instance, module.vm-spoke-2.instance] :
- instance.name => instance.network_interface.0.network_ip
+ for mod in [module.landing-r1-vm, module.dev-r2-vm, module.prod-r1-vm] :
+ mod.instance.name => mod.internal_ip
}
}
diff --git a/examples/networking/hub-and-spoke-vpn/provider.tf b/examples/networking/hub-and-spoke-vpn/provider.tf
deleted file mode 100644
index b36fb5ee5..000000000
--- a/examples/networking/hub-and-spoke-vpn/provider.tf
+++ /dev/null
@@ -1,18 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-provider "google" {
-}
-provider "google-beta" {
-}
diff --git a/examples/networking/hub-and-spoke-vpn/variables.tf b/examples/networking/hub-and-spoke-vpn/variables.tf
index f30bebc5a..98286e8ef 100644
--- a/examples/networking/hub-and-spoke-vpn/variables.tf
+++ b/examples/networking/hub-and-spoke-vpn/variables.tf
@@ -12,47 +12,40 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-variable "bgp_asn" {
- description = "BGP ASNs."
- type = map(number)
- default = {
- hub = 64513
- spoke-1 = 64514
- spoke-2 = 64515
- }
-}
-
-variable "bgp_custom_advertisements" {
- description = "BGP custom advertisement IP CIDR ranges."
- type = map(string)
- default = {
- hub-to-spoke-1 = "10.0.32.0/20"
- hub-to-spoke-2 = "10.0.16.0/20"
- }
-}
-
-variable "bgp_interface_ranges" {
- description = "BGP interface IP CIDR ranges."
- type = map(string)
- default = {
- spoke-1 = "169.254.1.0/30"
- spoke-2 = "169.254.1.4/30"
- }
-}
-
variable "ip_ranges" {
- description = "IP CIDR ranges."
+ description = "Subnet IP CIDR ranges."
type = map(string)
default = {
- hub-a = "10.0.0.0/24"
- hub-b = "10.0.8.0/24"
- spoke-1-a = "10.0.16.0/24"
- spoke-1-b = "10.0.24.0/24"
- spoke-2-a = "10.0.32.0/24"
- spoke-2-b = "10.0.40.0/24"
+ land-0-r1 = "10.0.0.0/24"
+ land-0-r2 = "10.0.8.0/24"
+ dev-0-r1 = "10.0.16.0/24"
+ dev-0-r2 = "10.0.24.0/24"
+ prod-0-r1 = "10.0.32.0/24"
+ prod-0-r2 = "10.0.40.0/24"
}
}
+variable "ip_secondary_ranges" {
+ description = "Subnet secondary ranges."
+ type = map(map(string))
+ default = {}
+}
+
+variable "prefix" {
+ description = "Prefix used in resource names."
+ type = string
+ default = null
+}
+
+variable "project_create_config" {
+ description = "Populate with billing account id to trigger project creation."
+ type = object({
+ billing_account_id = string
+ parent_id = string
+ })
+ default = null
+}
+
variable "project_id" {
description = "Project id for all resources."
type = string
@@ -62,7 +55,31 @@ variable "regions" {
description = "VPC regions."
type = map(string)
default = {
- a = "europe-west1"
- b = "europe-west2"
+ r1 = "europe-west1"
+ r2 = "europe-west4"
+ }
+}
+
+variable "vpn_configs" {
+ description = "VPN configurations."
+ type = map(object({
+ asn = number
+ custom_ranges = map(string)
+ }))
+ default = {
+ land-r1 = {
+ asn = 64513
+ custom_ranges = {
+ "10.0.0.0/8" = "internal default"
+ }
+ }
+ dev-r1 = {
+ asn = 64514
+ custom_ranges = null
+ }
+ prod-r1 = {
+ asn = 64515
+ custom_ranges = null
+ }
}
}
diff --git a/examples/networking/hub-and-spoke-vpn/vpn-dev-r1.tf b/examples/networking/hub-and-spoke-vpn/vpn-dev-r1.tf
new file mode 100644
index 000000000..238475aad
--- /dev/null
+++ b/examples/networking/hub-and-spoke-vpn/vpn-dev-r1.tf
@@ -0,0 +1,107 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# tfdoc:file:description Landing to Development VPN for region 1.
+
+module "landing-to-dev-vpn-r1" {
+ source = "../../../modules/net-vpn-ha"
+ project_id = var.project_id
+ network = module.landing-vpc.self_link
+ region = var.regions.r1
+ name = "${local.prefix}lnd-to-dev-r1"
+ router_create = false
+ router_name = "${local.prefix}lnd-vpn-r1"
+ # router is created and managed by the production VPN module
+ # so we don't configure advertisements here
+ peer_gcp_gateway = module.dev-to-landing-vpn-r1.self_link
+ tunnels = {
+ 0 = {
+ bgp_peer = {
+ address = "169.254.2.2"
+ asn = var.vpn_configs.dev-r1.asn
+ }
+ # use this attribute to configure different advertisements for dev
+ bgp_peer_options = null
+ bgp_session_range = "169.254.2.1/30"
+ ike_version = 2
+ peer_external_gateway_interface = null
+ router = null
+ shared_secret = null
+ vpn_gateway_interface = 0
+ }
+ 1 = {
+ bgp_peer = {
+ address = "169.254.2.6"
+ asn = var.vpn_configs.dev-r1.asn
+ }
+ # use this attribute to configure different advertisements for dev
+ bgp_peer_options = null
+ bgp_session_range = "169.254.2.5/30"
+ ike_version = 2
+ peer_external_gateway_interface = null
+ router = null
+ shared_secret = null
+ vpn_gateway_interface = 1
+ }
+ }
+}
+
+module "dev-to-landing-vpn-r1" {
+ source = "../../../modules/net-vpn-ha"
+ project_id = var.project_id
+ network = module.dev-vpc.self_link
+ region = var.regions.r1
+ name = "${local.prefix}dev-to-lnd-r1"
+ router_create = true
+ router_name = "${local.prefix}dev-vpn-r1"
+ router_asn = var.vpn_configs.dev-r1.asn
+ router_advertise_config = (
+ var.vpn_configs.dev-r1.custom_ranges == null
+ ? null
+ : {
+ groups = null
+ ip_ranges = coalesce(var.vpn_configs.dev-r1.custom_ranges, {})
+ mode = "CUSTOM"
+ }
+ )
+ peer_gcp_gateway = module.landing-to-dev-vpn-r1.self_link
+ tunnels = {
+ 0 = {
+ bgp_peer = {
+ address = "169.254.2.1"
+ asn = var.vpn_configs.land-r1.asn
+ }
+ bgp_peer_options = null
+ bgp_session_range = "169.254.2.2/30"
+ ike_version = 2
+ peer_external_gateway_interface = null
+ router = null
+ shared_secret = module.landing-to-dev-vpn-r1.random_secret
+ vpn_gateway_interface = 0
+ }
+ 1 = {
+ bgp_peer = {
+ address = "169.254.2.5"
+ asn = var.vpn_configs.land-r1.asn
+ }
+ bgp_peer_options = null
+ bgp_session_range = "169.254.2.6/30"
+ ike_version = 2
+ peer_external_gateway_interface = null
+ router = null
+ shared_secret = module.landing-to-dev-vpn-r1.random_secret
+ vpn_gateway_interface = 1
+ }
+ }
+}
diff --git a/examples/networking/hub-and-spoke-vpn/vpn-prod-r1.tf b/examples/networking/hub-and-spoke-vpn/vpn-prod-r1.tf
new file mode 100644
index 000000000..1c2e7028c
--- /dev/null
+++ b/examples/networking/hub-and-spoke-vpn/vpn-prod-r1.tf
@@ -0,0 +1,116 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# tfdoc:file:description Landing to Production VPN for region 1.
+
+module "landing-to-prod-vpn-r1" {
+ source = "../../../modules/net-vpn-ha"
+ project_id = var.project_id
+ network = module.landing-vpc.self_link
+ region = var.regions.r1
+ name = "${local.prefix}lnd-to-prd-r1"
+ router_create = true
+ router_name = "${local.prefix}lnd-vpn-r1"
+ router_asn = var.vpn_configs.land-r1.asn
+ router_advertise_config = (
+ var.vpn_configs.land-r1.custom_ranges == null
+ ? null
+ : {
+ groups = null
+ ip_ranges = coalesce(var.vpn_configs.land-r1.custom_ranges, {})
+ mode = "CUSTOM"
+ }
+ )
+ peer_gcp_gateway = module.prod-to-landing-vpn-r1.self_link
+ tunnels = {
+ 0 = {
+ bgp_peer = {
+ address = "169.254.0.2"
+ asn = var.vpn_configs.prod-r1.asn
+ }
+ bgp_peer_options = null
+ bgp_session_range = "169.254.0.1/30"
+ ike_version = 2
+ peer_external_gateway_interface = null
+ router = null
+ shared_secret = null
+ vpn_gateway_interface = 0
+ }
+ 1 = {
+ bgp_peer = {
+ address = "169.254.0.6"
+ asn = var.vpn_configs.prod-r1.asn
+ }
+ bgp_peer_options = null
+ bgp_session_range = "169.254.0.5/30"
+ ike_version = 2
+ peer_external_gateway_interface = null
+ router = null
+ shared_secret = null
+ vpn_gateway_interface = 1
+ }
+ }
+}
+
+module "prod-to-landing-vpn-r1" {
+ source = "../../../modules/net-vpn-ha"
+ project_id = var.project_id
+ network = module.prod-vpc.self_link
+ region = var.regions.r1
+ name = "${local.prefix}prd-to-lnd-r1"
+ router_create = true
+ router_name = "${local.prefix}prd-vpn-r1"
+ router_asn = var.vpn_configs.prod-r1.asn
+ # the router is managed here but shared with the dev VPN
+ router_advertise_config = (
+ var.vpn_configs.prod-r1.custom_ranges == null
+ ? null
+ : {
+ groups = null
+ ip_ranges = coalesce(var.vpn_configs.prod-r1.custom_ranges, {})
+ mode = "CUSTOM"
+ }
+ )
+ peer_gcp_gateway = module.landing-to-prod-vpn-r1.self_link
+ tunnels = {
+ 0 = {
+ bgp_peer = {
+ address = "169.254.0.1"
+ asn = var.vpn_configs.land-r1.asn
+ }
+ # use this attribute to configure different advertisements for prod
+ bgp_peer_options = null
+ bgp_session_range = "169.254.0.2/30"
+ ike_version = 2
+ peer_external_gateway_interface = null
+ router = null
+ shared_secret = module.landing-to-prod-vpn-r1.random_secret
+ vpn_gateway_interface = 0
+ }
+ 1 = {
+ bgp_peer = {
+ address = "169.254.0.5"
+ asn = var.vpn_configs.land-r1.asn
+ }
+ # use this attribute to configure different advertisements for prod
+ bgp_peer_options = null
+ bgp_session_range = "169.254.0.6/30"
+ ike_version = 2
+ peer_external_gateway_interface = null
+ router = null
+ shared_secret = module.landing-to-prod-vpn-r1.random_secret
+ vpn_gateway_interface = 1
+ }
+ }
+}
diff --git a/fast/stages/03-data-platform/dev/README.md b/fast/stages/03-data-platform/dev/README.md
index 5ecc2ad51..7bc41760f 100644
--- a/fast/stages/03-data-platform/dev/README.md
+++ b/fast/stages/03-data-platform/dev/README.md
@@ -31,10 +31,10 @@ The Data Platform manages:
As per our GCP best practices the Data Platform relies on user groups to assign roles to human identities. These are the specific groups used by the Data Platform and their access patterns, from the [module documentation](../../../../examples/data-solutions/data-platform-foundations/#groups):
- *Data Engineers* They handle and run the Data Hub, with read access to all resources in order to troubleshoot possible issues with pipelines. This team can also impersonate any service account.
-- *Data Analysts*. They perform analysis on datasets, with read access to the data lake L2 project, and BigQuery READ/WRITE access to the playground project.
+- *Data Analysts*. They perform analysis on datasets, with read access to the data warehouse Curated or Confidential projects depending on their privileges, and BigQuery READ/WRITE access to the playground project.
- *Data Security*:. They handle security configurations related to the Data Hub. This team has admin access to the common project to configure Cloud DLP templates or Data Catalog policy tags.
-|Group|Landing|Load|Transformation|Data Lake L0|Data Lake L1|Data Lake L2|Data Lake Playground|Orchestration|Common|
+|Group|Landing|Load|Transformation|Data Warehouse Landing|Data Warehouse Curated|Data Warehouse Confidential|Data Warehouse Playground|Orchestration|Common|
|-|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|
|Data Engineers|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|
|Data Analysts|-|-|-|-|-|`READ`|`READ`/`WRITE`|-|-|
@@ -69,6 +69,12 @@ As is often the case in real-world configurations, [VPC-SC](https://cloud.google
To configure the use of VPC-SC on the data platform, you have to specify the data platform project numbers on the `vpc_sc_perimeter_projects.dev` variable on [FAST security stage](../../02-security#perimeter-resources).
+In the case your Data Warehouse need to handle confidential data and you have the requirement to separate them deeply from other data and IAM is not enough, the suggested configuration is to keep the confidential project in a separate VPC-SC perimeter with the adequate ingress/egress rules needed for the load and tranformation service account. Below you can find an high level diagram describing the configuration.
+
+
+
+
+
## How to run this stage
This stage can be run in isolation by prviding the necessary variables, but it's really meant to be used as part of the FAST flow after the "foundational stages" ([`00-bootstrap`](../../00-bootstrap), [`01-resman`](../../01-resman), [`02-networking`](../../02-networking-vpn) and [`02-security`](../../02-security)).
@@ -131,7 +137,7 @@ terraform apply
## Demo pipeline
-The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `landing` area to the `DataLake L2` dataset suing different features.
+The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `landing` area to the `DataWarehouse Confidential` dataset suing different features.
You can find examples in the `[demo](../../../../examples/data-solutions/data-platform-foundations/demo)` folder.
diff --git a/fast/stages/03-data-platform/dev/diagram.png b/fast/stages/03-data-platform/dev/diagram.png
index 001c6f2ab..79b46e179 100644
Binary files a/fast/stages/03-data-platform/dev/diagram.png and b/fast/stages/03-data-platform/dev/diagram.png differ
diff --git a/fast/stages/03-data-platform/dev/diagram_vpcsc.png b/fast/stages/03-data-platform/dev/diagram_vpcsc.png
new file mode 100644
index 000000000..2bbaad0b2
Binary files /dev/null and b/fast/stages/03-data-platform/dev/diagram_vpcsc.png differ
diff --git a/modules/compute-vm/main.tf b/modules/compute-vm/main.tf
index 50c1fea42..55581fee4 100644
--- a/modules/compute-vm/main.tf
+++ b/modules/compute-vm/main.tf
@@ -348,6 +348,16 @@ resource "google_compute_instance_template" "default" {
scopes = local.service_account_scopes
}
+ dynamic "shielded_instance_config" {
+ for_each = var.shielded_config != null ? [var.shielded_config] : []
+ iterator = config
+ content {
+ enable_secure_boot = config.value.enable_secure_boot
+ enable_vtpm = config.value.enable_vtpm
+ enable_integrity_monitoring = config.value.enable_integrity_monitoring
+ }
+ }
+
lifecycle {
create_before_destroy = true
}
diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md
index 19a15a54a..33fb2ffd1 100644
--- a/modules/gke-cluster/README.md
+++ b/modules/gke-cluster/README.md
@@ -68,44 +68,44 @@ module "cluster-1" {
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
-| [location](variables.tf#L163) | Cluster zone or region. | string | ✓ | |
-| [name](variables.tf#L230) | Cluster name. | string | ✓ | |
-| [network](variables.tf#L235) | Name or self link of the VPC used for the cluster. Use the self link for Shared VPC. | string | ✓ | |
-| [project_id](variables.tf#L279) | Cluster project id. | string | ✓ | |
-| [secondary_range_pods](variables.tf#L302) | Subnet secondary range name used for pods. | string | ✓ | |
-| [secondary_range_services](variables.tf#L307) | Subnet secondary range name used for services. | string | ✓ | |
-| [subnetwork](variables.tf#L312) | VPC subnetwork name or self link. | string | ✓ | |
-| [addons](variables.tf#L17) | Addons enabled in the cluster (true means enabled). | object({…}) | | {…} |
-| [authenticator_security_group](variables.tf#L51) | RBAC security group for Google Groups for GKE, format is gke-security-groups@yourdomain.com. | string | | null |
-| [cluster_autoscaling](variables.tf#L57) | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({…}) | | {…} |
-| [database_encryption](variables.tf#L75) | Enable and configure GKE application-layer secrets encryption. | object({…}) | | {…} |
-| [default_max_pods_per_node](variables.tf#L89) | Maximum number of pods per node in this cluster. | number | | 110 |
-| [description](variables.tf#L95) | Cluster description. | string | | null |
-| [dns_config](variables.tf#L101) | Configuration for Using Cloud DNS for GKE. | object({…}) | | {…} |
-| [enable_autopilot](variables.tf#L115) | Create cluster in autopilot mode. With autopilot there's no need to create node-pools and some features are not supported (e.g. setting default_max_pods_per_node). | bool | | false |
-| [enable_binary_authorization](variables.tf#L121) | Enable Google Binary Authorization. | bool | | null |
-| [enable_dataplane_v2](variables.tf#L127) | Enable Dataplane V2 on the cluster, will disable network_policy addons config. | bool | | false |
-| [enable_intranode_visibility](variables.tf#L133) | Enable intra-node visibility to make same node pod to pod traffic visible. | bool | | null |
-| [enable_l4_ilb_subsetting](variables.tf#L139) | Enable L4ILB Subsetting. | bool | | null |
-| [enable_shielded_nodes](variables.tf#L145) | Enable Shielded Nodes features on all nodes in this cluster. | bool | | null |
-| [enable_tpu](variables.tf#L151) | Enable Cloud TPU resources in this cluster. | bool | | null |
-| [labels](variables.tf#L157) | Cluster resource labels. | map(string) | | null |
-| [logging_config](variables.tf#L168) | Logging configuration (enabled components). | list(string) | | null |
-| [logging_service](variables.tf#L174) | Logging service (disable with an empty string). | string | | "logging.googleapis.com/kubernetes" |
-| [maintenance_config](variables.tf#L180) | Maintenance window configuration. | object({…}) | | {…} |
-| [master_authorized_ranges](variables.tf#L206) | External Ip address ranges that can access the Kubernetes cluster master through HTTPS. | map(string) | | {} |
-| [min_master_version](variables.tf#L212) | Minimum version of the master, defaults to the version of the most recent official release. | string | | null |
-| [monitoring_config](variables.tf#L218) | Monitoring configuration (enabled components). | list(string) | | null |
-| [monitoring_service](variables.tf#L224) | Monitoring service (disable with an empty string). | string | | "monitoring.googleapis.com/kubernetes" |
-| [node_locations](variables.tf#L240) | Zones in which the cluster's nodes are located. | list(string) | | [] |
-| [notification_config](variables.tf#L246) | GKE Cluster upgrade notifications via PubSub. | bool | | false |
-| [peering_config](variables.tf#L252) | Configure peering with the master VPC for private clusters. | object({…}) | | null |
-| [pod_security_policy](variables.tf#L262) | Enable the PodSecurityPolicy feature. | bool | | null |
-| [private_cluster_config](variables.tf#L268) | Enable and configure private cluster, private nodes must be true if used. | object({…}) | | null |
-| [release_channel](variables.tf#L284) | Release channel for GKE upgrades. | string | | null |
-| [resource_usage_export_config](variables.tf#L290) | Configure the ResourceUsageExportConfig feature. | object({…}) | | {…} |
-| [vertical_pod_autoscaling](variables.tf#L317) | Enable the Vertical Pod Autoscaling feature. | bool | | null |
-| [workload_identity](variables.tf#L323) | Enable the Workload Identity feature. | bool | | true |
+| [location](variables.tf#L161) | Cluster zone or region. | string | ✓ | |
+| [name](variables.tf#L228) | Cluster name. | string | ✓ | |
+| [network](variables.tf#L233) | Name or self link of the VPC used for the cluster. Use the self link for Shared VPC. | string | ✓ | |
+| [project_id](variables.tf#L277) | Cluster project id. | string | ✓ | |
+| [secondary_range_pods](variables.tf#L300) | Subnet secondary range name used for pods. | string | ✓ | |
+| [secondary_range_services](variables.tf#L305) | Subnet secondary range name used for services. | string | ✓ | |
+| [subnetwork](variables.tf#L310) | VPC subnetwork name or self link. | string | ✓ | |
+| [addons](variables.tf#L17) | Addons enabled in the cluster (true means enabled). | object({…}) | | {…} |
+| [authenticator_security_group](variables.tf#L53) | RBAC security group for Google Groups for GKE, format is gke-security-groups@yourdomain.com. | string | | null |
+| [cluster_autoscaling](variables.tf#L59) | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({…}) | | {…} |
+| [database_encryption](variables.tf#L77) | Enable and configure GKE application-layer secrets encryption. | object({…}) | | {…} |
+| [default_max_pods_per_node](variables.tf#L91) | Maximum number of pods per node in this cluster. | number | | 110 |
+| [description](variables.tf#L97) | Cluster description. | string | | null |
+| [dns_config](variables.tf#L103) | Configuration for Using Cloud DNS for GKE. | object({…}) | | null |
+| [enable_autopilot](variables.tf#L113) | Create cluster in autopilot mode. With autopilot there's no need to create node-pools and some features are not supported (e.g. setting default_max_pods_per_node). | bool | | false |
+| [enable_binary_authorization](variables.tf#L119) | Enable Google Binary Authorization. | bool | | null |
+| [enable_dataplane_v2](variables.tf#L125) | Enable Dataplane V2 on the cluster, will disable network_policy addons config. | bool | | false |
+| [enable_intranode_visibility](variables.tf#L131) | Enable intra-node visibility to make same node pod to pod traffic visible. | bool | | null |
+| [enable_l4_ilb_subsetting](variables.tf#L137) | Enable L4ILB Subsetting. | bool | | null |
+| [enable_shielded_nodes](variables.tf#L143) | Enable Shielded Nodes features on all nodes in this cluster. | bool | | null |
+| [enable_tpu](variables.tf#L149) | Enable Cloud TPU resources in this cluster. | bool | | null |
+| [labels](variables.tf#L155) | Cluster resource labels. | map(string) | | null |
+| [logging_config](variables.tf#L166) | Logging configuration (enabled components). | list(string) | | null |
+| [logging_service](variables.tf#L172) | Logging service (disable with an empty string). | string | | "logging.googleapis.com/kubernetes" |
+| [maintenance_config](variables.tf#L178) | Maintenance window configuration. | object({…}) | | {…} |
+| [master_authorized_ranges](variables.tf#L204) | External Ip address ranges that can access the Kubernetes cluster master through HTTPS. | map(string) | | {} |
+| [min_master_version](variables.tf#L210) | Minimum version of the master, defaults to the version of the most recent official release. | string | | null |
+| [monitoring_config](variables.tf#L216) | Monitoring configuration (enabled components). | list(string) | | null |
+| [monitoring_service](variables.tf#L222) | Monitoring service (disable with an empty string). | string | | "monitoring.googleapis.com/kubernetes" |
+| [node_locations](variables.tf#L238) | Zones in which the cluster's nodes are located. | list(string) | | [] |
+| [notification_config](variables.tf#L244) | GKE Cluster upgrade notifications via PubSub. | bool | | false |
+| [peering_config](variables.tf#L250) | Configure peering with the master VPC for private clusters. | object({…}) | | null |
+| [pod_security_policy](variables.tf#L260) | Enable the PodSecurityPolicy feature. | bool | | null |
+| [private_cluster_config](variables.tf#L266) | Enable and configure private cluster, private nodes must be true if used. | object({…}) | | null |
+| [release_channel](variables.tf#L282) | Release channel for GKE upgrades. | string | | null |
+| [resource_usage_export_config](variables.tf#L288) | Configure the ResourceUsageExportConfig feature. | object({…}) | | {…} |
+| [vertical_pod_autoscaling](variables.tf#L315) | Enable the Vertical Pod Autoscaling feature. | bool | | null |
+| [workload_identity](variables.tf#L321) | Enable the Workload Identity feature. | bool | | true |
## Outputs
@@ -121,4 +121,4 @@ module "cluster-1" {
| [notifications](outputs.tf#L55) | GKE PubSub notifications topic. | |
| [self_link](outputs.tf#L60) | Cluster self link. | ✓ |
-
\ No newline at end of file
+
diff --git a/modules/gke-cluster/main.tf b/modules/gke-cluster/main.tf
index d8b5aff39..64762b674 100644
--- a/modules/gke-cluster/main.tf
+++ b/modules/gke-cluster/main.tf
@@ -96,6 +96,9 @@ resource "google_container_cluster" "cluster" {
config_connector_config {
enabled = var.addons.config_connector_config
}
+ gke_backup_agent_config {
+ enabled = var.addons.gke_backup_agent_config
+ }
}
# TODO(ludomagno): support setting address ranges instead of range names
@@ -278,12 +281,11 @@ resource "google_container_cluster" "cluster" {
}
dynamic "dns_config" {
- for_each = var.dns_config != null ? [var.dns_config] : []
- iterator = config
+ for_each = var.dns_config != null ? [""] : []
content {
- cluster_dns = config.value.cluster_dns
- cluster_dns_scope = config.value.cluster_dns_scope
- cluster_dns_domain = config.value.cluster_dns_domain
+ cluster_dns = var.dns_config.cluster_dns
+ cluster_dns_scope = var.dns_config.cluster_dns_scope
+ cluster_dns_domain = var.dns_config.cluster_dns_domain
}
}
diff --git a/modules/gke-cluster/variables.tf b/modules/gke-cluster/variables.tf
index d3043bda7..679390209 100644
--- a/modules/gke-cluster/variables.tf
+++ b/modules/gke-cluster/variables.tf
@@ -30,6 +30,7 @@ variable "addons" {
gcp_filestore_csi_driver_config = bool
config_connector_config = bool
kalm_config = bool
+ gke_backup_agent_config = bool
})
default = {
cloudrun_config = false
@@ -45,6 +46,7 @@ variable "addons" {
gcp_filestore_csi_driver_config = false
config_connector_config = false
kalm_config = false
+ gke_backup_agent_config = false
}
}
@@ -105,11 +107,7 @@ variable "dns_config" {
cluster_dns_scope = string
cluster_dns_domain = string
})
- default = {
- cluster_dns = "PROVIDER_UNSPECIFIED"
- cluster_dns_scope = "DNS_SCOPE_UNSPECIFIED"
- cluster_dns_domain = ""
- }
+ default = null
}
variable "enable_autopilot" {
diff --git a/modules/pubsub/README.md b/modules/pubsub/README.md
index 411511abb..a321c662e 100644
--- a/modules/pubsub/README.md
+++ b/modules/pubsub/README.md
@@ -38,6 +38,7 @@ module "pubsub" {
message_retention_duration = null
retain_acked_messages = true
expiration_policy_ttl = null
+ filter = null
}
}
}
@@ -93,17 +94,17 @@ module "pubsub" {
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
-| [name](variables.tf#L60) | PubSub topic name. | string | ✓ | |
-| [project_id](variables.tf#L65) | Project used for resources. | string | ✓ | |
+| [name](variables.tf#L62) | PubSub topic name. | string | ✓ | |
+| [project_id](variables.tf#L67) | Project used for resources. | string | ✓ | |
| [dead_letter_configs](variables.tf#L17) | Per-subscription dead letter policy configuration. | map(object({…})) | | {} |
-| [defaults](variables.tf#L26) | Subscription defaults for options. | object({…}) | | {…} |
-| [iam](variables.tf#L42) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} |
-| [kms_key](variables.tf#L48) | KMS customer managed encryption key. | string | | null |
-| [labels](variables.tf#L54) | Labels. | map(string) | | {} |
-| [push_configs](variables.tf#L70) | Push subscription configurations. | map(object({…})) | | {} |
-| [regions](variables.tf#L83) | List of regions used to set persistence policy. | list(string) | | [] |
-| [subscription_iam](variables.tf#L89) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} |
-| [subscriptions](variables.tf#L95) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | map(object({…})) | | {} |
+| [defaults](variables.tf#L26) | Subscription defaults for options. | object({…}) | | {…} |
+| [iam](variables.tf#L44) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} |
+| [kms_key](variables.tf#L50) | KMS customer managed encryption key. | string | | null |
+| [labels](variables.tf#L56) | Labels. | map(string) | | {} |
+| [push_configs](variables.tf#L72) | Push subscription configurations. | map(object({…})) | | {} |
+| [regions](variables.tf#L85) | List of regions used to set persistence policy. | list(string) | | [] |
+| [subscription_iam](variables.tf#L91) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} |
+| [subscriptions](variables.tf#L97) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | map(object({…})) | | {} |
## Outputs
diff --git a/modules/pubsub/main.tf b/modules/pubsub/main.tf
index 545abbbd7..af08de389 100644
--- a/modules/pubsub/main.tf
+++ b/modules/pubsub/main.tf
@@ -66,6 +66,7 @@ resource "google_pubsub_subscription" "default" {
ack_deadline_seconds = each.value.options.ack_deadline_seconds
message_retention_duration = each.value.options.message_retention_duration
retain_acked_messages = each.value.options.retain_acked_messages
+ filter = each.value.options.filter
dynamic "expiration_policy" {
for_each = each.value.options.expiration_policy_ttl == null ? [] : [""]
diff --git a/modules/pubsub/variables.tf b/modules/pubsub/variables.tf
index 78dd2fa62..592d80090 100644
--- a/modules/pubsub/variables.tf
+++ b/modules/pubsub/variables.tf
@@ -30,12 +30,14 @@ variable "defaults" {
message_retention_duration = string
retain_acked_messages = bool
expiration_policy_ttl = string
+ filter = string
})
default = {
ack_deadline_seconds = null
message_retention_duration = null
retain_acked_messages = null
expiration_policy_ttl = null
+ filter = null
}
}
@@ -101,6 +103,7 @@ variable "subscriptions" {
message_retention_duration = string
retain_acked_messages = bool
expiration_policy_ttl = string
+ filter = string
})
}))
default = {}
diff --git a/tests/examples/networking/hub_and_spoke_vpn/fixture/main.tf b/tests/examples/networking/hub_and_spoke_vpn/fixture/main.tf
index 5dc7d2aaf..006ad4c42 100644
--- a/tests/examples/networking/hub_and_spoke_vpn/fixture/main.tf
+++ b/tests/examples/networking/hub_and_spoke_vpn/fixture/main.tf
@@ -15,6 +15,10 @@
*/
module "test" {
- source = "../../../../../examples/networking/hub-and-spoke-vpn"
- project_id = var.project_id
+ source = "../../../../../examples/networking/hub-and-spoke-vpn"
+ project_create_config = {
+ billing_account_id = "ABCDE-123456-ABCDE"
+ parent_id = null
+ }
+ project_id = "test-1"
}
diff --git a/tests/examples/networking/hub_and_spoke_vpn/fixture/variables.tf b/tests/examples/networking/hub_and_spoke_vpn/fixture/variables.tf
deleted file mode 100644
index 626af0119..000000000
--- a/tests/examples/networking/hub_and_spoke_vpn/fixture/variables.tf
+++ /dev/null
@@ -1,18 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-variable "project_id" {
- type = string
- default = "project-1"
-}
diff --git a/tests/examples/networking/hub_and_spoke_vpn/test_plan.py b/tests/examples/networking/hub_and_spoke_vpn/test_plan.py
index 7d7716824..a24aaa596 100644
--- a/tests/examples/networking/hub_and_spoke_vpn/test_plan.py
+++ b/tests/examples/networking/hub_and_spoke_vpn/test_plan.py
@@ -12,8 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+
def test_resources(e2e_plan_runner):
"Test that plan works and the numbers of resources is as expected."
modules, resources = e2e_plan_runner()
- assert len(modules) == 17
- assert len(resources) == 71
+ assert len(modules) == 19
+ assert len(resources) == 73
diff --git a/tools/requirements.txt b/tools/requirements.txt
index 3a6cb8166..8c7dd54a4 100644
--- a/tools/requirements.txt
+++ b/tools/requirements.txt
@@ -1,4 +1,5 @@
click
+deepdiff
marko
requests
yamale