From 7874f9c02b3ebfa1308ec4bbf517229add124de5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Thu, 17 Mar 2022 20:08:58 +0100 Subject: [PATCH] refactoring main.py --- .../network-dashboard/cloud-function/main.py | 398 ++++++++++++------ .../network-dashboard/main.tf | 12 - 2 files changed, 270 insertions(+), 140 deletions(-) diff --git a/examples/cloud-operations/network-dashboard/cloud-function/main.py b/examples/cloud-operations/network-dashboard/cloud-function/main.py index 4680e04ff..ccbc05e77 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/main.py +++ b/examples/cloud-operations/network-dashboard/cloud-function/main.py @@ -14,20 +14,17 @@ # limitations under the License. # +import time +import os +import yaml from google.cloud import monitoring_v3 from googleapiclient import discovery from google.api import metric_pb2 as ga_metric -import time -import os -import google.api_core -import re -import random -import yaml # list of projects from which function will get quotas information MONITORED_PROJECTS_LIST = os.environ.get("MONITORED_PROJECTS_LIST").split(",") # project where the metrics and dahsboards will be created -MONITORING_PROJECT_ID = os.environ.get("MONITORING_PROJECT_ID") +MONITORING_PROJECT_ID = os.environ.get("MONITORING_PROJECT_ID") MONITORING_PROJECT_LINK = f"projects/{MONITORING_PROJECT_ID}" service = discovery.build('compute', 'v1') @@ -41,6 +38,17 @@ LIMIT_SUBNETS = os.environ.get("LIMIT_SUBNETS").split(",") LIMIT_L4_PPG = os.environ.get("LIMIT_L4_PPG").split(",") LIMIT_L7_PPG = os.environ.get("LIMIT_L7_PPG").split(",") +# Existing GCP metrics per network +L4_FORWARDING_RULES_USAGE_METRIC = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" +L4_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" +L7_FORWARDING_RULES_USAGE_METRIC = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage" +L7_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit" +SUBNET_RANGES_USAGE_METRIC = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage" +SUBNET_RANGES_LIMIT_METRIC = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit" +GCE_INSTANCES_USAGE_METRIC = "compute.googleapis.com/quota/instances_per_vpc_network/usage" +GCE_INSTANCES_LIMIT_METRIC = "compute.googleapis.com/quota/instances_per_vpc_network/limit" + + def main(event, context): ''' Cloud Function Entry point, called by the scheduler. @@ -59,32 +67,31 @@ def main(event, context): get_vpc_peering_data(metrics_dict) get_l4_forwarding_rules_data(metrics_dict) - # Existing GCP Monitoring metrics for L4 Forwarding Rules per Network - l4_forwarding_rules_usage = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" - l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" + get_pgg_data( + metrics_dict["metrics_per_peering_group"] + ["l4_forwarding_rules_per_peering_group"], + L4_FORWARDING_RULES_USAGE_METRIC, L4_FORWARDING_RULES_LIMIT_METRIC, + LIMIT_L4_PPG) - get_pgg_data(metrics_dict["metrics_per_peering_group"]["l4_forwarding_rules_per_peering_group"], l4_forwarding_rules_usage, l4_forwarding_rules_limit, LIMIT_L4_PPG) + get_pgg_data( + metrics_dict["metrics_per_peering_group"] + ["l7_forwarding_rules_per_peering_group"], + L7_FORWARDING_RULES_USAGE_METRIC, L7_FORWARDING_RULES_LIMIT_METRIC, + LIMIT_L7_PPG) - # Existing GCP Monitoring metrics for L7 Forwarding Rules per Network - l7_forwarding_rules_usage = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage" - l7_forwarding_rules_limit = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit" + get_pgg_data( + metrics_dict["metrics_per_peering_group"] + ["subnet_ranges_per_peering_group"], SUBNET_RANGES_USAGE_METRIC, + SUBNET_RANGES_LIMIT_METRIC, LIMIT_SUBNETS) - get_pgg_data(metrics_dict["metrics_per_peering_group"]["l7_forwarding_rules_per_peering_group"], l7_forwarding_rules_usage, l7_forwarding_rules_limit, LIMIT_L7_PPG) - - # Existing GCP Monitoring metrics for Subnet Ranges per Network - subnet_ranges_usage = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage" - subnet_ranges_limit = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit" - - get_pgg_data(metrics_dict["metrics_per_peering_group"]["subnet_ranges_per_peering_group"], subnet_ranges_usage, subnet_ranges_limit, LIMIT_SUBNETS) - - # Existing GCP Monitoring metrics for GCE per Network - gce_instances_usage = "compute.googleapis.com/quota/instances_per_vpc_network/usage" - gce_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit" - - get_pgg_data(metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], gce_instances_usage, gce_instances_limit, LIMIT_INSTANCES_PPG) + get_pgg_data( + metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], + GCE_INSTANCES_USAGE_METRIC, GCE_INSTANCES_LIMIT_METRIC, + LIMIT_INSTANCES_PPG) return 'Function executed successfully' + def create_client(): ''' Creates the monitoring API client, that will be used to create, read and update custom metrics. @@ -99,29 +106,36 @@ def create_client(): client = monitoring_v3.MetricServiceClient() now = time.time() seconds = int(now) - nanos = int((now - seconds) * 10 ** 9) - interval = monitoring_v3.TimeInterval( - { - "end_time": {"seconds": seconds, "nanos": nanos}, - "start_time": {"seconds": (seconds - 86400), "nanos": nanos}, + nanos = int((now - seconds) * 10**9) + interval = monitoring_v3.TimeInterval({ + "end_time": { + "seconds": seconds, + "nanos": nanos + }, + "start_time": { + "seconds": (seconds - 86400), + "nanos": nanos + }, }) return (client, interval) except Exception as e: raise Exception("Error occurred creating the client: {}".format(e)) + def create_metrics(): with open("metrics.yaml", 'r') as stream: try: - metrics_dict = yaml.safe_load(stream) + metrics_dict = yaml.safe_load(stream) - for metric_list in metrics_dict.values(): - for metric in metric_list.values(): - for sub_metric in metric.values(): - create_metric(sub_metric["name"], sub_metric["description"]) + for metric_list in metrics_dict.values(): + for metric in metric_list.values(): + for sub_metric in metric.values(): + create_metric(sub_metric["name"], sub_metric["description"]) - return metrics_dict + return metrics_dict except yaml.YAMLError as exc: - print(exc) + print(exc) + def create_metric(metric_name, description): ''' @@ -141,15 +155,17 @@ def create_metric(metric_name, description): for desc in client.list_metric_descriptors(name=MONITORING_PROJECT_LINK): types.append(desc.type) - if metric_link not in types: # If the metric doesn't exist yet, then we create it + if metric_link not in types: # If the metric doesn't exist yet, then we create it descriptor = ga_metric.MetricDescriptor() descriptor.type = f"custom.googleapis.com/{metric_name}" descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE descriptor.description = description - descriptor = client.create_metric_descriptor(name=MONITORING_PROJECT_LINK, metric_descriptor=descriptor) + descriptor = client.create_metric_descriptor(name=MONITORING_PROJECT_LINK, + metric_descriptor=descriptor) print("Created {}.".format(descriptor.name)) + def get_gce_instances_data(metrics_dict): ''' Gets the data for GCE instances per VPC Network and writes it to the metric defined in instance_metric. @@ -166,17 +182,27 @@ def get_gce_instances_data(metrics_dict): for project in MONITORED_PROJECTS_LIST: network_dict = get_networks(project) - current_quota_usage = get_quota_current_usage(f"projects/{project}", metric_instances_usage) - current_quota_limit = get_quota_current_limit(f"projects/{project}", metric_instances_limit) + current_quota_usage = get_quota_current_usage(f"projects/{project}", + metric_instances_usage) + current_quota_limit = get_quota_current_limit(f"projects/{project}", + metric_instances_limit) current_quota_usage_view = customize_quota_view(current_quota_usage) current_quota_limit_view = customize_quota_view(current_quota_limit) for net in network_dict: - set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, LIMIT_INSTANCES) - write_data_to_metric(project, net['usage'], metrics_dict["metrics_per_network"]["instance_per_network"]["usage"]["name"], net['network name']) - write_data_to_metric(project, net['limit'], metrics_dict["metrics_per_network"]["instance_per_network"]["limit"]["name"], net['network name']) - write_data_to_metric(project, net['usage']/ net['limit'], metrics_dict["metrics_per_network"]["instance_per_network"]["utilization"]["name"], net['network name']) + set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, + LIMIT_INSTANCES) + write_data_to_metric( + project, net['usage'], metrics_dict["metrics_per_network"] + ["instance_per_network"]["usage"]["name"], net['network name']) + write_data_to_metric( + project, net['limit'], metrics_dict["metrics_per_network"] + ["instance_per_network"]["limit"]["name"], net['network name']) + write_data_to_metric( + project, net['usage'] / net['limit'], + metrics_dict["metrics_per_network"]["instance_per_network"] + ["utilization"]["name"], net['network name']) print(f"Wrote number of instances to metric for projects/{project}") @@ -191,19 +217,38 @@ def get_vpc_peering_data(metrics_dict): None ''' for project in MONITORED_PROJECTS_LIST: - active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(project, LIMIT_VPC_PEER) + active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data( + project, LIMIT_VPC_PEER) for peering in active_vpc_peerings: - write_data_to_metric(project, peering['active_peerings'], metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]["usage"]["name"], peering['network_name']) - write_data_to_metric(project, peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]["limit"]["name"], peering['network_name']) - write_data_to_metric(project, peering['active_peerings'] / peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]["utilization"]["name"], peering['network_name']) - print("Wrote number of active VPC peerings to custom metric for project:", project) + write_data_to_metric( + project, peering['active_peerings'], + metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"] + ["usage"]["name"], peering['network_name']) + write_data_to_metric( + project, peering['network_limit'], metrics_dict["metrics_per_network"] + ["vpc_peering_active_per_network"]["limit"]["name"], + peering['network_name']) + write_data_to_metric( + project, peering['active_peerings'] / peering['network_limit'], + metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"] + ["utilization"]["name"], peering['network_name']) + print("Wrote number of active VPC peerings to custom metric for project:", + project) for peering in vpc_peerings: - write_data_to_metric(project, peering['peerings'], metrics_dict["metrics_per_network"]["vpc_peering_per_network"]["usage"]["name"], peering['network_name']) - write_data_to_metric(project, peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_per_network"]["limit"]["name"], peering['network_name']) - write_data_to_metric(project, peering['peerings'] / peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_per_network"]["utilization"]["name"], peering['network_name']) + write_data_to_metric( + project, peering['peerings'], metrics_dict["metrics_per_network"] + ["vpc_peering_per_network"]["usage"]["name"], peering['network_name']) + write_data_to_metric( + project, peering['network_limit'], metrics_dict["metrics_per_network"] + ["vpc_peering_per_network"]["limit"]["name"], peering['network_name']) + write_data_to_metric( + project, peering['peerings'] / peering['network_limit'], + metrics_dict["metrics_per_network"]["vpc_peering_per_network"] + ["utilization"]["name"], peering['network_name']) print("Wrote number of VPC peerings to custom metric for project:", project) + def gather_vpc_peerings_data(project_id, limit_list): ''' Gets the data for all VPC peerings (active or not) in project_id and writes it to the metric defined in vpc_peering_active_metric and vpc_peering_metric. @@ -215,7 +260,7 @@ def gather_vpc_peerings_data(project_id, limit_list): active_peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each active VPC peering. peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each VPC peering. ''' - active_peerings_dict = [] + active_peerings_dict = [] peerings_dict = [] request = service.networks().list(project=project_id) response = request.execute() @@ -224,22 +269,33 @@ def gather_vpc_peerings_data(project_id, limit_list): if 'peerings' in network: STATE = network['peerings'][0]['state'] if STATE == "ACTIVE": - active_peerings_count = len(network['peerings']) + active_peerings_count = len(network['peerings']) else: active_peerings_count = 0 - peerings_count = len(network['peerings']) + peerings_count = len(network['peerings']) else: peerings_count = 0 active_peerings_count = 0 - active_d = {'project_id': project_id,'network_name':network['name'],'active_peerings':active_peerings_count,'network_limit': get_limit(network['name'], limit_list)} + active_d = { + 'project_id': project_id, + 'network_name': network['name'], + 'active_peerings': active_peerings_count, + 'network_limit': get_limit(network['name'], limit_list) + } active_peerings_dict.append(active_d) - d = {'project_id': project_id,'network_name':network['name'],'peerings':peerings_count,'network_limit': get_limit(network['name'], limit_list)} + d = { + 'project_id': project_id, + 'network_name': network['name'], + 'peerings': peerings_count, + 'network_limit': get_limit(network['name'], limit_list) + } peerings_dict.append(d) return active_peerings_dict, peerings_dict + def get_limit(network_name, limit_list): ''' Checks if this network has a specific limit for a metric, if so, returns that limit, if not, returns the default limit. @@ -258,6 +314,7 @@ def get_limit(network_name, limit_list): else: return 0 + def get_l4_forwarding_rules_data(metrics_dict): ''' Gets the data for L4 Internal Forwarding Rules per VPC Network and writes it to the metric defined in forwarding_rules_metric. @@ -274,19 +331,33 @@ def get_l4_forwarding_rules_data(metrics_dict): for project in MONITORED_PROJECTS_LIST: network_dict = get_networks(project) - current_quota_usage = get_quota_current_usage(f"projects/{project}", l4_forwarding_rules_usage) - current_quota_limit = get_quota_current_limit(f"projects/{project}", l4_forwarding_rules_limit) + current_quota_usage = get_quota_current_usage(f"projects/{project}", + l4_forwarding_rules_usage) + current_quota_limit = get_quota_current_limit(f"projects/{project}", + l4_forwarding_rules_limit) current_quota_usage_view = customize_quota_view(current_quota_usage) current_quota_limit_view = customize_quota_view(current_quota_limit) for net in network_dict: - set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, LIMIT_L4) - write_data_to_metric(project, net['usage'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["usage"]["name"], net['network name']) - write_data_to_metric(project, net['limit'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["limit"]["name"], net['network name']) - write_data_to_metric(project, net['usage']/ net['limit'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["utilization"]["name"], net['network name']) + set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, + LIMIT_L4) + write_data_to_metric( + project, net['usage'], metrics_dict["metrics_per_network"] + ["l4_forwarding_rules_per_network"]["usage"]["name"], + net['network name']) + write_data_to_metric( + project, net['limit'], metrics_dict["metrics_per_network"] + ["l4_forwarding_rules_per_network"]["limit"]["name"], + net['network name']) + write_data_to_metric( + project, net['usage'] / net['limit'], + metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"] + ["utilization"]["name"], net['network name']) + + print( + f"Wrote number of L4 forwarding rules to metric for projects/{project}") - print(f"Wrote number of L4 forwarding rules to metric for projects/{project}") def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg): ''' @@ -307,34 +378,49 @@ def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg): # project_id, network_name, network_id, usage, limit, peerings (list of peered networks) # peerings is a list of dictionary (one for each peered network) and contains: # project_id, network_name, network_id - + # For each network in this GCP project for network_dict in network_dict_list: - current_quota_usage = get_quota_current_usage(f"projects/{project}", usage_metric) - current_quota_limit = get_quota_current_limit(f"projects/{project}", limit_metric) + current_quota_usage = get_quota_current_usage(f"projects/{project}", + usage_metric) + current_quota_limit = get_quota_current_limit(f"projects/{project}", + limit_metric) current_quota_usage_view = customize_quota_view(current_quota_usage) current_quota_limit_view = customize_quota_view(current_quota_limit) - usage, limit = get_usage_limit(network_dict, current_quota_usage_view, current_quota_limit_view, limit_ppg) + usage, limit = get_usage_limit(network_dict, current_quota_usage_view, + current_quota_limit_view, limit_ppg) # Here we add usage and limit to the network dictionary network_dict["usage"] = usage network_dict["limit"] = limit # For every peered network, get usage and limits for peered_network in network_dict['peerings']: - peering_project_usage = customize_quota_view(get_quota_current_usage(f"projects/{peered_network['project_id']}", usage_metric)) - peering_project_limit = customize_quota_view(get_quota_current_limit(f"projects/{peered_network['project_id']}", limit_metric)) + peering_project_usage = customize_quota_view( + get_quota_current_usage(f"projects/{peered_network['project_id']}", + usage_metric)) + peering_project_limit = customize_quota_view( + get_quota_current_limit(f"projects/{peered_network['project_id']}", + limit_metric)) - usage, limit = get_usage_limit(peered_network, peering_project_usage, peering_project_limit, limit_ppg) + usage, limit = get_usage_limit(peered_network, peering_project_usage, + peering_project_limit, limit_ppg) # Here we add usage and limit to the peered network dictionary peered_network["usage"] = usage peered_network["limit"] = limit - count_effective_limit(project, network_dict, metric_dict["usage"]["name"], metric_dict["limit"]["name"], metric_dict["utilization"]["name"], limit_ppg) - print(f"Wrote {metric_dict['usage']['name']} to metric for peering group {network_dict['network_name']} in {project}") + count_effective_limit(project, network_dict, metric_dict["usage"]["name"], + metric_dict["limit"]["name"], + metric_dict["utilization"]["name"], limit_ppg) + print( + f"Wrote {metric_dict['usage']['name']} to metric for peering group {network_dict['network_name']} in {project}" + ) -def count_effective_limit(project_id, network_dict, usage_metric_name, limit_metric_name, utilization_metric_name, limit_ppg): + +def count_effective_limit(project_id, network_dict, usage_metric_name, + limit_metric_name, utilization_metric_name, + limit_ppg): ''' Calculates the effective limits (using algorithm in the link below) for peering groups and writes data (usage, limit, utilization) to the custom metrics. Source: https://cloud.google.com/vpc/docs/quota#vpc-peering-effective-limit @@ -359,12 +445,15 @@ def count_effective_limit(project_id, network_dict, usage_metric_name, limit_met peering_group_usage += peered_network['usage'] # Calculates effective limit: Step 1: max(per network limit, per network_peering_group limit) - limit_step1 = max(network_dict['limit'], get_limit(network_dict['network_name'], limit_ppg)) + limit_step1 = max(network_dict['limit'], + get_limit(network_dict['network_name'], limit_ppg)) # Calculates effective limit: Step 2: List of max(per network limit, per network_peering_group limit) for each peered network limit_step2 = [] for peered_network in network_dict['peerings']: - limit_step2.append(max(peered_network['limit'], get_limit(peered_network['network_name'], limit_ppg))) + limit_step2.append( + max(peered_network['limit'], + get_limit(peered_network['network_name'], limit_ppg))) # Calculates effective limit: Step 3: Find minimum from the list created by Step 2 limit_step3 = min(limit_step2) @@ -373,9 +462,13 @@ def count_effective_limit(project_id, network_dict, usage_metric_name, limit_met effective_limit = max(limit_step1, limit_step3) utilization = peering_group_usage / effective_limit - write_data_to_metric(project_id, peering_group_usage, usage_metric_name, network_dict['network_name']) - write_data_to_metric(project_id, effective_limit, limit_metric_name, network_dict['network_name']) - write_data_to_metric(project_id, utilization, utilization_metric_name, network_dict['network_name']) + write_data_to_metric(project_id, peering_group_usage, usage_metric_name, + network_dict['network_name']) + write_data_to_metric(project_id, effective_limit, limit_metric_name, + network_dict['network_name']) + write_data_to_metric(project_id, utilization, utilization_metric_name, + network_dict['network_name']) + def get_networks(project_id): ''' @@ -393,10 +486,11 @@ def get_networks(project_id): for network in response['items']: NETWORK = network['name'] ID = network['id'] - d = {'project_id':project_id,'network name':NETWORK,'network id':ID} + d = {'project_id': project_id, 'network name': NETWORK, 'network id': ID} network_dict.append(d) return network_dict + def gather_peering_data(project_id): ''' Returns a dictionary of all peerings for all networks in a project. @@ -412,20 +506,36 @@ def gather_peering_data(project_id): network_list = [] if 'items' in response: for network in response['items']: - net = {'project_id':project_id,'network_name':network['name'],'network_id':network['id'], 'peerings':[]} + net = { + 'project_id': project_id, + 'network_name': network['name'], + 'network_id': network['id'], + 'peerings': [] + } if 'peerings' in network: STATE = network['peerings'][0]['state'] if STATE == "ACTIVE": - for peered_network in network['peerings']: # "projects/{project_name}/global/networks/{network_name}" - start = peered_network['network'].find("projects/") + len('projects/') + for peered_network in network[ + 'peerings']: # "projects/{project_name}/global/networks/{network_name}" + start = peered_network['network'].find("projects/") + len( + 'projects/') end = peered_network['network'].find("/global") peered_project = peered_network['network'][start:end] - peered_network_name = peered_network['network'].split("networks/")[1] - peered_net = {'project_id': peered_project, 'network_name':peered_network_name, 'network_id': get_network_id(peered_project, peered_network_name)} + peered_network_name = peered_network['network'].split( + "networks/")[1] + peered_net = { + 'project_id': + peered_project, + 'network_name': + peered_network_name, + 'network_id': + get_network_id(peered_project, peered_network_name) + } net["peerings"].append(peered_net) network_list.append(net) return network_list + def get_network_id(project_id, network_name): ''' Returns the network_id for a specific project / network name. @@ -446,12 +556,13 @@ def get_network_id(project_id, network_name): if network['name'] == network_name: network_id = network['id'] break - + if network_id == 0: print(f"Error: network_id not found for {network_name} in {project_id}") - + return network_id + def get_quota_current_usage(project_link, metric_name): ''' Retrieves quota usage for a specific metric. @@ -464,15 +575,17 @@ def get_quota_current_usage(project_link, metric_name): ''' client, interval = create_client() - results = client.list_time_series(request={ - "name": project_link, - "filter": f'metric.type = "{metric_name}"', - "interval": interval, - "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL - }) + results = client.list_time_series( + request={ + "name": project_link, + "filter": f'metric.type = "{metric_name}"', + "interval": interval, + "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL + }) results_list = list(results) return (results_list) + def get_quota_current_limit(project_link, metric_name): ''' Retrieves limit for a specific metric. @@ -485,15 +598,17 @@ def get_quota_current_limit(project_link, metric_name): ''' client, interval = create_client() - results = client.list_time_series(request={ - "name": project_link, - "filter": f'metric.type = "{metric_name}"', - "interval": interval, - "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL - }) + results = client.list_time_series( + request={ + "name": project_link, + "filter": f'metric.type = "{metric_name}"', + "interval": interval, + "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL + }) results_list = list(results) return results_list + def customize_quota_view(quota_results): ''' Customize the quota output for an easier parsable output. @@ -513,6 +628,7 @@ def customize_quota_view(quota_results): quotaViewList.append(quotaViewJson) return quotaViewList + def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list): ''' Updates the network dictionary with quota usage and limit values. @@ -526,9 +642,10 @@ def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list): None ''' if quota_usage: - for net in quota_usage: - if net['network_id'] == network_dict['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same - network_dict['usage'] = net['value'] # set network usage in dictionary + for net in quota_usage: + if net['network_id'] == network_dict[ + 'network id']: # if network ids in GCP quotas and in dictionary (using API) are the same + network_dict['usage'] = net['value'] # set network usage in dictionary break else: network_dict['usage'] = 0 # if network does not appear in GCP quotas @@ -537,19 +654,28 @@ def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list): if quota_limit: for net in quota_limit: - if net['network_id'] == network_dict['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same - network_dict['limit'] = net['value'] # set network limit in dictionary + if net['network_id'] == network_dict[ + 'network id']: # if network ids in GCP quotas and in dictionary (using API) are the same + network_dict['limit'] = net['value'] # set network limit in dictionary break else: - if network_dict['network name'] in limit_list: # if network limit is in the environmental variables - network_dict['limit'] = int(limit_list[limit_list.index(network_dict['network name']) + 1]) + if network_dict[ + 'network name'] in limit_list: # if network limit is in the environmental variables + network_dict['limit'] = int( + limit_list[limit_list.index(network_dict['network name']) + 1]) else: - network_dict['limit'] = int(limit_list[limit_list.index('default_value') + 1]) # set default value + network_dict['limit'] = int( + limit_list[limit_list.index('default_value') + + 1]) # set default value else: # if quotas does not appear in GCP quotas if network_dict['network name'] in limit_list: - network_dict['limit'] = int(limit_list[limit_list.index(network_dict['network name']) + 1]) # ["default", 100, "networkname", 200] + network_dict['limit'] = int( + limit_list[limit_list.index(network_dict['network name']) + + 1]) # ["default", 100, "networkname", 200] else: - network_dict['limit'] = int(limit_list[limit_list.index('default_value') + 1]) + network_dict['limit'] = int(limit_list[limit_list.index('default_value') + + 1]) + def get_usage_limit(network, quota_usage, quota_limit, limit_list): ''' @@ -568,30 +694,37 @@ def get_usage_limit(network, quota_usage, quota_limit, limit_list): limit = 0 if quota_usage: - for net in quota_usage: - if net['network_id'] == network['network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same - usage = net['value'] # set network usage in dictionary + for net in quota_usage: + if net['network_id'] == network[ + 'network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same + usage = net['value'] # set network usage in dictionary break if quota_limit: for net in quota_limit: - if net['network_id'] == network['network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same - limit = net['value'] # set network limit in dictionary + if net['network_id'] == network[ + 'network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same + limit = net['value'] # set network limit in dictionary break else: - if network['network_name'] in limit_list: # if network limit is in the environmental variables - limit = int(limit_list[limit_list.index(network['network_name']) + 1]) + if network[ + 'network_name'] in limit_list: # if network limit is in the environmental variables + limit = int(limit_list[limit_list.index(network['network_name']) + 1]) else: - limit = int(limit_list[limit_list.index('default_value') + 1]) # set default value + limit = int(limit_list[limit_list.index('default_value') + + 1]) # set default value else: # if quotas does not appear in GCP quotas if network['network_name'] in limit_list: - limit = int(limit_list[limit_list.index(network['network_name']) + 1]) # ["default", 100, "networkname", 200] + limit = int(limit_list[limit_list.index(network['network_name']) + + 1]) # ["default", 100, "networkname", 200] else: - limit = int(limit_list[limit_list.index('default_value') + 1]) + limit = int(limit_list[limit_list.index('default_value') + 1]) return usage, limit -def write_data_to_metric(monitored_project_id, value, metric_name, network_name): + +def write_data_to_metric(monitored_project_id, value, metric_name, + network_name): ''' Writes data to Cloud Monitoring custom metrics. @@ -608,15 +741,24 @@ def write_data_to_metric(monitored_project_id, value, metric_name, network_name) series = monitoring_v3.TimeSeries() series.metric.type = f"custom.googleapis.com/{metric_name}" - series.resource.type = "global" + series.resource.type = "global" series.metric.labels["network_name"] = network_name series.metric.labels["project"] = monitored_project_id now = time.time() seconds = int(now) - nanos = int((now - seconds) * 10 ** 9) - interval = monitoring_v3.TimeInterval({"end_time": {"seconds": seconds, "nanos": nanos}}) - point = monitoring_v3.Point({"interval": interval, "value": {"double_value": value}}) + nanos = int((now - seconds) * 10**9) + interval = monitoring_v3.TimeInterval( + {"end_time": { + "seconds": seconds, + "nanos": nanos + }}) + point = monitoring_v3.Point({ + "interval": interval, + "value": { + "double_value": value + } + }) series.points = [point] - client.create_time_series(name=MONITORING_PROJECT_LINK, time_series=[series]) \ No newline at end of file + client.create_time_series(name=MONITORING_PROJECT_LINK, time_series=[series]) diff --git a/examples/cloud-operations/network-dashboard/main.tf b/examples/cloud-operations/network-dashboard/main.tf index 4a797900e..f3ab73bc5 100644 --- a/examples/cloud-operations/network-dashboard/main.tf +++ b/examples/cloud-operations/network-dashboard/main.tf @@ -155,18 +155,6 @@ module "cloud-function" { } } -# TODO: How to secure Cloud Function invokation? Not member = "allUsers" but specific Scheduler service Account? -# Maybe "service-YOUR_PROJECT_NUMBER@gcp-sa-cloudscheduler.iam.gserviceaccount.com"? - -resource "google_cloudfunctions_function_iam_member" "invoker" { - project = module.project-monitoring.project_id - region = var.region - cloud_function = module.cloud-function.function_name - - role = "roles/cloudfunctions.invoker" - member = "allUsers" -} - ################################################ # Cloud Monitoring Dashboard creation # ################################################