From 94bf7bd63fd0848249d721a12d45c3da7f82e60a Mon Sep 17 00:00:00 2001 From: simonebruzzechesse <60114646+simonebruzzechesse@users.noreply.github.com> Date: Wed, 29 Mar 2023 14:54:06 +0200 Subject: [PATCH] Network Dashboard improvements and bug fixing (#1289) * update documentation and fix group by issue causing duplicated metric * fix format md * fix toc indentation * update required python version in README --- .../network-dashboard/README.md | 1 + .../network-dashboard/src/README.md | 15 ++++++++------- .../src/plugins/discover-group-networks.py | 5 +++-- .../src/plugins/series-firewall-rules.py | 10 ++++++---- .../src/plugins/series-networks.py | 17 ++++++++++++----- .../src/plugins/series-routes.py | 3 ++- 6 files changed, 32 insertions(+), 19 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/README.md b/blueprints/cloud-operations/network-dashboard/README.md index 1fe0960f7..01bc6bf6d 100644 --- a/blueprints/cloud-operations/network-dashboard/README.md +++ b/blueprints/cloud-operations/network-dashboard/README.md @@ -76,6 +76,7 @@ Refer to the [Cloud Function deployment instructions](./deploy-cloud-function/) - The tool assumes global routing is ON, this impacts dynamic routes usage calculation. - The tool assumes custom routes importing/exporting is ON, this impacts static and dynamic routes usage calculation. - The tool assumes all networks in peering groups have the same global routing and custom routes sharing configuration. +- The tool assumes both Shared VPCs service projects and host projects to be in scope. ## TODO diff --git a/blueprints/cloud-operations/network-dashboard/src/README.md b/blueprints/cloud-operations/network-dashboard/src/README.md index a7fad8217..cad11020f 100644 --- a/blueprints/cloud-operations/network-dashboard/src/README.md +++ b/blueprints/cloud-operations/network-dashboard/src/README.md @@ -2,9 +2,10 @@ This tool constitutes the discovery and data gathering side of the Network Dashboard, and can be used in combination with the related [Terraform deployment examples](../), or packaged in different ways including standalone manual use. -- [Quick Usage Example](#quick-usage-example) -- [High Level Architecture and Plugin Design](#high-level-architecture-and-plugin-design) -- [Debugging and Troubleshooting](#debugging-and-troubleshooting) +- [Network Dashboard Discovery Tool](#network-dashboard-discovery-tool) + - [Quick Usage Example](#quick-usage-example) + - [High Level Architecture and Plugin Design](#high-level-architecture-and-plugin-design) + - [Debugging and Troubleshooting](#debugging-and-troubleshooting) ## Quick Usage Example @@ -48,7 +49,7 @@ A typical invocation might look like this: ```bash ./main.py \ -dr organizations/1234567890 \ - -op my-monitoring-project \ + -mon my-monitoring-project \ --folder 1234567890 --folder 987654321 \ --project my-net-project \ --custom-quota-file custom-quotas.yaml @@ -75,7 +76,7 @@ The main module cycles through stages, calling stage plugins in succession itera ## Debugging and Troubleshooting -Note that python version > 3.8 is required. +Note that python version >= 3.10 is required. If you run into a `ModuleNotFoundError`, install the required dependencies: `pip3 install -r requirements.txt` @@ -90,7 +91,7 @@ This is an example call that stores discovery results to a file: ```bash ./main.py \ -dr organizations/1234567890 \ - -op my-monitoring-project \ + -mon my-monitoring-project \ --folder 1234567890 --folder 987654321 \ --project my-net-project \ --custom-quota-file custom-quotas.yaml \ @@ -102,7 +103,7 @@ And this is the corresponding call that skips the discovery phase and also runs ```bash ./main.py \ -dr organizations/1234567890 \ - -op my-monitoring-project \ + -mon my-monitoring-project \ --folder 1234567890 --folder 987654321 \ --project my-net-project \ --custom-quota-file custom-quotas.yaml \ diff --git a/blueprints/cloud-operations/network-dashboard/src/plugins/discover-group-networks.py b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-group-networks.py index 350c288b4..a0cfec068 100644 --- a/blueprints/cloud-operations/network-dashboard/src/plugins/discover-group-networks.py +++ b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-group-networks.py @@ -33,7 +33,8 @@ def init(resources): def start_discovery(resources, response=None): 'Plugin entry point, group and return discovered networks.' LOGGER.info(f'discovery (has response: {response is not None})') - grouped = itertools.groupby(resources['networks'].values(), - lambda v: v['project_id']) + grouped = itertools.groupby( + sorted(resources['networks'].values(), key=lambda i: i['project_id']), + lambda i: i['project_id']) for project_id, vpcs in grouped: yield Resource(NAME, project_id, [v['self_link'] for v in vpcs]) diff --git a/blueprints/cloud-operations/network-dashboard/src/plugins/series-firewall-rules.py b/blueprints/cloud-operations/network-dashboard/src/plugins/series-firewall-rules.py index 5490e6d3b..63cc5e20b 100644 --- a/blueprints/cloud-operations/network-dashboard/src/plugins/series-firewall-rules.py +++ b/blueprints/cloud-operations/network-dashboard/src/plugins/series-firewall-rules.py @@ -38,8 +38,9 @@ def timeseries(resources): yield MetricDescriptor(f'project/{dtype}', name, ('project',), dtype.endswith('ratio')) # group firewall rules by network then prepare and return timeseries - grouped = itertools.groupby(resources['firewall_rules'].values(), - lambda v: v['network']) + grouped = itertools.groupby( + sorted(resources['firewall_rules'].values(), key=lambda i: i['network']), + lambda i: i['network']) for network_id, rules in grouped: count = len(list(rules)) labels = { @@ -48,8 +49,9 @@ def timeseries(resources): } yield TimeSeries('network/firewall_rules_used', count, labels) # group firewall rules by project then prepare and return timeseries - grouped = itertools.groupby(resources['firewall_rules'].values(), - lambda v: v['project_id']) + grouped = itertools.groupby( + sorted(resources['firewall_rules'].values(), + key=lambda i: i['project_id']), lambda i: i['project_id']) for project_id, rules in grouped: count = len(list(rules)) limit = int(resources['quota'][project_id]['global']['FIREWALLS']) diff --git a/blueprints/cloud-operations/network-dashboard/src/plugins/series-networks.py b/blueprints/cloud-operations/network-dashboard/src/plugins/series-networks.py index 0ce7a4b30..40e7f42ab 100644 --- a/blueprints/cloud-operations/network-dashboard/src/plugins/series-networks.py +++ b/blueprints/cloud-operations/network-dashboard/src/plugins/series-networks.py @@ -81,8 +81,12 @@ def _forwarding_rules(resources): forwarding_rules_l7 = itertools.filterfalse( functools.partial(filter, 'INTERNAL_MANAGED'), forwarding_rules) # group each iterator by network and return timeseries - grouped_l4 = itertools.groupby(forwarding_rules_l4, lambda i: i['network']) - grouped_l7 = itertools.groupby(forwarding_rules_l7, lambda i: i['network']) + grouped_l4 = itertools.groupby( + sorted(forwarding_rules_l4, key=lambda i: i['network']), + lambda i: i['network']) + grouped_l7 = itertools.groupby( + sorted(forwarding_rules_l7, key=lambda i: i['network']), + lambda i: i['network']) return itertools.chain( _group_timeseries('forwarding_rules_l4', resources, grouped_l4, 'INTERNAL_FORWARDING_RULES_PER_NETWORK'), @@ -95,7 +99,9 @@ def _instances(resources): 'Groups instances by network and returns relevant timeseries.' instance_networks = itertools.chain.from_iterable( i['networks'] for i in resources['instances'].values()) - grouped = itertools.groupby(instance_networks, lambda i: i['network']) + grouped = itertools.groupby( + sorted(instance_networks, key=lambda i: i['network']), + lambda i: i['network']) return _group_timeseries('instances', resources, grouped, 'INSTANCES_PER_NETWORK_GLOBAL') @@ -120,8 +126,9 @@ def _peerings(resources): def _subnet_ranges(resources): 'Groups subnetworks by network and returns relevant timeseries.' - grouped = itertools.groupby(resources['subnetworks'].values(), - lambda v: v['network']) + grouped = itertools.groupby( + sorted(resources['subnetworks'].values(), key=lambda i: i['network']), + lambda i: i['network']) return _group_timeseries('subnets', resources, grouped, 'SUBNET_RANGES_PER_NETWORK') diff --git a/blueprints/cloud-operations/network-dashboard/src/plugins/series-routes.py b/blueprints/cloud-operations/network-dashboard/src/plugins/series-routes.py index 89011215c..2d951a703 100644 --- a/blueprints/cloud-operations/network-dashboard/src/plugins/series-routes.py +++ b/blueprints/cloud-operations/network-dashboard/src/plugins/series-routes.py @@ -60,7 +60,8 @@ def _static(resources): 'Computes network and project-level timeseries for dynamic routes.' filter = lambda v: v['next_hop_type'] in ('peering', 'network') routes = itertools.filterfalse(filter, resources['routes'].values()) - grouped = itertools.groupby(routes, lambda v: v['network']) + grouped = itertools.groupby(sorted(routes, key=lambda i: i['network']), + lambda i: i['network']) project_counts = {} for network_id, elements in grouped: network = resources['networks'].get(network_id)