Network Dashboard: PSA support for Filestore and Memorystore (#1106)

* Support for Filestore and Memorystore PSA ranges

Co-authored-by: Ludovico Magnocavallo <ludomagno@google.com>

blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json

@@ -1,5 +1,4 @@
 {
-  "category": "CUSTOM",
   "displayName": "quotas_utilization",
   "mosaicLayout": {
     "columns": 12,
@@ -18,7 +17,6 @@
                 "plotType": "LINE",
                 "targetAxis": "Y1",
                 "timeSeriesQuery": {
-                  "apiSource": "DEFAULT_CLOUD",
                   "timeSeriesFilter": {
                     "aggregation": {
                       "alignmentPeriod": "3600s",
@@ -40,9 +38,7 @@
             }
           }
         },
-        "width": 6,
+        "width": 6
-        "xPos": 0,
-        "yPos": 0
       },
       {
         "height": 4,
@@ -58,7 +54,6 @@
                 "plotType": "LINE",
                 "targetAxis": "Y1",
                 "timeSeriesQuery": {
-                  "apiSource": "DEFAULT_CLOUD",
                   "timeSeriesFilter": {
                     "aggregation": {
                       "alignmentPeriod": "3600s",
@@ -81,7 +76,6 @@
           }
         },
         "width": 6,
-        "xPos": 0,
         "yPos": 12
       },
       {
@@ -98,7 +92,6 @@
                 "plotType": "LINE",
                 "targetAxis": "Y1",
                 "timeSeriesQuery": {
-                  "apiSource": "DEFAULT_CLOUD",
                   "timeSeriesFilter": {
                     "aggregation": {
                       "alignmentPeriod": "3600s",
@@ -121,7 +114,6 @@
           }
         },
         "width": 6,
-        "xPos": 0,
         "yPos": 8
       },
       {
@@ -138,7 +130,6 @@
                 "plotType": "LINE",
                 "targetAxis": "Y1",
                 "timeSeriesQuery": {
-                  "apiSource": "DEFAULT_CLOUD",
                   "timeSeriesFilter": {
                     "aggregation": {
                       "alignmentPeriod": "3600s",
@@ -178,7 +169,6 @@
                 "plotType": "LINE",
                 "targetAxis": "Y1",
                 "timeSeriesQuery": {
-                  "apiSource": "DEFAULT_CLOUD",
                   "timeSeriesFilter": {
                     "aggregation": {
                       "alignmentPeriod": "3600s",
@@ -201,7 +191,6 @@
           }
         },
         "width": 6,
-        "xPos": 0,
         "yPos": 4
       },
       {
@@ -218,7 +207,6 @@
                 "plotType": "LINE",
                 "targetAxis": "Y1",
                 "timeSeriesQuery": {
-                  "apiSource": "DEFAULT_CLOUD",
                   "timeSeriesFilter": {
                     "aggregation": {
                       "alignmentPeriod": "3600s",
@@ -241,8 +229,7 @@
           }
         },
         "width": 6,
-        "xPos": 6,
+        "xPos": 6
-        "yPos": 0
       },
       {
         "height": 4,
@@ -258,7 +245,6 @@
                 "plotType": "LINE",
                 "targetAxis": "Y1",
                 "timeSeriesQuery": {
-                  "apiSource": "DEFAULT_CLOUD",
                   "timeSeriesFilter": {
                     "aggregation": {
                       "alignmentPeriod": "3600s",
@@ -298,7 +284,6 @@
                 "plotType": "LINE",
                 "targetAxis": "Y1",
                 "timeSeriesQuery": {
-                  "apiSource": "DEFAULT_CLOUD",
                   "timeSeriesFilter": {
                     "aggregation": {
                       "alignmentPeriod": "3600s",
@@ -330,17 +315,19 @@
             },
             "dataSets": [
               {
-                "minAlignmentPeriod": "60s",
+                "minAlignmentPeriod": "3600s",
                 "plotType": "LINE",
                 "targetAxis": "Y1",
                 "timeSeriesQuery": {
-                  "apiSource": "DEFAULT_CLOUD",
                   "timeSeriesFilter": {
                     "aggregation": {
-                      "alignmentPeriod": "60s",
+                      "alignmentPeriod": "3600s",
-                      "perSeriesAligner": "ALIGN_MEAN"
+                      "perSeriesAligner": "ALIGN_NEXT_OLDER"
                     },
-                    "filter": "metric.type=\"custom.googleapis.com/netmon/peering_group/routes_dynamic_used_ratio\" resource.type=\"global\""
+                    "filter": "metric.type=\"custom.googleapis.com/netmon/peering_group/routes_dynamic_used_ratio\" resource.type=\"global\"",
+                    "secondaryAggregation": {
+                      "alignmentPeriod": "60s"
+                    }
                   }
                 }
               }
@@ -353,7 +340,6 @@
           }
         },
         "width": 6,
-        "xPos": 0,
         "yPos": 20
       },
       {
@@ -366,21 +352,23 @@
             },
             "dataSets": [
               {
-                "minAlignmentPeriod": "60s",
+                "minAlignmentPeriod": "3600s",
                 "plotType": "LINE",
                 "targetAxis": "Y1",
                 "timeSeriesQuery": {
-                  "apiSource": "DEFAULT_CLOUD",
                   "timeSeriesFilter": {
                     "aggregation": {
-                      "alignmentPeriod": "60s",
+                      "alignmentPeriod": "3600s",
                       "crossSeriesReducer": "REDUCE_SUM",
                       "groupByFields": [
                         "metric.label.\"project\""
                       ],
-                      "perSeriesAligner": "ALIGN_MEAN"
+                      "perSeriesAligner": "ALIGN_NEXT_OLDER"
                     },
-                    "filter": "metric.type=\"custom.googleapis.com/netmon/project/firewall_rules_used_ratio\" resource.type=\"global\""
+                    "filter": "metric.type=\"custom.googleapis.com/netmon/project/firewall_rules_used_ratio\" resource.type=\"global\"",
+                    "secondaryAggregation": {
+                      "alignmentPeriod": "60s"
+                    }
                   }
                 }
               }
@@ -393,8 +381,7 @@
           }
         },
         "width": 6,
-        "xPos": 0,
+        "yPos": 28
-        "yPos": 32
       },
       {
         "height": 4,
@@ -406,17 +393,19 @@
             },
             "dataSets": [
               {
-                "minAlignmentPeriod": "60s",
+                "minAlignmentPeriod": "3600s",
                 "plotType": "LINE",
                 "targetAxis": "Y1",
                 "timeSeriesQuery": {
-                  "apiSource": "DEFAULT_CLOUD",
                   "timeSeriesFilter": {
                     "aggregation": {
-                      "alignmentPeriod": "60s",
+                      "alignmentPeriod": "3600s",
-                      "perSeriesAligner": "ALIGN_MEAN"
+                      "perSeriesAligner": "ALIGN_NEXT_OLDER"
                     },
-                    "filter": "metric.type=\"custom.googleapis.com/netmon/firewall_policy/tuples_used_ratio\" resource.type=\"global\""
+                    "filter": "metric.type=\"custom.googleapis.com/netmon/firewall_policy/tuples_used_ratio\" resource.type=\"global\"",
+                    "secondaryAggregation": {
+                      "alignmentPeriod": "60s"
+                    }
                   }
                 }
               }
@@ -430,7 +419,7 @@
         },
         "width": 6,
         "xPos": 6,
-        "yPos": 28
+        "yPos": 24
       },
       {
         "height": 4,
@@ -442,17 +431,135 @@
             },
             "dataSets": [
               {
-                "minAlignmentPeriod": "60s",
+                "minAlignmentPeriod": "3600s",
                 "plotType": "LINE",
                 "targetAxis": "Y1",
                 "timeSeriesQuery": {
-                  "apiSource": "DEFAULT_CLOUD",
                   "timeSeriesFilter": {
                     "aggregation": {
-                      "alignmentPeriod": "60s",
+                      "alignmentPeriod": "3600s",
-                      "perSeriesAligner": "ALIGN_MEAN"
+                      "perSeriesAligner": "ALIGN_NEXT_OLDER"
                     },
-                    "filter": "metric.type=\"custom.googleapis.com/netmon/subnetwork/addresses_used_ratio\" resource.type=\"global\""
+                    "filter": "metric.type=\"custom.googleapis.com/netmon/subnetwork/addresses_used_ratio\" resource.type=\"global\"",
+                    "secondaryAggregation": {
+                      "alignmentPeriod": "60s"
+                    }
+                  }
+                }
+              }
+            ],
+            "timeshiftDuration": "0s",
+            "yAxis": {
+              "label": "y1Axis",
+              "scale": "LINEAR"
+            }
+          }
+        },
+        "width": 6,
+        "yPos": 16
+      },
+      {
+        "height": 4,
+        "widget": {
+          "title": "Project static routes used",
+          "xyChart": {
+            "chartOptions": {
+              "mode": "COLOR"
+            },
+            "dataSets": [
+              {
+                "minAlignmentPeriod": "3600s",
+                "plotType": "LINE",
+                "targetAxis": "Y1",
+                "timeSeriesQuery": {
+                  "timeSeriesFilter": {
+                    "aggregation": {
+                      "alignmentPeriod": "3600s",
+                      "crossSeriesReducer": "REDUCE_SUM",
+                      "groupByFields": [
+                        "metric.label.\"project\""
+                      ],
+                      "perSeriesAligner": "ALIGN_NEXT_OLDER"
+                    },
+                    "filter": "metric.type=\"custom.googleapis.com/netmon/project/routes_static_used_ratio\" resource.type=\"global\"",
+                    "secondaryAggregation": {
+                      "alignmentPeriod": "60s"
+                    }
+                  }
+                }
+              }
+            ],
+            "timeshiftDuration": "0s",
+            "yAxis": {
+              "label": "y1Axis",
+              "scale": "LINEAR"
+            }
+          }
+        },
+        "width": 6,
+        "xPos": 6,
+        "yPos": 20
+      },
+      {
+        "height": 4,
+        "widget": {
+          "title": "Peering group static routes used",
+          "xyChart": {
+            "chartOptions": {
+              "mode": "COLOR"
+            },
+            "dataSets": [
+              {
+                "minAlignmentPeriod": "3600s",
+                "plotType": "LINE",
+                "targetAxis": "Y1",
+                "timeSeriesQuery": {
+                  "timeSeriesFilter": {
+                    "aggregation": {
+                      "alignmentPeriod": "3600s",
+                      "perSeriesAligner": "ALIGN_NEXT_OLDER"
+                    },
+                    "filter": "metric.type=\"custom.googleapis.com/netmon/peering_group/routes_static_used_ratio\" resource.type=\"global\"",
+                    "secondaryAggregation": {
+                      "alignmentPeriod": "60s"
+                    }
+                  }
+                }
+              }
+            ],
+            "timeshiftDuration": "0s",
+            "yAxis": {
+              "label": "y1Axis",
+              "scale": "LINEAR"
+            }
+          }
+        },
+        "width": 6,
+        "yPos": 24
+      },
+      {
+        "height": 4,
+        "widget": {
+          "title": "Addresses used ratio per psa range [NEXT OLDER]",
+          "xyChart": {
+            "chartOptions": {
+              "mode": "COLOR"
+            },
+            "dataSets": [
+              {
+                "minAlignmentPeriod": "3600s",
+                "plotType": "LINE",
+                "targetAxis": "Y1",
+                "timeSeriesQuery": {
+                  "timeSeriesFilter": {
+                    "aggregation": {
+                      "alignmentPeriod": "3600s",
+                      "perSeriesAligner": "ALIGN_NEXT_OLDER"
+                    },
+                    "filter": "metric.type=\"custom.googleapis.com/netmon/network/psa/addresses_used_ratio\" resource.type=\"global\"",
+                    "secondaryAggregation": {
+                      "alignmentPeriod": "60s"
+                    }
                   }
                 }
               }
@@ -467,88 +574,6 @@
         "width": 6,
         "xPos": 6,
         "yPos": 16
-      },
-      {
-        "height": 4,
-        "widget": {
-          "title": "Project static routes used",
-          "xyChart": {
-            "chartOptions": {
-              "mode": "COLOR"
-            },
-            "dataSets": [
-              {
-                "minAlignmentPeriod": "60s",
-                "plotType": "LINE",
-                "targetAxis": "Y1",
-                "timeSeriesQuery": {
-                  "apiSource": "DEFAULT_CLOUD",
-                  "timeSeriesFilter": {
-                    "aggregation": {
-                      "alignmentPeriod": "60s",
-                      "crossSeriesReducer": "REDUCE_SUM",
-                      "groupByFields": [
-                        "metric.label.\"project\""
-                      ],
-                      "perSeriesAligner": "ALIGN_MEAN"
-                    },
-                    "filter": "metric.type=\"custom.googleapis.com/netmon/project/routes_static_used_ratio\" resource.type=\"global\"",
-                    "secondaryAggregation": {
-                      "alignmentPeriod": "60s",
-                      "perSeriesAligner": "ALIGN_NONE"
-                    }
-                  }
-                }
-              }
-            ],
-            "thresholds": [],
-            "timeshiftDuration": "0s",
-            "yAxis": {
-              "label": "y1Axis",
-              "scale": "LINEAR"
-            }
-          }
-        },
-        "width": 6,
-        "xPos": 0,
-        "yPos": 24
-      },
-      {
-        "height": 4,
-        "widget": {
-          "title": "Peering group static routes used",
-          "xyChart": {
-            "chartOptions": {
-              "mode": "COLOR"
-            },
-            "dataSets": [
-              {
-                "minAlignmentPeriod": "60s",
-                "plotType": "LINE",
-                "targetAxis": "Y1",
-                "timeSeriesQuery": {
-                  "apiSource": "DEFAULT_CLOUD",
-                  "timeSeriesFilter": {
-                    "aggregation": {
-                      "alignmentPeriod": "60s",
-                      "perSeriesAligner": "ALIGN_MEAN"
-                    },
-                    "filter": "metric.type=\"custom.googleapis.com/netmon/peering_group/routes_static_used_ratio\" resource.type=\"global\""
-                  }
-                }
-              }
-            ],
-            "thresholds": [],
-            "timeshiftDuration": "0s",
-            "yAxis": {
-              "label": "y1Axis",
-              "scale": "LINEAR"
-            }
-          }
-        },
-        "width": 6,
-        "xPos": 0,
-        "yPos": 28
       }
     ]
   }

blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md

@@ -74,7 +74,7 @@ dashboard_json_path = "../dashboards/quotas-utilization.json"
 | [name](variables.tf#L75) | Name used to create Cloud Function related resources. | <code>string</code> |  | <code>&#34;net-dash&#34;</code> |
 | [project_create_config](variables.tf#L81) | Optional configuration if project creation is required. | <code title="object&#40;&#123;&#10;  billing_account_id &#61; string&#10;  parent_id          &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
 | [region](variables.tf#L95) | Compute region where the Cloud Function will be deployed. | <code>string</code> |  | <code>&#34;europe-west1&#34;</code> |
-| [schedule_config](variables.tf#L101) | Schedule timer configuration in crontab format. | <code>string</code> |  | <code>&#34;0&#47;30 &#42; &#42; &#42; &#42;&#34;</code> |
+| [schedule_config](variables.tf#L101) | Schedule timer configuration in crontab format. | <code>string</code> |  | <code>&#34;&#42;&#47;30 &#42; &#42; &#42; &#42;&#34;</code> |
 
 ## Outputs
 

blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf

@@ -101,5 +101,5 @@ variable "region" {
 variable "schedule_config" {
   description = "Schedule timer configuration in crontab format."
   type        = string
-  default     = "0/30 * * * *"
+  default     = "*/30 * * * *"
 }

blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai.py

@@ -39,7 +39,9 @@ TYPES = {
     'subnetworks': 'compute.googleapis.com/Subnetwork',
     'routers': 'compute.googleapis.com/Router',
     'routes': 'compute.googleapis.com/Route',
-    'sql_instances': 'sqladmin.googleapis.com/Instance'
+    'sql_instances': 'sqladmin.googleapis.com/Instance',
+    'filestore_instances': 'file.googleapis.com/Instance',
+    'memorystore_instances': 'redis.googleapis.com/Instance',
 }
 NAMES = {v: k for k, v in TYPES.items()}
 
@@ -82,10 +84,16 @@ def _handle_resource(resources, asset_type, data):
   # e.g. assetType = GlobalAddress but discoveryName = Address
   resource_name = NAMES[asset_type]
   resource = {
-      'id': attrs.get('id'),
+      'id':
-      'name': attrs['name'],
+          attrs.get('id'),
-      'self_link': _self_link(attrs['selfLink']),
+      'name':
-      'assetType': asset_type
+          attrs['name'],
+      # Some resources (ex: Filestore) don't have a self_link, using parent + name in that case
+      'self_link':
+          f'{data["parent"]}/{attrs["name"]}'
+          if not 'selfLink' in attrs else _self_link(attrs['selfLink']),
+      'assetType':
+          asset_type
   }
   # derive parent type and id and skip if parent is not within scope
   parent_data = _get_parent(data['parent'], resources)
@@ -212,6 +220,38 @@ def _handle_sql_instances(resource, data):
       ],
       'region': data['region'],
       'availabilityType': data['settings']['availabilityType'],
+      'network': data['settings']['ipConfiguration']['privateNetwork']
+  }
+
+
+def _handle_filestore_instances(resource, data):
+  'Handles filestore instance type resource data.'
+  return {
+      # Getting only the instance name, removing the rest
+      'name': data['name'].split('/')[-1],
+      # Is a list but for now, only one network is supported for Filestore
+      'network': data['networks'][0]['network'],
+      'reservedIpRange': data['networks'][0]['reservedIpRange'],
+      'ipAddresses': data['networks'][0]['ipAddresses']
+  }
+
+
+def _handle_memorystore_instances(resource, data):
+  'Handles Memorystore (Redis) instance type resource data.'
+  return {
+      # Getting only the instance name, removing the rest
+      'name':
+          data['name'].split('/')[-1],
+      'locationId':
+          data['locationId'],
+      'replicaCount':
+          0 if not 'replicaCount' in data else data['replicaCount'],
+      'network':
+          data['authorizedNetwork'],
+      'reservedIpRange':
+          '' if not 'reservedIpRange' in data else data['reservedIpRange'],
+      'host':
+          '' if not 'host' in data else data['host'],
   }
 
 

blueprints/cloud-operations/network-dashboard/src/plugins/series-psa.py

@@ -34,7 +34,28 @@ def _sql_addresses(sql_instances):
     if not v['ipAddresses']:
       continue
     # 1 IP for the instance + 1 IP for the ILB + 1 IP if HA
-    yield v['ipAddresses'][0], 2 if v['availabilityType'] != 'REGIONAL' else 3
+    yield v['ipAddresses'][
+        0], 2 if v['availabilityType'] != 'REGIONAL' else 3, v['network']
+
+
+def _filestore_addresses(filestore_instances):
+  'Returns counts of Filestore instances per PSA range.'
+  for v in filestore_instances.values():
+    if not v['ipAddresses'] or not v['reservedIpRange']:
+      continue
+    # Subnet size (reservedIpRange) can be /29, /26 or /24
+    yield v['ipAddresses'][0], ipaddress.ip_network(
+        v['reservedIpRange']).num_addresses, v['network']
+
+
+def _memorystore_addresses(memorystore_instances):
+  'Returns counts of Memorystore (Redis) instances per PSA range.'
+  for v in memorystore_instances.values():
+    if not v['reservedIpRange'] or v['reservedIpRange'] == '':
+      continue
+    # Subnet size (reservedIpRange) can be minimum /28 or /29
+    yield v['host'], ipaddress.ip_network(
+        v['reservedIpRange']).num_addresses, v['network']
 
 
 @register_timeseries
@@ -46,26 +67,34 @@ def timeseries(resources):
                            ('project', 'network', 'subnetwork'),
                            dtype.endswith('ratio'))
   psa_nets = {
-      k: ipaddress.ip_network('{}/{}'.format(v['address'], v['prefixLength']))
+      k: {
-      for k, v in resources['global_addresses'].items()
+          'network_link':
-      if v['prefixLength']
+              v['network'],
+          'network_prefix':
+              ipaddress.ip_network('{}/{}'.format(v['address'],
+                                                  v['prefixLength']))
+      } for k, v in resources['global_addresses'].items() if v['prefixLength']
   }
   psa_counts = {}
-  for address, ip_count in _sql_addresses(resources.get('sql_instances', {})):
+  for address, ip_count, network in itertools.chain(
+      _sql_addresses(resources.get('sql_instances', {})),
+      _filestore_addresses(resources.get('filestore_instances', {})),
+      _memorystore_addresses(resources.get('memorystore_instances', {}))):
     ip_address = ipaddress.ip_address(address)
     for k, v in psa_nets.items():
-      if ip_address in v:
+      if network == v['network_link'] and ip_address in v['network_prefix']:
         psa_counts[k] = psa_counts.get(k, 0) + ip_count
         break
 
   for k, v in psa_counts.items():
-    max_ips = psa_nets[k].num_addresses - 4
+    max_ips = psa_nets[k]['network_prefix'].num_addresses - 4
     psa_range = resources['global_addresses'][k]
     labels = {
         'network': psa_range['network'],
         'project': psa_range['project_id'],
         'psa_range': psa_range['name']
     }
+
     yield TimeSeries('network/psa/addresses_available', max_ips, labels)
     yield TimeSeries('network/psa/addresses_used', v, labels)
     yield TimeSeries('network/psa/addresses_used_ratio',