From 1dc69656941e3133f3d0b56d2d070248539d38b1 Mon Sep 17 00:00:00 2001 From: maunope <44614195+maunope@users.noreply.github.com> Date: Tue, 12 Dec 2023 19:17:01 +0100 Subject: [PATCH] Update quota monitor blueprint to support project discovery (#1924) * fist test * dev complete * update tf with permissions, enabled APIs and discovery root management * updated readme * moved projects discovery to a separate method * reviewed Mauri's changes * add missing lines from last change * - fixed discovery page size to 100 - removed last_asset_page_reached var from discover_projects - added cast to list for projects var in _main, to make the script work both using CLI and pub/sub * fixed discovery_root default value to work when no value is passed * fixed tfdoc * fixed tftest resources # --------- Co-authored-by: Ludo --- .../quota-monitoring/README.md | 13 ++--- .../cloud-operations/quota-monitoring/main.tf | 58 ++++++++++++++++++- .../quota-monitoring/src/main.py | 50 ++++++++++++---- .../quota-monitoring/variables.tf | 19 ++++-- 4 files changed, 116 insertions(+), 24 deletions(-) diff --git a/blueprints/cloud-operations/quota-monitoring/README.md b/blueprints/cloud-operations/quota-monitoring/README.md index aeaa9b4d9..26a702e82 100644 --- a/blueprints/cloud-operations/quota-monitoring/README.md +++ b/blueprints/cloud-operations/quota-monitoring/README.md @@ -38,9 +38,10 @@ The region, location of the bundle used to deploy the function, and scheduling f The `quota_config` variable mirrors the arguments accepted by the Python program, and allows configuring several different aspects of its behaviour: +- `quota_config.discover_root` organization or folder to be used to discover all underlying projects to track quotas for, in `organizations/nnnnn` or `folders/nnnnn` format - `quota_config.exclude` do not generate metrics for quotas matching prefixes listed here - `quota_config.include` only generate metrics for quotas matching prefixes listed here -- `quota_config.projects` projects to track quotas for, defaults to the project where metrics are stored +- `quota_config.projects` projects to track quotas for, defaults to the project where metrics are stored, if projects are automatically discovered, those in this list are appended. - `quota_config.regions` regions to track quotas for, defaults to the `global` region for project-level quotas - `dry_run` do not write actual metrics - `verbose` increase logging verbosity @@ -54,7 +55,6 @@ Clone this repository or [open it in cloud shell](https://ssh.cloud.google.com/c - `terraform init` - `terraform apply -var project_id=my-project-id` - ## Variables | name | description | type | required | default | @@ -64,10 +64,9 @@ Clone this repository or [open it in cloud shell](https://ssh.cloud.google.com/c | [bundle_path](variables.tf#L33) | Path used to write the intermediate Cloud Function code bundle. | string | | "./bundle.zip" | | [name](variables.tf#L39) | Arbitrary string used to name created resources. | string | | "quota-monitor" | | [project_create_config](variables.tf#L45) | Create project instead of using an existing one. | object({…}) | | null | -| [quota_config](variables.tf#L59) | Cloud function configuration. | object({…}) | | {} | -| [region](variables.tf#L76) | Compute region used in the example. | string | | "europe-west1" | -| [schedule_config](variables.tf#L82) | Schedule timer configuration in crontab format. | string | | "0 * * * *" | - +| [quota_config](variables.tf#L59) | Cloud function configuration. | object({…}) | | {} | +| [region](variables.tf#L85) | Compute region used in the example. | string | | "europe-west1" | +| [schedule_config](variables.tf#L91) | Schedule timer configuration in crontab format. | string | | "0 * * * *" | ## Test @@ -80,5 +79,5 @@ module "test" { billing_account = "12345-ABCDE-12345" } } -# tftest modules=4 resources=14 +# tftest modules=4 resources=19 ``` diff --git a/blueprints/cloud-operations/quota-monitoring/main.tf b/blueprints/cloud-operations/quota-monitoring/main.tf index a49891c00..d5f9a4866 100644 --- a/blueprints/cloud-operations/quota-monitoring/main.tf +++ b/blueprints/cloud-operations/quota-monitoring/main.tf @@ -20,6 +20,8 @@ locals { ? [var.project_id] : var.quota_config.projects ) + discovery_root_type = split("/", coalesce(var.quota_config["discovery_root"], "/"))[0] + discovery_root_id = split("/", coalesce(var.quota_config["discovery_root"], "/"))[1] } module "project" { @@ -29,8 +31,11 @@ module "project" { parent = try(var.project_create_config.parent, null) project_create = var.project_create_config != null services = [ - "compute.googleapis.com", - "cloudfunctions.googleapis.com" + "cloudasset.googleapis.com", + "cloudbuild.googleapis.com", + "cloudfunctions.googleapis.com", + "cloudscheduler.googleapis.com", + "compute.googleapis.com" ] } @@ -81,6 +86,55 @@ resource "google_cloud_scheduler_job" "default" { } } +resource "google_organization_iam_member" "org_asset_viewer" { + count = local.discovery_root_type == "organizations" ? 1 : 0 + org_id = local.discovery_root_id + role = "roles/cloudasset.viewer" + member = module.cf.service_account_iam_email +} + + +# role with the least privilege including compute.projects.get permission +resource "google_organization_iam_member" "org_network_viewer" { + count = local.discovery_root_type == "organizations" ? 1 : 0 + org_id = local.discovery_root_id + role = "roles/compute.networkViewer" + member = module.cf.service_account_iam_email +} + +resource "google_organization_iam_member" "org_quota_viewer" { + count = local.discovery_root_type == "organizations" ? 1 : 0 + org_id = local.discovery_root_id + role = "roles/servicemanagement.quotaViewer" + member = module.cf.service_account_iam_email +} + +resource "google_folder_iam_member" "folder_asset_viewer" { + count = local.discovery_root_type == "folders" ? 1 : 0 + folder = local.discovery_root_id + role = "roles/cloudasset.viewer" + member = module.cf.service_account_iam_email +} + +# role with the least privilege including compute.projects.get permission +resource "google_folder_iam_member" "folder_network_viewer" { + count = local.discovery_root_type == "folders" ? 1 : 0 + folder = local.discovery_root_id + role = "roles/compute.networkViewer" + member = module.cf.service_account_iam_email +} + +resource "google_folder_iam_member" "folder_quota_viewer" { + count = local.discovery_root_type == "folders" ? 1 : 0 + folder = local.discovery_root_id + role = "roles/servicemanagement.quotaViewer" + member = module.cf.service_account_iam_email +} + + + + + resource "google_project_iam_member" "metric_writer" { project = module.project.project_id role = "roles/monitoring.metricWriter" diff --git a/blueprints/cloud-operations/quota-monitoring/src/main.py b/blueprints/cloud-operations/quota-monitoring/src/main.py index 5a8453641..e681afada 100755 --- a/blueprints/cloud-operations/quota-monitoring/src/main.py +++ b/blueprints/cloud-operations/quota-monitoring/src/main.py @@ -39,6 +39,9 @@ HTTP_HEADERS = {'content-type': 'application/json; charset=UTF-8'} URL_PROJECT = 'https://compute.googleapis.com/compute/v1/projects/{}' URL_REGION = 'https://compute.googleapis.com/compute/v1/projects/{}/regions/{}' URL_TS = 'https://monitoring.googleapis.com/v3/projects/{}/timeSeries' +URL_DISCOVERY = ('https://cloudasset.googleapis.com/v1/{}/assets?' + 'assetTypes=cloudresourcemanager.googleapis.com%2FProject&' + 'contentType=RESOURCE&pageSize=100&pageToken={}') _Quota = collections.namedtuple('_Quota', 'project region tstamp metric limit usage') @@ -80,8 +83,8 @@ class Quota(_Quota): else: d['valueType'] = 'INT64' d['points'][0]['value'] = {'int64Value': value} - # remove this label if cardinality gets too high - d['metric']['labels']['quota'] = f'{self.usage}/{self.limit}' + # re-enable the following line if cardinality is not a problem + # d['metric']['labels']['quota'] = f'{self.usage}/{self.limit}' return d @property @@ -92,7 +95,7 @@ class Quota(_Quota): ratio = 0 yield self._api_format('ratio', ratio) yield self._api_format('usage', self.usage) - # yield self._api_format('limit', self.limit) + yield self._api_format('limit', self.limit) def batched(iterable, n): @@ -112,6 +115,23 @@ def configure_logging(verbose=True): warnings.filterwarnings('ignore', r'.*end user credentials.*', UserWarning) +def discover_projects(discovery_root): + 'Discovers projects under a folder or organization.' + if discovery_root.partition('/')[0] not in ('folders', 'organizations'): + raise SystemExit(f'Invalid discovery root {discovery_root}.') + next_page_token = '' + while True: + list_assets_results = fetch( + HTTPRequest(URL_DISCOVERY.format(discovery_root, next_page_token))) + if 'assets' in list_assets_results: + for asset in list_assets_results['assets']: + if (asset['resource']['data']['lifecycleState'] == 'ACTIVE'): + yield asset['resource']['data']['projectId'] + next_page_token = list_assets_results.get('nextPageToken') + if not next_page_token: + break + + def fetch(request, delete=False): 'Minimal HTTP client interface for API calls.' logging.debug(f'fetch {"POST" if request.data else "GET"} {request.url}') @@ -163,9 +183,13 @@ def get_quotas(project, region='global'): @click.command() @click.argument('project-id', required=True) +@click.option( + '--discovery-root', '-dr', required=False, help= + 'Root node used to dynamically fetch projects, in organizations/nnn or folders/nnn format.' +) @click.option( '--project-ids', multiple=True, help= - 'Project ids to monitor (multiple). Defaults to monitoring project if not set.' + 'Project ids to monitor (multiple). Defaults to monitoring project if not set, values are appended to those found under discovery-root' ) @click.option('--regions', multiple=True, help='Regions (multiple). Defaults to "global" if not set.') @@ -175,11 +199,13 @@ def get_quotas(project, region='global'): help='Exclude quotas starting with keyword (multiple).') @click.option('--dry-run', is_flag=True, help='Do not write metrics.') @click.option('--verbose', is_flag=True, help='Verbose output.') -def main_cli(project_id=None, project_ids=None, regions=None, include=None, - exclude=None, dry_run=False, verbose=False): +def main_cli(project_id=None, discovery_root=None, project_ids=None, + regions=None, include=None, exclude=None, dry_run=False, + verbose=False): 'Fetch GCE quotas and writes them as custom metrics to Stackdriver.' try: - _main(project_id, project_ids, regions, include, exclude, dry_run, verbose) + _main(project_id, discovery_root, project_ids, regions, include, exclude, + dry_run, verbose) except RuntimeError as e: logging.exception(f'exception raised: {e.args[0]}') @@ -193,14 +219,18 @@ def main(event, context): raise -def _main(monitoring_project, projects=None, regions=None, include=None, - exclude=None, dry_run=False, verbose=False): +def _main(monitoring_project, discovery_root=None, projects=None, regions=None, + include=None, exclude=None, dry_run=False, verbose=False): """Module entry point used by cli and cloud function wrappers.""" configure_logging(verbose=verbose) - projects = projects or [monitoring_project] + + # default to monitoring scope project if projects parameter is not passed, then merge the list with discovered projects, if any regions = regions or ['global'] include = set(include or []) exclude = set(exclude or []) + projects = projects or [monitoring_project] + if (discovery_root): + projects = set(list(projects) + list(discover_projects(discovery_root))) for k in ('monitoring_project', 'projects', 'regions', 'include', 'exclude'): logging.debug(f'{k} {locals().get(k)}') timeseries = [] diff --git a/blueprints/cloud-operations/quota-monitoring/variables.tf b/blueprints/cloud-operations/quota-monitoring/variables.tf index 21cf76539..737cf0f91 100644 --- a/blueprints/cloud-operations/quota-monitoring/variables.tf +++ b/blueprints/cloud-operations/quota-monitoring/variables.tf @@ -63,14 +63,23 @@ variable "quota_config" { "a2", "c2", "c2d", "committed", "g2", "interconnect", "m1", "m2", "m3", "nvidia", "preemptible" ]) - include = optional(list(string)) - projects = optional(list(string)) - regions = optional(list(string)) - dry_run = optional(bool, false) - verbose = optional(bool, false) + discovery_root = optional(string, "") + dry_run = optional(bool, false) + include = optional(list(string)) + projects = optional(list(string)) + regions = optional(list(string)) + verbose = optional(bool, false) }) nullable = false default = {} + validation { + condition = ( + var.quota_config.discovery_root == "" || + startswith(var.quota_config.discovery_root, "folders/") || + startswith(var.quota_config.discovery_root, "organizations/") + ) + error_message = "non-null discovery root needs to start with folders/ or organizations/" + } } variable "region" {