diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index e7cf8773a..7e0ffff73 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -80,6 +80,11 @@ jobs: run: | python3 tools/check_documentation.py --show-diffs --no-show-summary modules fast + - name: Check schema docs + id: schema-docs + run: | + python3 tools/check_schema_docs.py --show-diffs --no-show-summary modules fast blueprints + - name: Check documentation links id: documentation-links-fabric run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a0f1364ae..70c3d2977 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -111,6 +111,14 @@ repos: pass_filenames: false files: ^(fast|modules) entry: tools/check_yaml_schema.py modules fast + - id: check-schema-docs + name: Check Markdown generated from JSON schemas + language: python + additional_dependencies: + - click + pass_filenames: false + files: ^(fast|modules).*schema\.json$ + entry: tools/check_schema_docs.py --no-show-summary modules fast - id: check-links name: Check links in markdown files language: python diff --git a/fast/stages/0-org-setup/schemas/budget.schema.md b/fast/stages/0-org-setup/schemas/budget.schema.md index 33bb16038..dabf990df 100644 --- a/fast/stages/0-org-setup/schemas/budget.schema.md +++ b/fast/stages/0-org-setup/schemas/budget.schema.md @@ -20,6 +20,7 @@ - **exclude_all**: *boolean* - **include_specified**: *array* - items: *string* +
*enum: ['COMMITTED_USAGE_DISCOUNT', 'COMMITTED_USAGE_DISCOUNT_DOLLAR_BASE', 'DISCOUNT', 'FREE_TIER', 'PROMOTION', 'RESELLER_MARGIN', 'SUBSCRIPTION_BENEFIT', 'SUSTAINED_USAGE_DISCOUNT']* - **label**: *object*
*additional properties: false* - **key**: *string* diff --git a/fast/stages/0-org-setup/schemas/folder.schema.md b/fast/stages/0-org-setup/schemas/folder.schema.md index 62f2d79ba..168f85f72 100644 --- a/fast/stages/0-org-setup/schemas/folder.schema.md +++ b/fast/stages/0-org-setup/schemas/folder.schema.md @@ -6,6 +6,13 @@ *additional properties: false* +- **asset_search**: *object* +
*additional properties: false* + - **`^[a-z0-9-]+$`**: *object* +
*additional properties: false* + - ⁺**asset_types**: *array* + - items: *string* + - **query**: *string* - **asset_feeds**: *object*
*additional properties: false* - **`^[a-z0-9-]+$`**: *object* @@ -75,6 +82,26 @@ - **exempted_members**: *array* - items: *string* - **deletion_protection**: *boolean* +- **id**: *string* +
*pattern: ^(folders/[0-9]+|\$folder_ids:[a-z0-9_/-]+)$* +- **firewall_policy**: *object* +
*additional properties: false* + - ⁺**name**: *string* + - ⁺**policy**: *string* +- **logging**: *object* +
*additional properties: false* + - **kms_key_name**: *string* + - **storage_location**: *string* + - **sinks**: *object* +
*additional properties: false* + - **`^[a-z][a-z0-9-_]+$`**: *object* +
*additional properties: false* + - **description**: *string* + - **destination**: *string* + - **exclusions**: *object* + - **filter**: *string* + - **type**: *string* +
*default: logging*, *enum: ['bigquery', 'logging', 'project', 'pubsub', 'storage']* - **factories_config**: *object*
*additional properties: false* - **org_policies**: *string* diff --git a/fast/stages/0-org-setup/schemas/observability.schema.md b/fast/stages/0-org-setup/schemas/observability.schema.md index e3e411782..9eed8240d 100644 --- a/fast/stages/0-org-setup/schemas/observability.schema.md +++ b/fast/stages/0-org-setup/schemas/observability.schema.md @@ -153,13 +153,14 @@
*additional properties: false* - **forecast_horizon**: *string* - **trigger**: *reference([trigger](#refs-trigger))* -- **aggregations**: *object* -
*additional properties: false* - - **per_series_aligner**: *string* - - **group_by_fields**: *array* - - items: *string* - - **cross_series_reducer**: *string* - - **alignment_period**: *string* +- **aggregations**: *array* + - items: *object* +
*additional properties: false* + - **per_series_aligner**: *string* + - **group_by_fields**: *array* + - items: *string* + - **cross_series_reducer**: *string* + - **alignment_period**: *string* - **trigger**: *object*
*additional properties: false* - **count**: *number* diff --git a/fast/stages/0-org-setup/schemas/project.schema.md b/fast/stages/0-org-setup/schemas/project.schema.md index da7303641..d2cceab50 100644 --- a/fast/stages/0-org-setup/schemas/project.schema.md +++ b/fast/stages/0-org-setup/schemas/project.schema.md @@ -324,7 +324,7 @@
*enum: ['LIVE', 'ARCHIVED', 'ANY']* - **logging_config**: *object*
*additional properties: false* - - **log_bucket**: *string* + - ⁺**log_bucket**: *string* - **log_object_prefix**: *string* - **location**: *string* - **managed_folders**: *object* diff --git a/fast/stages/2-networking/schemas/project.schema.md b/fast/stages/2-networking/schemas/project.schema.md index da7303641..d2cceab50 100644 --- a/fast/stages/2-networking/schemas/project.schema.md +++ b/fast/stages/2-networking/schemas/project.schema.md @@ -324,7 +324,7 @@
*enum: ['LIVE', 'ARCHIVED', 'ANY']* - **logging_config**: *object*
*additional properties: false* - - **log_bucket**: *string* + - ⁺**log_bucket**: *string* - **log_object_prefix**: *string* - **location**: *string* - **managed_folders**: *object* diff --git a/fast/stages/2-networking/schemas/vlan-attachments.schema.md b/fast/stages/2-networking/schemas/vlan-attachments.schema.md index a97a6e8c0..a39117016 100644 --- a/fast/stages/2-networking/schemas/vlan-attachments.schema.md +++ b/fast/stages/2-networking/schemas/vlan-attachments.schema.md @@ -15,6 +15,8 @@ - **bgp_priority**: *number* - ⁺**interconnect**: *string* - ⁺**vlan_tag**: *string* + - **candidate_cloud_router_ip_address**: *string* + - **candidate_customer_router_ip_address**: *string* - **description**: *string* - **ipsec_gateway_ip_ranges**: *object*
*additional properties: string* diff --git a/fast/stages/2-networking/schemas/vpc.schema.md b/fast/stages/2-networking/schemas/vpc.schema.md index 68b344753..d35f0eeef 100644 --- a/fast/stages/2-networking/schemas/vpc.schema.md +++ b/fast/stages/2-networking/schemas/vpc.schema.md @@ -9,6 +9,12 @@ - ⁺**project_id**: *string* - ⁺**name**: *string* - **description**: *string* +- **factories_config**: *object* +
*additional properties: false* + - **firewall_rules**: *string* + - **subnets**: *string* + - **vlan_attachments**: *string* + - **vpns**: *string* - **auto_create_subnetworks**: *boolean* - **delete_default_routes_on_create**: *boolean* - **mtu**: *number* @@ -16,12 +22,6 @@
*enum: ['GLOBAL', 'REGIONAL']* - **firewall_policy_enforcement_order**: *string*
*enum: ['BEFORE_CLASSIC_FIREWALL', 'AFTER_CLASSIC_FIREWALL']* -- **factories_config**: *object* -
*additional properties: false* - - **firewall_rules**: *string* - - **subnets**: *string* - - **vlan_attachments**: *string* - - **vpns**: *string* - **create_googleapis_routes**: *reference([create_googleapis_routes](#refs-create_googleapis_routes))* - **dns_policy**: *reference([dns_policy](#refs-dns_policy))* - **ipv6_config**: *reference([ipv6_config](#refs-ipv6_config))* diff --git a/fast/stages/2-project-factory/schemas/budget.schema.md b/fast/stages/2-project-factory/schemas/budget.schema.md index 33bb16038..dabf990df 100644 --- a/fast/stages/2-project-factory/schemas/budget.schema.md +++ b/fast/stages/2-project-factory/schemas/budget.schema.md @@ -20,6 +20,7 @@ - **exclude_all**: *boolean* - **include_specified**: *array* - items: *string* +
*enum: ['COMMITTED_USAGE_DISCOUNT', 'COMMITTED_USAGE_DISCOUNT_DOLLAR_BASE', 'DISCOUNT', 'FREE_TIER', 'PROMOTION', 'RESELLER_MARGIN', 'SUBSCRIPTION_BENEFIT', 'SUSTAINED_USAGE_DISCOUNT']* - **label**: *object*
*additional properties: false* - **key**: *string* diff --git a/fast/stages/2-project-factory/schemas/folder.schema.md b/fast/stages/2-project-factory/schemas/folder.schema.md index 62f2d79ba..168f85f72 100644 --- a/fast/stages/2-project-factory/schemas/folder.schema.md +++ b/fast/stages/2-project-factory/schemas/folder.schema.md @@ -6,6 +6,13 @@ *additional properties: false* +- **asset_search**: *object* +
*additional properties: false* + - **`^[a-z0-9-]+$`**: *object* +
*additional properties: false* + - ⁺**asset_types**: *array* + - items: *string* + - **query**: *string* - **asset_feeds**: *object*
*additional properties: false* - **`^[a-z0-9-]+$`**: *object* @@ -75,6 +82,26 @@ - **exempted_members**: *array* - items: *string* - **deletion_protection**: *boolean* +- **id**: *string* +
*pattern: ^(folders/[0-9]+|\$folder_ids:[a-z0-9_/-]+)$* +- **firewall_policy**: *object* +
*additional properties: false* + - ⁺**name**: *string* + - ⁺**policy**: *string* +- **logging**: *object* +
*additional properties: false* + - **kms_key_name**: *string* + - **storage_location**: *string* + - **sinks**: *object* +
*additional properties: false* + - **`^[a-z][a-z0-9-_]+$`**: *object* +
*additional properties: false* + - **description**: *string* + - **destination**: *string* + - **exclusions**: *object* + - **filter**: *string* + - **type**: *string* +
*default: logging*, *enum: ['bigquery', 'logging', 'project', 'pubsub', 'storage']* - **factories_config**: *object*
*additional properties: false* - **org_policies**: *string* diff --git a/fast/stages/2-project-factory/schemas/project.schema.md b/fast/stages/2-project-factory/schemas/project.schema.md index da7303641..d2cceab50 100644 --- a/fast/stages/2-project-factory/schemas/project.schema.md +++ b/fast/stages/2-project-factory/schemas/project.schema.md @@ -324,7 +324,7 @@
*enum: ['LIVE', 'ARCHIVED', 'ANY']* - **logging_config**: *object*
*additional properties: false* - - **log_bucket**: *string* + - ⁺**log_bucket**: *string* - **log_object_prefix**: *string* - **location**: *string* - **managed_folders**: *object* diff --git a/fast/stages/2-security/schemas/folder.schema.md b/fast/stages/2-security/schemas/folder.schema.md index 62f2d79ba..168f85f72 100644 --- a/fast/stages/2-security/schemas/folder.schema.md +++ b/fast/stages/2-security/schemas/folder.schema.md @@ -6,6 +6,13 @@ *additional properties: false* +- **asset_search**: *object* +
*additional properties: false* + - **`^[a-z0-9-]+$`**: *object* +
*additional properties: false* + - ⁺**asset_types**: *array* + - items: *string* + - **query**: *string* - **asset_feeds**: *object*
*additional properties: false* - **`^[a-z0-9-]+$`**: *object* @@ -75,6 +82,26 @@ - **exempted_members**: *array* - items: *string* - **deletion_protection**: *boolean* +- **id**: *string* +
*pattern: ^(folders/[0-9]+|\$folder_ids:[a-z0-9_/-]+)$* +- **firewall_policy**: *object* +
*additional properties: false* + - ⁺**name**: *string* + - ⁺**policy**: *string* +- **logging**: *object* +
*additional properties: false* + - **kms_key_name**: *string* + - **storage_location**: *string* + - **sinks**: *object* +
*additional properties: false* + - **`^[a-z][a-z0-9-_]+$`**: *object* +
*additional properties: false* + - **description**: *string* + - **destination**: *string* + - **exclusions**: *object* + - **filter**: *string* + - **type**: *string* +
*default: logging*, *enum: ['bigquery', 'logging', 'project', 'pubsub', 'storage']* - **factories_config**: *object*
*additional properties: false* - **org_policies**: *string* diff --git a/fast/stages/2-security/schemas/project.schema.md b/fast/stages/2-security/schemas/project.schema.md index da7303641..d2cceab50 100644 --- a/fast/stages/2-security/schemas/project.schema.md +++ b/fast/stages/2-security/schemas/project.schema.md @@ -324,7 +324,7 @@
*enum: ['LIVE', 'ARCHIVED', 'ANY']* - **logging_config**: *object*
*additional properties: false* - - **log_bucket**: *string* + - ⁺**log_bucket**: *string* - **log_object_prefix**: *string* - **location**: *string* - **managed_folders**: *object* diff --git a/modules/billing-account/schemas/budget.schema.md b/modules/billing-account/schemas/budget.schema.md index 33bb16038..dabf990df 100644 --- a/modules/billing-account/schemas/budget.schema.md +++ b/modules/billing-account/schemas/budget.schema.md @@ -20,6 +20,7 @@ - **exclude_all**: *boolean* - **include_specified**: *array* - items: *string* +
*enum: ['COMMITTED_USAGE_DISCOUNT', 'COMMITTED_USAGE_DISCOUNT_DOLLAR_BASE', 'DISCOUNT', 'FREE_TIER', 'PROMOTION', 'RESELLER_MARGIN', 'SUBSCRIPTION_BENEFIT', 'SUSTAINED_USAGE_DISCOUNT']* - **label**: *object*
*additional properties: false* - **key**: *string* diff --git a/modules/net-vpc-factory/schemas/defaults.schema.md b/modules/net-vpc-factory/schemas/defaults.schema.md new file mode 100644 index 000000000..1b863b5d7 --- /dev/null +++ b/modules/net-vpc-factory/schemas/defaults.schema.md @@ -0,0 +1,28 @@ +# Net VPC Factory Defaults + + + +## Properties + +*additional properties: false* + +- **context**: *object* +
*additional properties: false* + - **cidr_ranges_sets**: *object* +
*additional properties: array* + - **iam_principals**: *object* +
*additional properties: string* + - **locations**: *object* +
*additional properties: string* + - **project_ids**: *object* +
*additional properties: string* +- **vpcs**: *object* +
*additional properties: false* + - **auto_create_subnetworks**: *boolean* + - **delete_default_route_on_create**: *boolean* + - **mtu**: *number* +
*default: 1500* + +## Definitions + + diff --git a/modules/project-factory/schemas/budget.schema.md b/modules/project-factory/schemas/budget.schema.md index 33bb16038..dabf990df 100644 --- a/modules/project-factory/schemas/budget.schema.md +++ b/modules/project-factory/schemas/budget.schema.md @@ -20,6 +20,7 @@ - **exclude_all**: *boolean* - **include_specified**: *array* - items: *string* +
*enum: ['COMMITTED_USAGE_DISCOUNT', 'COMMITTED_USAGE_DISCOUNT_DOLLAR_BASE', 'DISCOUNT', 'FREE_TIER', 'PROMOTION', 'RESELLER_MARGIN', 'SUBSCRIPTION_BENEFIT', 'SUSTAINED_USAGE_DISCOUNT']* - **label**: *object*
*additional properties: false* - **key**: *string* diff --git a/modules/project-factory/schemas/folder.schema.md b/modules/project-factory/schemas/folder.schema.md index c6f12f894..168f85f72 100644 --- a/modules/project-factory/schemas/folder.schema.md +++ b/modules/project-factory/schemas/folder.schema.md @@ -82,6 +82,8 @@ - **exempted_members**: *array* - items: *string* - **deletion_protection**: *boolean* +- **id**: *string* +
*pattern: ^(folders/[0-9]+|\$folder_ids:[a-z0-9_/-]+)$* - **firewall_policy**: *object*
*additional properties: false* - ⁺**name**: *string* diff --git a/modules/project-factory/schemas/project.schema.md b/modules/project-factory/schemas/project.schema.md index da7303641..d2cceab50 100644 --- a/modules/project-factory/schemas/project.schema.md +++ b/modules/project-factory/schemas/project.schema.md @@ -324,7 +324,7 @@
*enum: ['LIVE', 'ARCHIVED', 'ANY']* - **logging_config**: *object*
*additional properties: false* - - **log_bucket**: *string* + - ⁺**log_bucket**: *string* - **log_object_prefix**: *string* - **location**: *string* - **managed_folders**: *object* diff --git a/modules/project/schemas/observability.schema.md b/modules/project/schemas/observability.schema.md index e3e411782..9eed8240d 100644 --- a/modules/project/schemas/observability.schema.md +++ b/modules/project/schemas/observability.schema.md @@ -153,13 +153,14 @@
*additional properties: false* - **forecast_horizon**: *string* - **trigger**: *reference([trigger](#refs-trigger))* -- **aggregations**: *object* -
*additional properties: false* - - **per_series_aligner**: *string* - - **group_by_fields**: *array* - - items: *string* - - **cross_series_reducer**: *string* - - **alignment_period**: *string* +- **aggregations**: *array* + - items: *object* +
*additional properties: false* + - **per_series_aligner**: *string* + - **group_by_fields**: *array* + - items: *string* + - **cross_series_reducer**: *string* + - **alignment_period**: *string* - **trigger**: *object*
*additional properties: false* - **count**: *number* diff --git a/tools/check_schema_docs.py b/tools/check_schema_docs.py new file mode 100755 index 000000000..67837511c --- /dev/null +++ b/tools/check_schema_docs.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +'''Recursively check freshness of generated markdown from JSON schemas. + +This tool recursively checks that the markdown files generated from JSON schemas +match what is generated at runtime by schema_docs based on current sources. +''' + +import difflib +import enum +import json +import logging +import pathlib +import sys + +import click + +try: + import schema_docs +except ImportError: + sys.path.append(str(pathlib.Path(__file__).resolve().parent)) + import schema_docs + +BASEDIR = pathlib.Path(__file__).resolve().parents[1] + + +class State(enum.IntEnum): + SKIP = enum.auto() + OK = enum.auto() + FAIL_STALE_DOC = enum.auto() + FAIL_MISSING_DOC = enum.auto() + + @property + def failed(self): + return self.value > State.OK + + @property + def label(self): + return { + State.SKIP: ' ', + State.OK: '✓ ', + State.FAIL_STALE_DOC: '✗D', + State.FAIL_MISSING_DOC: '✗M', + }[self.value] + + +def _check_dir(dir_name): + 'Invoke schema_docs on folder, using the relevant options.' + dir_path = BASEDIR / dir_name + for schema_path in sorted(dir_path.glob('**/*.schema.json')): + if '.terraform' in str(schema_path): + continue + + diff = None + schema_rel = str(schema_path.relative_to(BASEDIR)) + doc_path = schema_path.with_suffix('.md') + + try: + schema = json.load(schema_path.open()) + except json.JSONDecodeError as e: + raise SystemExit(f'error decoding file {schema_path}: {e.args[0]}') + + # schema_docs uses logging.DEBUG heavily + logging.getLogger().setLevel(logging.CRITICAL) + + tree = schema_docs.parse_node(schema) + props, defs = schema_docs.render_node(tree) + doc = schema_docs.DOC.format(title=schema.get('title'), properties=props, + definitions=defs or '') + new_doc_content = f'{doc}\n' + + state = State.OK + + if not doc_path.exists(): + state = State.FAIL_MISSING_DOC + diff = f'----- {schema_rel} missing doc -----\nFile {doc_path.relative_to(BASEDIR)} does not exist.' + else: + current_doc_content = doc_path.read_text() + if new_doc_content != current_doc_content: + state = State.FAIL_STALE_DOC + header = f'----- {schema_rel} diff -----\n' + ndiff = difflib.ndiff(current_doc_content.splitlines(keepends=True), + new_doc_content.splitlines(keepends=True)) + diff = ''.join([header] + [x for x in ndiff if x[0] != ' ']) + + yield schema_rel, state, diff + + +@click.command() +@click.argument('dirs', type=str, nargs=-1) +@click.option('--show-diffs/--no-show-diffs', default=False) +@click.option('--show-summary/--no-show-summary', default=True) +def main(dirs, show_diffs=False, show_summary=True): + 'Cycle through modules and ensure schema docs are up-to-date.' + errors = [] + for dir_name in dirs: + result = _check_dir(dir_name) + for schema_path, state, diff in result: + if state.failed: + errors.append((schema_path, diff)) + if show_summary: + print(f'[{state.label}] {schema_path}') + + if errors: + print('\nErrored schemas:\n') + for e in errors: + module, diff = e + print(f'- {module}') + if show_diffs: + print() + print(''.join(diff)) + print() + print() + raise SystemExit('Errors found.') + + +if __name__ == '__main__': + main() diff --git a/tools/lint.sh b/tools/lint.sh index 2674acb09..bf0f86ae6 100755 --- a/tools/lint.sh +++ b/tools/lint.sh @@ -25,6 +25,9 @@ terraform fmt -recursive -check -diff $PWD echo -- READMEs -- python3 tools/check_documentation.py --no-show-summary modules fast blueprints +echo -- Schema docs -- +python3 tools/check_schema_docs.py --no-show-summary modules fast blueprints + echo -- Links -- python3 tools/check_links.py --no-show-summary $PWD diff --git a/tools/schema_docs.py b/tools/schema_docs.py index 0c0033424..90302eb84 100755 --- a/tools/schema_docs.py +++ b/tools/schema_docs.py @@ -184,5 +184,5 @@ def main(paths=None): if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig(level=logging.INFO) main()