From 4f123ccc74f35ef6bdf1ae1bc5f66d430714b6ae Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Fri, 28 Jul 2023 14:18:28 +0200 Subject: [PATCH] Extend tfdoc to generate TOCs --- modules/artifact-registry/README.md | 14 ++++-- modules/organization/README.md | 20 ++++++--- tools/check_documentation.py | 23 ++++++++-- tools/tfdoc.py | 69 ++++++++++++++++++++++++----- 4 files changed, 102 insertions(+), 24 deletions(-) diff --git a/modules/artifact-registry/README.md b/modules/artifact-registry/README.md index c222754ee..829a61a46 100644 --- a/modules/artifact-registry/README.md +++ b/modules/artifact-registry/README.md @@ -2,6 +2,14 @@ This module simplifies the creation of repositories using Google Cloud Artifact Registry. + +- [Standard Repository](#standard-repository) +- [Remote and Virtual Repositories](#remote-and-virtual-repositories) +- [itional Docker and Maven Options](#itional-docker-and-maven-options) +- [Variables](#variables) +- [Outputs](#outputs) + + ## Standard Repository ```hcl @@ -58,12 +66,12 @@ module "registry-virtual" { } } -# tftest modules=3 resources=3 inventory=remote-virtual.yaml +# tst modules=3 resources=3 inventory=remote-virtual.yaml ``` -## Additional Docker and Maven Options +## itional Docker and Maven Options -```hcl +``` module "registry-docker" { source = "./fabric/modules/artifact-registry" diff --git a/modules/organization/README.md b/modules/organization/README.md index 7ae03da6f..19c59e9d2 100644 --- a/modules/organization/README.md +++ b/modules/organization/README.md @@ -10,20 +10,26 @@ This module allows managing several organization properties: To manage organization policies, the `orgpolicy.googleapis.com` service should be enabled in the quota project. -## Features - +## TOC + +- [TOC](#toc) +- [Example](#example) - [IAM](#iam) - [Organization Policies](#organization-policies) - - [Factory](#organization-policy-factory) - - [Custom Constraints](#organization-policy-custom-constraints) - - [Custom Constraints Factory](#organization-policy-custom-constraints-factory) + - [Organization Policy Factory](#organization-policy-factory) + - [Organization Policy Custom Constraints](#organization-policy-custom-constraints) + - [Organization Policy Custom Constraints Factory](#organization-policy-custom-constraints-factory) - [Hierarchical Firewall Policies](#hierarchical-firewall-policies) - - [Directly Defined](#directly-defined-firewall-policies) - - [Factory](#firewall-policy-factory) + - [Directly Defined Firewall Policies](#directly-defined-firewall-policies) + - [Firewall Policy Factory](#firewall-policy-factory) - [Log Sinks](#log-sinks) - [Data Access Logs](#data-access-logs) - [Custom Roles](#custom-roles) - [Tags](#tags) +- [Files](#files) +- [Variables](#variables) +- [Outputs](#outputs) + ## Example diff --git a/tools/check_documentation.py b/tools/check_documentation.py index 619cff57e..50faeaf8b 100755 --- a/tools/check_documentation.py +++ b/tools/check_documentation.py @@ -35,6 +35,7 @@ class State(enum.IntEnum): SKIP = enum.auto() OK = enum.auto() FAIL_STALE_README = enum.auto() + FAIL_STALE_TOC = enum.auto() FAIL_UNSORTED_VARS = enum.auto() FAIL_UNSORTED_OUTPUTS = enum.auto() FAIL_VARIABLE_PERIOD = enum.auto() @@ -52,6 +53,7 @@ class State(enum.IntEnum): State.SKIP: ' ', State.OK: '✓ ', State.FAIL_STALE_README: '✗R', + State.FAIL_STALE_TOC: '✗T', State.FAIL_UNSORTED_VARS: 'SV', State.FAIL_UNSORTED_OUTPUTS: 'SO', State.FAIL_VARIABLE_PERIOD: '.V', @@ -71,8 +73,9 @@ def _check_dir(dir_name, exclude_files=None, files=False, show_extra=False): diff = None readme = readme_path.read_text() mod_name = str(readme_path.relative_to(dir_path).parent) - result = tfdoc.get_doc(readme) - if not result: + doc_result = tfdoc.get_doc(readme) + toc_result = tfdoc.get_toc(readme) + if not doc_result: state = State.SKIP else: try: @@ -82,18 +85,30 @@ def _check_dir(dir_name, exclude_files=None, files=False, show_extra=False): newouts = new_doc.outputs variables = [v.name for v in newvars if v.file.endswith('variables.tf')] outputs = [o.name for o in newouts if o.file.endswith('outputs.tf')] + + new_toc = tfdoc.create_toc(readme) + except SystemExit: state = state.SKIP else: state = State.OK - if new_doc.content != result['doc']: + if new_doc.content.strip() != doc_result['doc'].strip(): state = State.FAIL_STALE_README header = f'----- {mod_name} diff -----\n' - ndiff = difflib.ndiff(result['doc'].split('\n'), + ndiff = difflib.ndiff(doc_result['doc'].split('\n'), new_doc.content.split('\n')) diff = '\n'.join([header] + list(ndiff)) + if new_toc.strip() != toc_result['toc'].strip(): + print(f"=====new\n{new_toc}") + print(f"=====result\n{toc_result['toc']}") + state = State.FAIL_STALE_TOC + header = f'----- {mod_name} diff -----\n' + ndiff = difflib.ndiff(toc_result['toc'].split('\n'), + new_toc.split('\n')) + diff = '\n'.join([header] + list(ndiff)) + elif empty := [v.name for v in newvars if not v.description]: state = state.FAIL_VARIABLE_DESCRIPTION diff = "\n".join([ diff --git a/tools/tfdoc.py b/tools/tfdoc.py index 662f7aeb9..a6d985274 100755 --- a/tools/tfdoc.py +++ b/tools/tfdoc.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -46,6 +46,7 @@ import string import urllib.parse import click +import marko __version__ = '2.1.0' @@ -80,6 +81,8 @@ OUT_RE = re.compile(r'''(?smx) ''') OUT_TEMPLATE = ('description', 'value', 'sensitive') TAG_RE = re.compile(r'(?sm)^\s*#\stfdoc:([^:]+:\S+)\s+(.*?)\s*$') +TOC_BEGIN = '' +TOC_END = '' UNESCAPED = string.digits + string.ascii_letters + ' .,;:_-' VAR_ENUM = enum.Enum('V', 'OPEN ATTR ATTR_DATA SKIP CLOSE COMMENT TXT') VAR_RE = re.compile(r'''(?smx) @@ -322,17 +325,40 @@ def format_variables(items, show_extra=True): yield format +def create_toc(readme): + doc = marko.parse(readme) + lines = [] + headings = [x for x in doc.children if x.get_type() == 'Heading'] + for h in headings[1:]: + title = h.children[0].children + slug = title.lower().strip() + slug = re.sub('[^\w\s-]', '', slug) + slug = re.sub('[-\s]+', '-', slug) + link = f'- [{title}](#{slug})' + indent = ' ' * (h.level - 2) + lines.append(f'{indent}{link}') + return "\n".join(lines) + + # replace functions def get_doc(readme): 'Check if README file is marked, and return current doc.' - m = re.search('(?sm)%s\n(.*)\n%s' % (MARK_BEGIN, MARK_END), readme) + m = re.search("(?sm)%s(.*)%s" % (MARK_BEGIN, MARK_END), readme) if not m: return return {'doc': m.group(1), 'start': m.start(), 'end': m.end()} +def get_toc(readme): + 'Check if README file is marked, and return current doc.' + t = re.search("(?sm)%s(.*)%s" % (TOC_BEGIN, TOC_END), readme) + if not t: + return + return {'toc': t.group(1), 'start': t.start(), 'end': t.end()} + + def get_doc_opts(readme): 'Check if README file is setting options via a mark, and return options.' m = MARK_OPTS_RE.search(readme) @@ -373,22 +399,41 @@ def get_readme(readme_path): raise SystemExit(f'Error opening README {readme_path}: {e}') -def replace_doc(readme_path, doc, readme=None): +def render_doc(readme, doc): 'Replace document in module\'s README.md file.' - readme = readme or get_readme(readme_path) result = get_doc(readme) if not result: - raise SystemExit(f'Mark not found in README {readme_path}') + raise SystemExit(f'Mark not found in README {readme}') if doc == result['doc']: - return + return readme try: - open(readme_path, 'w').write('\n'.join([ + return '\n'.join([ readme[:result['start']].rstrip(), MARK_BEGIN, doc, MARK_END, readme[result['end']:].lstrip(), - ])) + ]) + except (IOError, OSError) as e: + raise SystemExit(f'Error replacing README {readme_path}: {e}') + + +def render_toc(readme, toc): + 'Replace document in module\'s README.md file.' + result = get_toc(readme) + if not result: + raise SystemExit(f'TOC not found in README {readme}') + if toc == result['toc']: + return readme + try: + return '\n'.join([ + readme[:result['start']].rstrip(), + TOC_BEGIN, + toc, + TOC_END, + "", + readme[result['end']:].lstrip(), + ]) except (IOError, OSError) as e: raise SystemExit(f'Error replacing README {readme_path}: {e}') @@ -405,10 +450,14 @@ def main(module_path=None, exclude_file=None, files=False, replace=True, readme_path = os.path.join(module_path, 'README.md') readme = get_readme(readme_path) doc = create_doc(module_path, files, show_extra, exclude_file, readme) + toc = create_toc(readme) + tmp = render_doc(readme, doc.content) + final = render_toc(tmp, toc) if replace: - replace_doc(readme_path, doc.content, readme) + with open(readme_path, 'w') as f: + f.write(final) else: - print(doc) + print(final) if __name__ == '__main__':