From e7dd12fa29c6af9ef20587a560c41ba2e6bf544c Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Mon, 14 Feb 2022 12:13:42 +0100 Subject: [PATCH] Reformat all code using yapf --- .github/workflows/linting.yml | 7 +- tools/check_boilerplate.py | 14 +-- tools/check_documentation.py | 11 +-- tools/check_links.py | 7 +- tools/check_names.py | 18 ++-- tools/{REQUIREMENTS.txt => requirements.txt} | 1 + tools/state_iam.py | 32 ++++--- tools/tfdoc.py | 93 +++++++------------- tools/validate_schema.py | 15 ++-- 9 files changed, 78 insertions(+), 120 deletions(-) rename tools/{REQUIREMENTS.txt => requirements.txt} (79%) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 080422e7b..26a7b6521 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -41,7 +41,7 @@ jobs: - name: Install dependencies run: | - pip install -r tools/REQUIREMENTS.txt + pip install -r tools/requirements.txt - name: Boilerplate id: boilerplate @@ -67,3 +67,8 @@ jobs: id: name-length-fast run: | python3 tools/check_names.py --prefix-length=10 fast/stages + + - name: Check python formatting + id: yapf + run: | + yapf --style="{based_on_style: google, indent_width: 2, SPLIT_BEFORE_NAMED_ASSIGNS: false}" -p -d tools/*.py diff --git a/tools/check_boilerplate.py b/tools/check_boilerplate.py index 2095e0482..839b19524 100755 --- a/tools/check_boilerplate.py +++ b/tools/check_boilerplate.py @@ -13,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - '''Check that boilerplate is present in relevant files. This tools offers a simple way of ensuring that the required boilerplate header @@ -30,17 +29,12 @@ import os import re import sys - _EXCLUDE_DIRS = ('.git', '.terraform') _EXCLUDE_RE = re.compile(r'# skip boilerplate check') -_MATCH_FILES = ( - 'Dockerfile', '.py', '.sh', '.tf', '.yaml', '.yml' -) -_MATCH_STRING = ( - r'^\s*[#\*]\sCopyright [0-9]{4} Google LLC$\s+[#\*]\s+' - r'[#\*]\sLicensed under the Apache License, Version 2.0 ' - r'\(the "License"\);\s+' -) +_MATCH_FILES = ('Dockerfile', '.py', '.sh', '.tf', '.yaml', '.yml') +_MATCH_STRING = (r'^\s*[#\*]\sCopyright [0-9]{4} Google LLC$\s+[#\*]\s+' + r'[#\*]\sLicensed under the Apache License, Version 2.0 ' + r'\(the "License"\);\s+') _MATCH_RE = re.compile(_MATCH_STRING, re.M) diff --git a/tools/check_documentation.py b/tools/check_documentation.py index 215838146..1902570c1 100755 --- a/tools/check_documentation.py +++ b/tools/check_documentation.py @@ -13,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - '''Recursively check freshness of tfdoc's generated tables in README files. This tool recursively checks that the embedded variables and outputs tables in @@ -29,10 +28,8 @@ import pathlib import click import tfdoc - BASEDIR = pathlib.Path(__file__).resolve().parents[1] - State = enum.Enum('State', 'OK FAIL SKIP') @@ -59,11 +56,9 @@ def _check_dir(dir_name, exclude_files=None, files=False, show_extra=False): state = State.OK else: state = State.FAIL - diff = '\n'.join( - [f'----- {mod_name} diff -----\n'] + - list(difflib.ndiff( - result['doc'].split('\n'), new_doc.split('\n') - ))) + header = f'----- {mod_name} diff -----\n' + ndiff = difflib.ndiff(result['doc'].split('\n'), new_doc.split('\n')) + diff = '\n'.join([header] + list(ndiff)) yield mod_name, state, diff diff --git a/tools/check_links.py b/tools/check_links.py index a0185d0f7..8fc57a9aa 100755 --- a/tools/check_links.py +++ b/tools/check_links.py @@ -13,14 +13,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - '''Recursively check link destination validity in Markdown files. This tool recursively checks that local links in Markdown files point to valid destinations. Its main use is in CI pipelines triggered by pull requests. ''' - import collections import pathlib import urllib.parse @@ -28,7 +26,6 @@ import urllib.parse import click import marko - BASEDIR = pathlib.Path(__file__).resolve().parents[1] DOC = collections.namedtuple('DOC', 'path relpath links') LINK = collections.namedtuple('LINK', 'dest valid') @@ -57,8 +54,8 @@ def check_docs(dir_name): yield DOC(readme_path, str(readme_path.relative_to(dir_path)), links) -@ click.command() -@ click.argument('dirs', type=str, nargs=-1) +@click.command() +@click.argument('dirs', type=str, nargs=-1) def main(dirs): 'Check links in Markdown files contained in dirs.' errors = 0 diff --git a/tools/check_names.py b/tools/check_names.py index c487d6934..7e45fd877 100755 --- a/tools/check_names.py +++ b/tools/check_names.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - 'Parse names from specific Terraform resources and optionally check length.' import collections @@ -23,20 +22,17 @@ import re import click - BASEDIR = pathlib.Path(__file__).resolve().parents[1] LOGGER = logging.getLogger() MOD_TOKENS = [ ('NAME', r'\s*module\s*"([^"]+)"\s*\{\s*'), ('SOURCE', r'\s*source\s*=\s*"([^"]+)"\s*'), ('VALUE', r'\s*name\s*=\s*"([^"]+)"\s*'), - ('REST', r'(.*)') + ('REST', r'(.*)'), ] MOD = enum.Enum('MOD', ' '.join(name for name, _ in MOD_TOKENS)) MOD_RE = re.compile('|'.join(f'(?:{pattern})' for _, pattern in MOD_TOKENS)) -MOD_LIMITS = { - 'project': 30, 'iam-service-account': 30, 'gcs': 63 -} +MOD_LIMITS = {'project': 30, 'iam-service-account': 30, 'gcs': 63} Name = collections.namedtuple('Name', 'source name value length') @@ -91,12 +87,10 @@ def main(dirs, prefix_length=None): for name in names: name_length = name.length + prefix_length flag = '✗' if name_length >= MOD_LIMITS[name.source] else '✓' - print(( - f'[{flag}] {name.source.ljust(source_just)} ' - f'{name.name.ljust(name_just)} ' - f'{name.value.ljust(value_just)} ' - f'({name_length})' - )) + print(f'[{flag}] {name.source.ljust(source_just)} ' + f'{name.name.ljust(name_just)} ' + f'{name.value.ljust(value_just)} ' + f'({name_length})') if __name__ == '__main__': diff --git a/tools/REQUIREMENTS.txt b/tools/requirements.txt similarity index 79% rename from tools/REQUIREMENTS.txt rename to tools/requirements.txt index 84827ce5b..5aa1448c3 100644 --- a/tools/REQUIREMENTS.txt +++ b/tools/requirements.txt @@ -1,3 +1,4 @@ click marko yamale +yapf diff --git a/tools/state_iam.py b/tools/state_iam.py index 83360179c..7817a7ee3 100755 --- a/tools/state_iam.py +++ b/tools/state_iam.py @@ -22,11 +22,8 @@ import sys import click - -FIELDS = ( - 'authoritative', 'resource_type', 'resource_id', 'role', 'member_type', - 'member_id', 'conditions' -) +FIELDS = ('authoritative', 'resource_type', 'resource_id', 'role', + 'member_type', 'member_id', 'conditions') ORG_IDS = {} RESOURCE_SORT = {'organization': 0, 'folder': 1, 'project': 2} RESOURCE_TYPE_RE = re.compile(r'^google_([^_]+)_iam_([^_]+)$') @@ -111,20 +108,17 @@ def output_principals(bindings): if b.role.startswith('organizations/'): roles.append(f'{b.role} {additive}{conditions}') else: - url = ( - 'https://cloud.google.com/iam/docs/understanding-roles#' - f'{b.role.replace("roles/", "")}' - ) + url = ('https://cloud.google.com/iam/docs/understanding-roles#' + f'{b.role.replace("roles/", "")}') roles.append(f'[{b.role}]({url}) {additive}{conditions}') - print(( - f'|{principal[1]}
{principal[0]}|' - f'{"
".join(roles)}|' - )) + print(f'|{principal[1]}
{principal[0]}|' + f'{"
".join(roles)}|') @click.command() @click.argument('state-file', type=click.File('r'), default=sys.stdin) -@click.option('--format', type=click.Choice(['csv', 'principals', 'raw']), default='raw') +@click.option('--format', type=click.Choice(['csv', 'principals', 'raw']), + default='raw') @click.option('--prefix', default=None) def main(state_file, format, prefix=None): 'Output IAM bindings parsed from Terraform state file or standard input.' @@ -133,9 +127,13 @@ def main(state_file, format, prefix=None): resources = data.get('resources', []) folders = dict(get_folders(resources)) bindings = get_bindings(resources, prefix=prefix, folders=folders) - bindings = sorted(bindings, key=lambda b: ( - RESOURCE_SORT.get(b.resource_type, 99), b.resource_id, - b.member_type, b.member_id)) + bindings = sorted( + bindings, key=lambda b: ( + RESOURCE_SORT.get(b.resource_type, 99), + b.resource_id, + b.member_type, + b.member_id, + )) if format == 'raw': for b in bindings: print(b) diff --git a/tools/tfdoc.py b/tools/tfdoc.py index e987e6fca..a142d5203 100755 --- a/tools/tfdoc.py +++ b/tools/tfdoc.py @@ -13,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - '''Generate tables for Terraform root module files, outputs and variables. This tool generates nicely formatted Markdown tables from Terraform source @@ -48,13 +47,10 @@ import urllib.parse import click - __version__ = '2.1.0' - # TODO(ludomagno): decide if we want to support variables*.tf and outputs*.tf - FILE_DESC_DEFAULTS = { 'main.tf': 'Module-level locals and resources.', 'outputs.tf': 'Module outputs.', @@ -63,11 +59,8 @@ FILE_DESC_DEFAULTS = { 'versions.tf': 'Version pins.', } FILE_RE_MODULES = re.compile( - r'(?sm)module\s*"[^"]+"\s*\{[^\}]*?source\s*=\s*"([^"]+)"' -) -FILE_RE_RESOURCES = re.compile( - r'(?sm)resource\s*"([^"]+)"' -) + r'(?sm)module\s*"[^"]+"\s*\{[^\}]*?source\s*=\s*"([^"]+)"') +FILE_RE_RESOURCES = re.compile(r'(?sm)resource\s*"([^"]+)"') HEREDOC_RE = re.compile(r'(?sm)^<<\-?END(\s*.*?)\s*END$') MARK_BEGIN = '' MARK_END = '' @@ -88,8 +81,7 @@ OUT_RE = re.compile(r'''(?smx) OUT_TEMPLATE = ('description', 'value', 'sensitive') TAG_RE = re.compile(r'(?sm)^\s*#\stfdoc:([^:]+:\S+)\s+(.*?)\s*$') UNESCAPED = string.digits + string.ascii_letters + ' .,;:_-' -VAR_ENUM = enum.Enum( - 'V', 'OPEN ATTR ATTR_DATA SKIP CLOSE COMMENT TXT') +VAR_ENUM = enum.Enum('V', 'OPEN ATTR ATTR_DATA SKIP CLOSE COMMENT TXT') VAR_RE = re.compile(r'''(?smx) # variable open (?:^\s*variable\s*"([^"]+)"\s*\{\s*$) | @@ -107,14 +99,12 @@ VAR_RE = re.compile(r'''(?smx) VAR_RE_TYPE = re.compile(r'([\(\{\}\)])') VAR_TEMPLATE = ('default', 'description', 'type', 'nullable') - File = collections.namedtuple('File', 'name description modules resources') Output = collections.namedtuple('Output', 'name description sensitive consumers line') Variable = collections.namedtuple( 'Variable', 'name description type default required nullable source line') - # parsing functions @@ -144,7 +134,7 @@ def _parse(body, enum=VAR_ENUM, re=VAR_RE, template=VAR_TEMPLATE): elif token == enum.ATTR_DATA: if not item: continue - context = m.group(m.lastindex-1) + context = m.group(m.lastindex - 1) item[context].append(data) elif token == enum.SKIP: context = token @@ -170,12 +160,11 @@ def parse_files(basepath, exclude_files=None): except (IOError, OSError) as e: raise SystemExit(f'Cannot read file {name}: {e}') tags = _extract_tags(body) - description = tags.get( - 'file:description', FILE_DESC_DEFAULTS.get(shortname)) + description = tags.get('file:description', + FILE_DESC_DEFAULTS.get(shortname)) modules = set( os.path.basename(urllib.parse.urlparse(m).path) - for m in FILE_RE_MODULES.findall(body) - ) + for m in FILE_RE_MODULES.findall(body)) resources = set(FILE_RE_RESOURCES.findall(body)) yield File(shortname, description, modules, resources) @@ -188,11 +177,11 @@ def parse_outputs(basepath): except (IOError, OSError) as e: raise SystemExit(f'No outputs file in {basepath}.') for item in _parse(body, enum=OUT_ENUM, re=OUT_RE, template=OUT_TEMPLATE): - yield Output(name=item['name'], - description=''.join(item['description']), - sensitive=item['sensitive'] != [], - consumers=item['tags'].get('output:consumers', ''), - line=item['line']) + description = ''.join(item['description']) + sensitive = item['sensitive'] != [] + consumers = item['tags'].get('output:consumers', '') + yield Output(name=item['name'], description=description, + sensitive=sensitive, consumers=consumers, line=item['line']) def parse_variables(basepath): @@ -203,20 +192,18 @@ def parse_variables(basepath): except (IOError, OSError) as e: raise SystemExit(f'No variables file in {basepath}.') for item in _parse(body): + description = ''.join(item['description']) + vtype = '\n'.join(item['type']) default = HEREDOC_RE.sub(r'\1', '\n'.join(item['default'])) required = not item['default'] - vtype = '\n'.join(item['type']) nullable = item.get('nullable') != ['false'] + source = item['tags'].get('variable:source', '') if not required and default != 'null' and vtype == 'string': default = f'"{default}"' - yield Variable(name=item['name'], - description=''.join(item['description']), - type=vtype, - default=default, - required=required, - source=item['tags'].get('variable:source', ''), - line=item['line'], - nullable=nullable) + + yield Variable(name=item['name'], description=description, type=vtype, + default=default, required=required, source=source, + line=item['line'], nullable=nullable) # formatting functions @@ -251,25 +238,19 @@ def format_files(items): num_resources = sum(len(i.resources) for i in items) yield '| name | description |{}{}'.format( ' modules |' if num_modules else '', - ' resources |' if num_resources else '' - ) - yield '|---|---|{}{}'.format( - '---|' if num_modules else '', - '---|' if num_resources else '' - ) + ' resources |' if num_resources else '') + yield '|---|---|{}{}'.format('---|' if num_modules else '', + '---|' if num_resources else '') for i in items: modules = resources = '' if i.modules: - modules = '%s' % ' · '.join( - sorted(i.modules)) + modules = '%s' % ' · '.join(sorted(i.modules)) if i.resources: resources = '%s' % ' · '.join( sorted(i.resources)) yield '| [{}](./{}) | {} |{}{}'.format( - i.name, i.name, i.description, - f' {modules} |' if num_modules else '', - f' {resources} |' if num_resources else '' - ) + i.name, i.name, i.description, f' {modules} |' if num_modules else '', + f' {resources} |' if num_resources else '') def format_outputs(items, show_extra=True): @@ -277,17 +258,13 @@ def format_outputs(items, show_extra=True): if not items: return items.sort(key=lambda i: i.name) - yield '| name | description | sensitive |' + ( - ' consumers |' if show_extra else '' - ) - yield '|---|---|:---:|' + ( - '---|' if show_extra else '' - ) + yield '| name | description | sensitive |' + (' consumers |' + if show_extra else '') + yield '|---|---|:---:|' + ('---|' if show_extra else '') for i in items: consumers = i.consumers or '' if consumers: - consumers = '%s' % ' · '.join( - consumers.split()) + consumers = '%s' % ' · '.join(consumers.split()) sensitive = '✓' if i.sensitive else '' format = f'| [{i.name}](outputs.tf#L{i.line}) | {i.description or ""} | {sensitive} |' format += f' {consumers} |' if show_extra else '' @@ -301,11 +278,8 @@ def format_variables(items, show_extra=True): items.sort(key=lambda i: i.name) items.sort(key=lambda i: i.required, reverse=True) yield '| name | description | type | required | default |' + ( - ' producer |' if show_extra else '' - ) - yield '|---|---|:---:|:---:|:---:|' + ( - ':---:|' if show_extra else '' - ) + ' producer |' if show_extra else '') + yield '|---|---|:---:|:---:|:---:|' + (':---:|' if show_extra else '') for i in items: vars = { 'default': f'{_escape(i.default)}' if i.default else '', @@ -326,8 +300,7 @@ def format_variables(items, show_extra=True): vars[k] = f'{_escape(value)}' format = ( f'| [{i.name}](variables.tf#L{i.line}) | {i.description or ""} | {vars["type"]} ' - f'| {vars["required"]} | {vars["default"]} |' - ) + f'| {vars["required"]} | {vars["default"]} |') format += f' {vars["source"]} |' if show_extra else '' yield format @@ -396,7 +369,7 @@ def replace_doc(readme_path, doc, readme=None): MARK_BEGIN, doc, MARK_END, - readme[result['end']:].lstrip() + readme[result['end']:].lstrip(), ])) except (IOError, OSError) as e: raise SystemExit(f'Error replacing README {readme_path}: {e}') diff --git a/tools/validate_schema.py b/tools/validate_schema.py index b1a9020fd..7556667b7 100755 --- a/tools/validate_schema.py +++ b/tools/validate_schema.py @@ -13,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - '''Validate YaML document against yamale schemas. Fast includes YaML driven resource factories, along with their schemas which are available at `fast/assets/schemas`. @@ -29,12 +28,14 @@ import click import yamale -@ click.command() -@ click.argument('schema', type=click.Path(exists=True)) -@ click.option('--directory', multiple=True, type=click.Path(exists=True, file_okay=False, dir_okay=True)) -@ click.option('--file', multiple=True, type=click.Path(exists=True, file_okay=True, dir_okay=False)) -@ click.option('--recursive', is_flag=True, default=False) -@ click.option('--quiet', is_flag=True, default=False) +@click.command() +@click.argument('schema', type=click.Path(exists=True)) +@click.option('--directory', multiple=True, + type=click.Path(exists=True, file_okay=False, dir_okay=True)) +@click.option('--file', multiple=True, + type=click.Path(exists=True, file_okay=True, dir_okay=False)) +@click.option('--recursive', is_flag=True, default=False) +@click.option('--quiet', is_flag=True, default=False) def main(directory=None, file=None, schema=None, recursive=False, quiet=False): 'Program entry point.'