Add pre-commit hook configuration (#2326)

* Pre-commit config

Run following linters on commit:
Terraform:
- terraform fmt
- terraform tflint

Python specific:
- yapf

Shell scripts
- shellcheck
- shfmt

YAML files:
- yamllint (disabled as of now)
- check-yaml

Other:
- end-of-file-fixer
- trailing-whitespace fixer

Fabric specific
- tools/tfdoc.py
- tools/check_boilerplate.py

* linting fixes

* Fix boilerplate check

.pre-commit-config.yaml

@@ -0,0 +1,82 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+repos:
+- repo: https://github.com/antonbabenko/pre-commit-terraform
+  rev: v1.86.0
+  hooks:
+  - id: terraform_fmt
+  - id: terraform_tflint
+    args:
+    - --args=--config=__GIT_WORKING_DIR__/.tflint.hcl
+    files: ^modules/
+    exclude: (tests/fixtures/|tools/lockfile/)
+  # - id: terraform_validate
+  #   args:
+  #   - --hook-config=--retry-once-with-cleanup=true
+- repo: local
+  hooks:
+  - id: cff-readme
+    name: Regenerate README.md with tfdoc.py
+    entry: tools/pre-commit-tfdoc.sh
+    language: script
+    #types: [terraform]
+    files: ^(modules|fast).*(tf|README.md)$
+    pass_filenames: true
+    require_serial: true
+  - id: licenese
+    name: Check license presence and other boilerplate checks
+    language: system
+    entry: tools/check_boilerplate.py
+    pass_filenames: true
+    args:
+    -  --scan-files
+  - id: tflint-fast
+    name: Checking FAST code with tflint
+    entry: tools/tflint-fast.py
+    language: system
+    pass_filenames: false
+    require_serial: true
+    files: ^fast/.*tf
+
+# - repo: https://github.com/adrienverge/yamllint
+#   rev: v1.34.0
+#   hooks:
+#   - id: yamllint
+#     args: [-c=.yamllint, --no-warnings]
+#     exclude: (/templates/|modules/cloud-config-container/)
+
+- repo: https://github.com/jumanjihouse/pre-commit-hooks
+  rev: "3.0.0"
+  hooks:
+  - id: script-must-have-extension
+  - id: shellcheck
+  - id: shfmt
+    exclude: ".*tpl"
+
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.5.0
+  hooks:
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+  - id: check-yaml
+    args:
+    - --allow-multiple-documents
+    exclude: (/templates/|modules/cloud-config-container/)
+
+- repo: https://github.com/google/yapf/
+  rev: v0.40.2
+  hooks:
+  - id: yapf

.yamllint

@@ -0,0 +1,16 @@
+---
+
+yaml-files:
+- '*.yaml'
+- '*.yml'
+- '.yamllint'
+
+extends: default
+
+rules:
+  indentation:
+    indent-sequences: consistent
+  line-length:
+    max: 120
+    level: warning
+  braces: disable

blueprints/apigee/bigquery-analytics/send-requests.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -11,18 +13,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
- 
-#!/bin/bash
 
 if [ $# -lt 2 ]; then
-    echo "Usage: $0 ENVGROUP_HOSTNAME NUM_REQUESTS"
+	echo "Usage: $0 ENVGROUP_HOSTNAME NUM_REQUESTS"
-    exit 1
+	exit 1
-fi    
+fi
 
 ENVGROUP_HOSTNAME=$1
 NUM_REQUESTS=$2
 
-for i in $(seq 1 $NUM_REQUESTS)
+# shellcheck disable=SC2034
-do
+for i in $(seq 1 "$NUM_REQUESTS"); do
-curl -v https://$ENVGROUP_HOSTNAME/httpbin/headers
+	curl -v "https://$ENVGROUP_HOSTNAME/httpbin/headers"
 done

blueprints/cloud-operations/workload-identity-federation/setup.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -11,11 +13,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
- 
-#!/bin/bash
 
 sudo apt -y update
 sudo apt -y install apt-transport-https ca-certificates gnupg jq
 echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
 curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
-sudo apt-get update && sudo apt-get install google-cloud-sdk
+sudo apt-get update && sudo apt-get install google-cloud-sdk

tests/modules/artifact_registry/examples/cleanup-policies.yaml

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-values:
 values:
   module.registry-docker.google_artifact_registry_repository.registry:
     cleanup_policies:
@@ -38,7 +37,7 @@ values:
     mode: STANDARD_REPOSITORY
     project: project-id
     repository_id: docker-cleanup-policies
-    
+
 
 counts:
   google_artifact_registry_repository: 1

tests/modules/gcs/examples/iam-authoritative.yaml

@@ -14,7 +14,6 @@
 
 values:
   module.bucket.google_storage_bucket.bucket:
-    autoclass: []
     cors: []
     custom_placement_config: []
     default_event_based_hold: null
@@ -46,4 +45,4 @@ counts:
   google_storage_bucket: 1
   google_storage_bucket_iam_binding: 1
   modules: 1
-  resources: 2
+  resources: 2

tests/modules/gcs/examples/iam-bindings-additive.yaml

@@ -14,7 +14,6 @@
 
 values:
   module.bucket.google_storage_bucket.bucket:
-    autoclass: []
     cors: []
     custom_placement_config: []
     default_event_based_hold: null
@@ -48,4 +47,4 @@ counts:
   google_storage_bucket: 1
   google_storage_bucket_iam_member: 1
   modules: 1
-  resources: 2
+  resources: 2

tests/modules/gcs/examples/iam-bindings.yaml

@@ -14,7 +14,6 @@
 
 values:
   module.bucket.google_storage_bucket.bucket:
-    autoclass: []
     cors: []
     custom_placement_config: []
     default_event_based_hold: null
@@ -49,4 +48,4 @@ counts:
   google_storage_bucket: 1
   google_storage_bucket_iam_binding: 1
   modules: 1
-  resources: 2
+  resources: 2

tests/modules/secret_manager/examples/secret-cmek.yaml

@@ -71,5 +71,3 @@ counts:
   resources: 11
 
 outputs: {}
-
-outputs: {}

tools/check_boilerplate.py

@@ -24,11 +24,12 @@ folder paths as arguments, as this tool is designed to be run in CI pipelines
 triggered by pull requests.
 '''
 
-import glob
 import os
 import re
 import sys
 
+import click
+
 _EXCLUDE_DIRS = ('.git', '.terraform')
 _EXCLUDE_RE = re.compile(r'# skip boilerplate check')
 _MATCH_FILES = ('Dockerfile', '.py', '.sh', '.tf', '.yaml', '.yml')
@@ -38,23 +39,34 @@ _MATCH_STRING = (r'^\s*[#\*]\sCopyright [0-9]{4} Google LLC$\s+[#\*]\s+'
 _MATCH_RE = re.compile(_MATCH_STRING, re.M)
 
 
-def main(base_dirs):
+def check_files(root, files, errors, warnings):
-  "Cycle through files in base_dirs and check for the Apache 2.0 boilerplate."
+  for fname in files:
+    if fname in _MATCH_FILES or os.path.splitext(fname)[1] in _MATCH_FILES:
+      fpath = os.path.abspath(os.path.join(root, fname))
+      content = open(fpath).read()
+      if _EXCLUDE_RE.search(content):
+        continue
+      try:
+        if not _MATCH_RE.search(content):
+          errors.append(fpath)
+      except (IOError, OSError):
+        warnings.append(fpath)
+
+
+@click.command()
+@click.argument('paths', type=str, nargs=-1)
+@click.option('--scan-files', default=False, is_flag=True)
+def main(paths, scan_files=False):
+  "Cycle through files in paths and check for the Apache 2.0 boilerplate."
   errors, warnings = [], []
-  for dir in base_dirs:
+  if scan_files:
-    for root, dirs, files in os.walk(dir):
+    check_files("./", paths, errors, warnings)
-      dirs[:] = [d for d in dirs if d not in _EXCLUDE_DIRS]
+  else:
-      for fname in files:
+    for dir in paths:
-        if fname in _MATCH_FILES or os.path.splitext(fname)[1] in _MATCH_FILES:
+      for root, dirs, files in os.walk(dir):
-          fpath = os.path.abspath(os.path.join(root, fname))
+        dirs[:] = [d for d in dirs if d not in _EXCLUDE_DIRS]
-          content = open(fpath).read()
+        check_files(root, files, errors, warnings)
-          if _EXCLUDE_RE.search(content):
+
-            continue
-          try:
-            if not _MATCH_RE.search(content):
-              errors.append(fpath)
-          except (IOError, OSError):
-            warnings.append(fpath)
   if warnings:
     print('The following files cannot be accessed:')
     print('\n'.join(' - {}'.format(s) for s in warnings))
@@ -65,6 +77,4 @@ def main(base_dirs):
 
 
 if __name__ == '__main__':
-  if len(sys.argv) < 2:
+  main()
-    raise SystemExit('No directory to check.')
-  main(sys.argv[1:])

tools/pre-commit-tfdoc.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+
+files=("$@")
+declare -A directories
+
+for file in "${files[@]}"; do
+	dir=$(dirname "${file}")
+	if [ -f "${dir}/README.md" ] && [ -f "${dir}/main.tf" ]; then
+		directories["${dir}"]=1
+	fi
+done
+
+for dir in "${!directories[@]}"; do # iterate over keys in directories
+	echo python "${SCRIPT_DIR}/tfdoc.py" "${dir}"
+	python "${SCRIPT_DIR}/tfdoc.py" "${dir}"
+done

tools/tflint-fast.py

@@ -42,8 +42,11 @@ def main(junit):
         args += ['--format=junit']
       args += [
           '--chdir',
-          str((BASEDIR / module_path).absolute()), '--var-file',
+          str((BASEDIR / module_path).absolute()),
-          str((BASEDIR / var_path).absolute())
+          '--var-file',
+          str((BASEDIR / var_path).absolute()),
+          '--config',
+          str((BASEDIR / ".tflint.hcl").absolute()),
       ]
       print(' '.join(args))
       if junit: