From 4b798fb34da6a0bb3e9952866b8a4c1f14ae27cb Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 14 Oct 2022 14:56:16 +0200 Subject: [PATCH] Run tests in parallel using `pytest-xdist` (#881) * test terraform cache * try pytest-xdist * revert cache changes * extend to other jobs * change dist, bump processes to 4 * revert * mark tests * run init to prime providers cache * prime providers cache * prime providers cache for all jobs * add local provider to versions * remove leftover code --- .github/workflows/tests.yml | 44 +++++++++++++++++++++++--- default-versions.tf | 4 +++ tests/blueprints/conftest.py | 21 ++++++++++++ tests/examples/conftest.py | 4 ++- tests/fast/conftest.py | 20 ++++++++++++ tests/modules/api_gateway/test_plan.py | 11 ++----- tests/modules/conftest.py | 20 ++++++++++++ tests/requirements.txt | 1 + 8 files changed, 111 insertions(+), 14 deletions(-) create mode 100644 tests/blueprints/conftest.py create mode 100644 tests/fast/conftest.py create mode 100644 tests/modules/conftest.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2a9035253..114f04fde 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,6 +54,15 @@ jobs: terraform_version: ${{ env.TF_VERSION }} terraform_wrapper: false + - name: Config Terraform plugin cache + run: | + echo 'plugin_cache_dir="$HOME/.terraform.d/plugin-cache"' >~/.terraformrc + mkdir --parents ~/.terraform.d/plugin-cache + + - name: Initialize providers + run: | + terraform init + # avoid conflicts with user-installed providers on local machines - name: Pin provider versions run: | @@ -66,7 +75,7 @@ jobs: run: | mkdir -p ${{ env.TF_PLUGIN_CACHE_DIR }} pip install -r tests/requirements.txt - pytest -vv tests/examples + pytest -vv --numprocesses auto --dist loadgroup tests/examples examples: runs-on: ubuntu-latest @@ -89,6 +98,15 @@ jobs: terraform_version: ${{ env.TF_VERSION }} terraform_wrapper: false + - name: Config Terraform plugin cache + run: | + echo 'plugin_cache_dir="$HOME/.terraform.d/plugin-cache"' >~/.terraformrc + mkdir --parents ~/.terraform.d/plugin-cache + + - name: Initialize providers + run: | + terraform init + # avoid conflicts with user-installed providers on local machines - name: Pin provider versions run: | @@ -101,7 +119,7 @@ jobs: run: | mkdir -p ${{ env.TF_PLUGIN_CACHE_DIR }} pip install -r tests/requirements.txt - pytest -vv tests/blueprints + pytest -vv --numprocesses auto --dist loadgroup tests/blueprints modules: runs-on: ubuntu-latest @@ -124,6 +142,15 @@ jobs: terraform_version: ${{ env.TF_VERSION }} terraform_wrapper: false + - name: Config Terraform plugin cache + run: | + echo 'plugin_cache_dir="$HOME/.terraform.d/plugin-cache"' >~/.terraformrc + mkdir --parents ~/.terraform.d/plugin-cache + + - name: Initialize providers + run: | + terraform init + # avoid conflicts with user-installed providers on local machines - name: Pin provider versions run: | @@ -136,7 +163,7 @@ jobs: run: | mkdir -p ${{ env.TF_PLUGIN_CACHE_DIR }} pip install -r tests/requirements.txt - pytest -vv tests/modules + pytest -vv --numprocesses auto --dist loadgroup tests/modules fast: runs-on: ubuntu-latest @@ -159,6 +186,15 @@ jobs: terraform_version: ${{ env.TF_VERSION }} terraform_wrapper: false + - name: Config Terraform plugin cache + run: | + echo 'plugin_cache_dir="$HOME/.terraform.d/plugin-cache"' >~/.terraformrc + mkdir --parents ~/.terraform.d/plugin-cache + + - name: Initialize providers + run: | + terraform init + # avoid conflicts with user-installed providers on local machines - name: Pin provider versions run: | @@ -171,4 +207,4 @@ jobs: run: | mkdir -p ${{ env.TF_PLUGIN_CACHE_DIR }} pip install -r tests/requirements.txt - pytest -vv tests/fast + pytest -vv --numprocesses auto --dist loadgroup tests/fast diff --git a/default-versions.tf b/default-versions.tf index b1c8c910d..b2e7c1c3a 100644 --- a/default-versions.tf +++ b/default-versions.tf @@ -23,6 +23,10 @@ terraform { source = "hashicorp/google-beta" version = ">= 4.36.0" # tftest } + # used in fast + local = { + source = "hashicorp/local" + } } } diff --git a/tests/blueprints/conftest.py b/tests/blueprints/conftest.py new file mode 100644 index 000000000..ed29d5bb6 --- /dev/null +++ b/tests/blueprints/conftest.py @@ -0,0 +1,21 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +def pytest_collection_modifyitems(config, items): + for item in items: + item.add_marker( + pytest.mark.xdist_group(name='/'.join(item.path.parent.parts[-2:]))) diff --git a/tests/examples/conftest.py b/tests/examples/conftest.py index aeb0054f0..2cea9e368 100644 --- a/tests/examples/conftest.py +++ b/tests/examples/conftest.py @@ -15,6 +15,7 @@ from pathlib import Path import marko +import pytest FABRIC_ROOT = Path(__file__).parents[2] MODULES_PATH = FABRIC_ROOT / 'modules/' @@ -36,13 +37,14 @@ def pytest_generate_tests(metafunc): doc = marko.parse(readme.read_text()) index = 0 last_header = None + mark = pytest.mark.xdist_group(name=module.name) for child in doc.children: if isinstance(child, marko.block.FencedCode) and child.lang == 'hcl': index += 1 code = child.children[0].children if 'tftest skip' in code: continue - examples.append(code) + examples.append(pytest.param(code, marks=mark)) path = module.relative_to(FABRIC_ROOT) name = f'{path}:{last_header}' if index > 1: diff --git a/tests/fast/conftest.py b/tests/fast/conftest.py new file mode 100644 index 000000000..3d559e6f0 --- /dev/null +++ b/tests/fast/conftest.py @@ -0,0 +1,20 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +def pytest_collection_modifyitems(config, items): + for item in items: + item.add_marker(pytest.mark.xdist_group(name=item.path.parent.name)) diff --git a/tests/modules/api_gateway/test_plan.py b/tests/modules/api_gateway/test_plan.py index 0e94cdec9..18ecdd329 100644 --- a/tests/modules/api_gateway/test_plan.py +++ b/tests/modules/api_gateway/test_plan.py @@ -12,15 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -@pytest.fixture -def resources(plan_runner): - _, resources = plan_runner() - return resources - - -def test_resource_count(resources): +def test_resource_count(plan_runner): "Test number of resources created." + _, resources = plan_runner() assert len(resources) == 5 diff --git a/tests/modules/conftest.py b/tests/modules/conftest.py new file mode 100644 index 000000000..3d559e6f0 --- /dev/null +++ b/tests/modules/conftest.py @@ -0,0 +1,20 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +def pytest_collection_modifyitems(config, items): + for item in items: + item.add_marker(pytest.mark.xdist_group(name=item.path.parent.name)) diff --git a/tests/requirements.txt b/tests/requirements.txt index e98216cc8..931b17307 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,5 @@ pytest>=6.2.5 +pytest-xdist PyYAML>=6.0 tftest>=1.6.3 marko>=1.2.0