Introduce test isolation and fix missing GCS service account

modules/cloudsql-instance/README.md

@@ -77,7 +77,7 @@ module "db" {
   gcp_deletion_protection       = false
   terraform_deletion_protection = false
 }
-# tftest modules=3 resources=15 inventory=simple.yaml e2e
+# tftest modules=3 resources=15 inventory=simple.yaml isolated e2e
 ```
 
 ### Cross-regional read replica
@@ -227,7 +227,7 @@ module "db" {
   terraform_deletion_protection = false
 }
 
-# tftest modules=4 resources=21 e2e
+# tftest modules=4 resources=21 isolated e2e
 ```
 
 ### Instance with PSC enabled

modules/net-vpc/README.md

@@ -304,7 +304,7 @@ module "vpc" {
     }
   ]
 }
-# tftest inventory=psa-prefix-services.yaml e2e
+# tftest inventory=psa-prefix-services.yaml isolated e2e
 ```
 
 ### Private Service Networking with peering routes and peered Cloud DNS domains
@@ -360,7 +360,7 @@ module "vpc" {
     }
   ]
 }
-# tftest modules=1 resources=10 inventory=psa-multiple-providers.yaml e2e
+# tftest modules=1 resources=10 inventory=psa-multiple-providers.yaml isolated e2e
 ```
 
 ### Subnets for Private Service Connect, Proxy-only subnets

tests/examples_e2e/conftest.py

@@ -19,4 +19,11 @@ from ..examples.conftest import \
 
 def pytest_generate_tests(metafunc):
   """Find all README.md files and collect code examples tagged for testing."""
-  _examples_generate_test(metafunc, "examples_e2e", lambda x: 'e2e' in x)
+  match metafunc.function.__name__.lower():
+  # split examples by isolated tag. Those tagged with `isolated` test with `test_isolated_examples`
+    case "test_example":
+      _examples_generate_test(metafunc, "examples_e2e", lambda x:
+                              ('e2e' in x and 'isolated' not in x))
+    case "test_isolated_example":
+      _examples_generate_test(metafunc, "examples_e2e", lambda x:
+                              ('e2e' in x and 'isolated' in x))

tests/examples_e2e/setup_module/main.tf

@@ -229,6 +229,16 @@ resource "google_project_service_identity" "jit_si" {
   depends_on = [google_project_service.project_service]
 }
 
+data "google_storage_project_service_account" "gcs_sa" {
+  project    = google_project.project.project_id
+  depends_on = [google_project_service.project_service]
+}
+
+data "google_bigquery_default_service_account" "bq_sa" {
+  project    = google_project.project.project_id
+  depends_on = [google_project_service.project_service]
+}
+
 resource "google_project_iam_binding" "agents" {
   for_each = {
     for k, v in local.jit_services : k => v if v != null

tests/examples_e2e/test_plan.py

@@ -20,7 +20,8 @@ from ..examples.utils import get_tftest_directive
 BASE_PATH = Path(__file__).parent
 
 
-def test_example(e2e_validator, tmp_path, examples_e2e, e2e_tfvars_path):
+def test_example(e2e_validator, tmp_path, examples_e2e,
+                 e2e_tfvars_path_session):
   (tmp_path / 'fabric').symlink_to(BASE_PATH.parents[1])
   (tmp_path / 'variables.tf').symlink_to(BASE_PATH.parent / 'examples' /
                                          'variables.tf')
@@ -29,7 +30,7 @@ def test_example(e2e_validator, tmp_path, examples_e2e, e2e_tfvars_path):
       '-', '_') / 'assets'
   if assets_path.exists():
     (tmp_path / 'assets').symlink_to(assets_path)
-  (tmp_path / 'terraform.tfvars').symlink_to(e2e_tfvars_path)
+  (tmp_path / 'terraform.tfvars').symlink_to(e2e_tfvars_path_session)
 
   # add files the same way as it is done for examples
   directive = get_tftest_directive(examples_e2e.code)
@@ -39,3 +40,14 @@ def test_example(e2e_validator, tmp_path, examples_e2e, e2e_tfvars_path):
 
   e2e_validator(module_path=tmp_path, extra_files=[],
                 tf_var_files=[(tmp_path / 'terraform.tfvars')])
+
+
+# use a function scoped fixture, so for each test gets a brand-new test project
+# Some tests (especially PSA), which use abandon strategy when removing, leave the project in
+# unclean state, which may prevent successful completion of the next test.
+# This allows marking such cases as isolated, and those tests will get a separate project, which won't be reused
+# for other tests
+def test_isolated_example(e2e_validator, tmp_path, examples_e2e,
+                          e2e_tfvars_path_function):
+  return test_example(e2e_validator, tmp_path, examples_e2e,
+                      e2e_tfvars_path_function)

tests/fixtures.py

@@ -391,7 +391,24 @@ def e2e_validator_fixture(request):
   return inner
 
 
-@pytest.fixture(scope='session', name='e2e_tfvars_path')
+@pytest.fixture(scope='session', name='e2e_tfvars_path_session')
+def e2e_tfvars_path_session():
+  """Session scoped fixture preparing end-to-end environment
+
+  Creates a GCP project for each thread. Tests reuse the same GCP project.
+  """
+  yield from e2e_tfvars_path()
+
+
+@pytest.fixture(scope='function', name='e2e_tfvars_path_function')
+def e2e_tfvars_path_function():
+  """Function scoped fixture preparing end-to-end environment
+
+  Creates a separate GCP project for each of E2E test run.
+  """
+  yield from e2e_tfvars_path()
+
+
 def e2e_tfvars_path():
   """Fixture preparing end-to-end test environment