End-to-end tests for terraform modules (#1751)

Add end-to-end tests (apply, plan, destroy) for examples.

When run, `tests/examples_e2e`:
1. Create an environment for tests to run (folder, project vpc network) 
2. For each marked example (with `e2e` tftest directive), run apply, plan, destroy
3. Verify:
* no failure in apply
* empty plan after apply
* no failure during destroy
4. When all tests are done, destroy test environment

More details in `tests/examples_e2e/README.md`

modules/cloud-function-v1/README.md

@@ -33,15 +33,15 @@ This deploys a Cloud Function with an HTTP endpoint, using a pre-existing GCS bu
 ```hcl
 module "cf-http" {
   source      = "./fabric/modules/cloud-function-v1"
-  project_id  = "my-project"
+  project_id  = var.project_id
   name        = "test-cf-http"
-  bucket_name = "test-cf-bundles"
+  bucket_name = var.bucket
   bundle_config = {
-    source_dir  = "fabric/assets/"
+    source_dir  = "assets/sample-function/"
     output_path = "bundle.zip"
   }
 }
-# tftest modules=1 resources=2
+# tftest modules=1 resources=2 e2e
 ```
 
 ### PubSub and non-HTTP triggers

modules/cloud-run/README.md

@@ -28,7 +28,7 @@ IAM bindings support the usual syntax. Container environment values can be decla
 ```hcl
 module "cloud_run" {
   source     = "./fabric/modules/cloud-run"
-  project_id = "my-project"
+  project_id = var.project_id
   name       = "hello"
   containers = {
     hello = {
@@ -49,7 +49,7 @@ module "cloud_run" {
     "roles/run.invoker" = ["allUsers"]
   }
 }
-# tftest modules=1 resources=2 inventory=simple.yaml
+# tftest modules=1 resources=2 inventory=simple.yaml e2e
 ```
 
 ### Mounting secrets as volumes

modules/net-lb-int/README.md

@@ -248,7 +248,7 @@ module "instance-group" {
   source     = "./fabric/modules/compute-vm"
   for_each   = toset(["b", "c"])
   project_id = var.project_id
-  zone       = "europe-west1-${each.key}"
+  zone       = "${var.region}-${each.key}"
   name       = "ilb-test-${each.key}"
   network_interfaces = [{
     network    = var.vpc.self_link
@@ -273,7 +273,7 @@ module "instance-group" {
 module "ilb" {
   source        = "./fabric/modules/net-lb-int"
   project_id    = var.project_id
-  region        = "europe-west1"
+  region        = var.region
   name          = "ilb-test"
   service_label = "ilb-test"
   vpc_config = {
@@ -296,7 +296,7 @@ module "ilb" {
     }
   }
 }
-# tftest modules=3 resources=7
+# tftest modules=3 resources=7 e2e
 ```
 <!-- BEGIN TFDOC -->
 ## Variables

tests/examples/conftest.py

@@ -27,9 +27,18 @@ Example = collections.namedtuple('Example', 'name code module files')
 File = collections.namedtuple('File', 'path content')
 
 
-def pytest_generate_tests(metafunc):
+def get_tftest_directive(s):
+  """Returns tftest directive from code block or None when directive is not found"""
+  for x in s.splitlines():
+    if x.strip().startswith("#") and 'tftest' in x:
+      return x
+  return None
+
+
+def pytest_generate_tests(metafunc, test_group='example',
+                          filter_tests=lambda x: True):
   """Find all README.md files and collect code examples tagged for testing."""
-  if 'example' in metafunc.fixturenames:
+  if test_group in metafunc.fixturenames:
     readmes = FABRIC_ROOT.glob('**/README.md')
     examples = []
     ids = []
@@ -59,7 +68,9 @@ def pytest_generate_tests(metafunc):
         if isinstance(child, marko.block.FencedCode):
           index += 1
           code = child.children[0].children
-          if 'tftest skip' in code:
+          tftest_tag = get_tftest_directive(code)
+          if tftest_tag and ('skip' in tftest_tag or
+                             not filter_tests(tftest_tag)):
             continue
           if child.lang == 'hcl':
             path = module.relative_to(FABRIC_ROOT)
@@ -72,4 +83,4 @@ def pytest_generate_tests(metafunc):
           last_header = child.children[0].children
           index = 0
 
-    metafunc.parametrize('example', examples, ids=ids)
+    metafunc.parametrize(test_group, examples, ids=ids)

tests/examples/test_plan.py

@@ -28,6 +28,10 @@ def test_example(plan_validator, tmp_path, example):
     (tmp_path / 'fabric').symlink_to(BASE_PATH.parents[1])
     (tmp_path / 'variables.tf').symlink_to(BASE_PATH / 'variables.tf')
     (tmp_path / 'main.tf').write_text(example.code)
+    assets_path = BASE_PATH.parent / str(example.module).replace('-',
+                                                                 '_') / 'assets'
+    if assets_path.exists():
+      (tmp_path / 'assets').symlink_to(assets_path)
 
     expected_modules = int(match.group(1))
     expected_resources = int(match.group(2))

tests/examples_e2e/README.md

@@ -0,0 +1,88 @@
+# Prerequisites
+Prepare following information:
+* billing account id
+* your organization id
+* parent folder under which resources will be created
+  * (you may want to disable / restore to default some organization policies under this folder) 
+* decide in which region you want to deploy (choose one, that has wide service coverage)
+* prepare a prefix, suffix and a timestamp for you (this is to provide project and other resources name uniqueness)
+* prepare service account that has necessary permissions (able to assign billing account to project, resource creation etc)
+
+# How does it work
+Each test case is provided by additional environment defined in [variables.tf](./variables.tf). This simplifies writing the examples as this follows the same structure as for non-end-to-end tests, and allows multiple, independent and concurrent runs of tests.
+
+The test environment can be provisioned automatically during the test run (which now takes ~2 minutes) and destroyed and the end, when of the tests (Option 1 below), which is targeting automated runs in CI/CD pipeline, or can be provisioned manually to reduce test time, which might be typical use case for tests run locally.
+
+# Option 1 - automatically provision and de-provision testing infrastructure
+
+## Create `e2e.tfvars` file
+```hcl
+billing_account = "123456-123456-123456"  # billing account id to associate projects
+organization_id = "1234567890" # your organization id
+parent          = "folders/1234567890"  # folder under which test resources will be created
+prefix          = "your-unique-prefix"  # unique prefix for projects
+region          = "europe-west4"  # region to use
+
+# tftest skip
+```
+And set environment variable pointing to the file:
+```bash
+export TFTEST_E2E_SETUP_TFVARS_PATH=<path to e2e.tfvars file>
+```
+
+Or set above variables in environment:
+```bash
+export TF_VAR_billing_account="123456-123456-123456"  # billing account id to associate projects
+export TF_VAR_organization_id="1234567890" # your organization id
+export TF_VAR_parent="folders/1234567890"  # folder under which test resources will be created
+export TF_VAR_prefix="your-unique-prefix"  # unique prefix for projects
+export TF_VAR_region="europe-west4"  # region to use
+```
+
+To use Service Account Impersonation, use provider environment variable
+```bash
+export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=<username>@<project-id>.iam.gserviceaccount.com
+```
+
+You can keep the prefix the same for all the tests run, the tests will add necessary suffix for subsequent runs, and in case tests are run in parallel, use separate suffix for the workers.
+# Run the tests
+```bash
+pytest tests/examples_e2e
+```
+
+# Option 2 - Provision manually test environment and use it for tests
+## Provision manually test environment
+In `tests/examples_e2e/setup_module` create `terraform.tfvars` with following values:
+```hcl
+billing_account = "123456-123456-123456"  # billing account id to associate projects
+organization_id = "1234567890"  # your organization id
+parent          = "folders/1234567890"  # folder under which test resources will be created
+prefix          = "your-unique-prefix"  # unique prefix for projects
+region          = "europe-west4"  # region to use
+suffix          = "1" # suffix, keep 1 for now
+timestamp       = "1696444185" # generate your own timestamp - will be used as a part of prefix
+# tftest skip
+```
+
+If you use service account impersonation, set `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT`
+```bash
+export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=<username>@<project-id>.iam.gserviceaccount.com
+```
+
+Provision the environment using terraform
+```bash
+(cd tests/examples_e2e/setup_module/ && terraform init && terraform apply)
+```
+
+This will generate also `tests/examples_e2e/setup_module/e2e_tests.tfvars` for you, which can be used by tests.
+
+## Setup your environment
+```bash
+export TFTEST_E2E_TFVARS_PATH=`pwd`/tests/examples_e2e/setup_module/e2e_tests.tfvars  # generated above
+```
+
+## Run tests
+Run tests using:
+```bash
+pytest tests/examples_e2e
+```

tests/examples_e2e/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/examples_e2e/conftest.py

@@ -0,0 +1,21 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Pytest configuration for testing code examples."""
+
+from ..examples.conftest import pytest_generate_tests as _examples_generate_test
+
+
+def pytest_generate_tests(metafunc):
+  """Find all README.md files and collect code examples tagged for testing."""
+  _examples_generate_test(metafunc, "examples_e2e", lambda x: 'e2e' in x)

tests/examples_e2e/setup_module/e2e_tests.tfvars.tftpl

@@ -0,0 +1,50 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bucket= "${bucket}"
+billing_account_id = "${billing_account_id}"
+kms_key= {
+    self_link = "kms_key_self_link"
+}
+organization_id = "${organization_id}"
+folder_id = "${folder_id}"
+prefix = "${prefix}"
+project_id = "${project_id}"
+region = "${region}"
+service_account = {
+    id        = "${service_account.id}"
+    email     = "${service_account.email}"
+    iam_email = "${service_account.iam_email}"
+  }
+subnet = {
+    name      = "${subnet.name}"
+    region    = "${subnet.region}"
+    cidr      = "${subnet.ip_cidr_range}"
+    self_link = "${subnet.self_link}"
+  }
+vpc = {
+    name      = "${vpc.name}"
+    self_link = "${vpc.self_link}"
+    id        = "${vpc.id}"
+  }
+
+# vpc1 =  {
+#    name      = "vpc_name"
+#    self_link = "projects/xxx/global/networks/bbb"
+#  }
+#vpc2 = {
+#    name      = "vpc2_name"
+#    self_link = "projects/xxx/global/networks/ccc"
+#  }
+zone = "${region}-a"

tests/examples_e2e/setup_module/main.tf

@@ -0,0 +1,107 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+locals {
+  prefix = "${var.prefix}-${var.timestamp}-${var.suffix}"
+  services = [
+    # trimmed down list of services, to be extended as needed
+    "cloudbuild.googleapis.com",
+    "cloudfunctions.googleapis.com",
+    "cloudresourcemanager.googleapis.com",
+    "compute.googleapis.com",
+    "iam.googleapis.com",
+    "run.googleapis.com",
+    "serviceusage.googleapis.com",
+    "stackdriver.googleapis.com",
+    "storage-component.googleapis.com",
+    "storage.googleapis.com",
+  ]
+}
+
+resource "google_folder" "folder" {
+  display_name = "E2E Tests ${var.timestamp}-${var.suffix}"
+  parent       = var.parent
+}
+
+resource "google_project" "project" {
+  name            = "${local.prefix}-prj"
+  billing_account = var.billing_account
+  folder_id       = google_folder.folder.id
+  project_id      = "${local.prefix}-prj"
+}
+
+resource "google_project_service" "project_service" {
+  for_each                   = toset(local.services)
+  service                    = each.value
+  project                    = google_project.project.project_id
+  disable_dependent_services = true
+}
+
+resource "google_storage_bucket" "bucket" {
+  location      = var.region
+  name          = "${local.prefix}-bucket"
+  project       = google_project.project.project_id
+  force_destroy = true
+  depends_on    = [google_project_service.project_service]
+}
+
+resource "google_compute_network" "network" {
+  name                    = "e2e-test"
+  project                 = google_project.project.project_id
+  auto_create_subnetworks = false
+  depends_on              = [google_project_service.project_service]
+}
+
+resource "google_compute_subnetwork" "subnetwork" {
+  ip_cidr_range = "10.0.16.0/24"
+  name          = "e2e-test-1"
+  network       = google_compute_network.network.name
+  project       = google_project.project.project_id
+  region        = var.region
+}
+
+resource "google_service_account" "service_account" {
+  account_id = "e2e-service-account"
+  project    = google_project.project.project_id
+  depends_on = [google_project_service.project_service]
+}
+
+resource "local_file" "terraform_tfvars" {
+  filename = "e2e_tests.tfvars"
+  content = templatefile("e2e_tests.tfvars.tftpl", {
+    bucket             = google_storage_bucket.bucket.name
+    billing_account_id = var.billing_account
+    organization_id    = var.organization_id
+    folder_id          = google_folder.folder.folder_id
+    prefix             = local.prefix
+    project_id         = google_project.project.project_id
+    region             = var.region
+    service_account = {
+      id        = google_service_account.service_account.id
+      email     = google_service_account.service_account.email
+      iam_email = "serviceAccount:${google_service_account.service_account.email}"
+    }
+    subnet = {
+      name          = google_compute_subnetwork.subnetwork.name
+      region        = google_compute_subnetwork.subnetwork.region
+      ip_cidr_range = google_compute_subnetwork.subnetwork.ip_cidr_range
+      self_link     = google_compute_subnetwork.subnetwork.self_link
+    }
+    vpc = {
+      name      = google_compute_network.network.name
+      self_link = google_compute_network.network.self_link
+      id        = google_compute_network.network.id
+    }
+  })
+}

tests/examples_e2e/setup_module/variables.tf

@@ -0,0 +1,35 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+variable "billing_account" {
+  type = string
+}
+variable "organization_id" {
+  type = string
+}
+variable "parent" {
+  type = string
+}
+variable "prefix" {
+  type = string
+}
+variable "region" {
+  type = string
+}
+variable "suffix" {
+  type = string
+}
+variable "timestamp" {
+  type = string
+}

tests/examples_e2e/test_plan.py

@@ -0,0 +1,35 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import re
+
+from pathlib import Path
+
+BASE_PATH = Path(__file__).parent
+COUNT_TEST_RE = re.compile(r'# tftest +modules=(\d+) +resources=(\d+)' +
+                           r'(?: +files=([\w@,_-]+))?' +
+                           r'(?: +inventory=([\w\-.]+))?')
+
+
+def test_example(e2e_validator, tmp_path, examples_e2e, e2e_tfvars_path):
+  (tmp_path / 'fabric').symlink_to(BASE_PATH.parents[1])
+  (tmp_path / 'variables.tf').symlink_to(BASE_PATH / 'variables.tf')
+  (tmp_path / 'main.tf').write_text(examples_e2e.code)
+  assets_path = BASE_PATH.parent / str(examples_e2e.module).replace(
+      '-', '_') / 'assets'
+  if assets_path.exists():
+    (tmp_path / 'assets').symlink_to(assets_path)
+  (tmp_path / 'terraform.tfvars').symlink_to(e2e_tfvars_path)
+
+  e2e_validator(module_path=tmp_path, extra_files=[],
+                tf_var_files=[(tmp_path / 'terraform.tfvars')])

tests/examples_e2e/variables.tf

@@ -0,0 +1,92 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# common variables used for examples
+
+variable "bucket" {
+  default = "bucket"
+}
+
+variable "billing_account_id" {
+  default = "123456-123456-123456"
+}
+
+variable "kms_key" {
+  default = {
+    self_link = "kms_key_self_link"
+  }
+}
+
+variable "organization_id" {
+  default = "organizations/1122334455"
+}
+
+variable "folder_id" {
+  default = "folders/1122334455"
+}
+
+variable "prefix" {
+  default = "test"
+}
+
+variable "project_id" {
+  default = "project-id"
+}
+
+variable "region" {
+  default = "region"
+}
+
+variable "service_account" {
+  default = {
+    id        = "service_account_id"
+    email     = "service_account_email"
+    iam_email = "service_account_iam_email"
+  }
+}
+
+variable "subnet" {
+  default = {
+    name      = "subnet_name"
+    region    = "subnet_region"
+    cidr      = "subnet_cidr"
+    self_link = "subnet_self_link"
+  }
+}
+
+variable "vpc" {
+  default = {
+    name      = "vpc_name"
+    self_link = "projects/xxx/global/networks/aaa"
+    id        = "projects/xxx/global/networks/aaa"
+  }
+}
+
+variable "vpc1" {
+  default = {
+    name      = "vpc_name"
+    self_link = "projects/xxx/global/networks/bbb"
+  }
+}
+
+variable "vpc2" {
+  default = {
+    name      = "vpc2_name"
+    self_link = "projects/xxx/global/networks/ccc"
+  }
+}
+
+variable "zone" {
+  default = "zone"
+}

tests/fixtures.py

@@ -19,6 +19,7 @@ import glob
 import os
 import shutil
 import tempfile
+import time
 from pathlib import Path
 
 import pytest
@@ -38,23 +39,23 @@ def _prepare_root_module(path):
   terraform.tfvars) are deleted to ensure a clean test environment.
   Otherwise, `path` is simply returned untouched.
   """
+  # if we're copying the module, we might as well ignore files and
+  # directories that are automatically read by terraform. Useful
+  # to avoid surprises if, for example, you have an active fast
+  # deployment with links to configs)
+  ignore_patterns = shutil.ignore_patterns('*.auto.tfvars',
+                                           '*.auto.tfvars.json',
+                                           '[0-9]-*-providers.tf',
+                                           'terraform.tfstate*',
+                                           '.terraform.lock.hcl',
+                                           'terraform.tfvars', '.terraform')
+
   if os.environ.get('TFTEST_COPY'):
     # if the TFTEST_COPY is set, create temp dir and copy the root
     # module there
     with tempfile.TemporaryDirectory(dir=path.parent) as tmp_path:
       tmp_path = Path(tmp_path)
 
-      # if we're copying the module, we might as well ignore files and
-      # directories that are automatically read by terraform. Useful
-      # to avoid surprises if, for example, you have an active fast
-      # deployment with links to configs)
-      ignore_patterns = shutil.ignore_patterns('*.auto.tfvars',
-                                               '*.auto.tfvars.json',
-                                               '[0-9]-*-providers.tf',
-                                               'terraform.tfstate*',
-                                               '.terraform.lock.hcl',
-                                               'terraform.tfvars', '.terraform')
-
       shutil.copytree(path, tmp_path, dirs_exist_ok=True,
                       ignore=ignore_patterns)
       lockfile = _REPO_ROOT / 'tools' / 'lockfile' / '.terraform.lock.hcl'
@@ -63,6 +64,13 @@ def _prepare_root_module(path):
 
       yield tmp_path
   else:
+    # check if any ignore_patterns files are present in path
+    if unwanted_files := ignore_patterns(path, os.listdir(path=path)):
+      # prevent shooting yourself in the foot (unexpected test results) when ignored files are present
+      raise RuntimeError(
+          f'Test in path {path} contains {", ".join(unwanted_files)} which may affect '
+          f'test results. Please run tests with TFTEST_COPY=1 environment variable'
+      )
     # if TFTEST_COPY is not set, just return the same path
     yield path
 
@@ -190,7 +198,8 @@ def plan_validator(module_path, inventory_paths, basedir, tf_var_files=None,
     # - include a descriptive error message to the assert
 
     if 'values' in inventory:
-      validate_plan_object(inventory['values'], summary.values, relative_path, "")
+      validate_plan_object(inventory['values'], summary.values, relative_path,
+                           "")
 
     if 'counts' in inventory:
       expected_counts = inventory['counts']
@@ -216,7 +225,8 @@ def plan_validator(module_path, inventory_paths, basedir, tf_var_files=None,
   return summary
 
 
-def validate_plan_object(expected_value, plan_value, relative_path, relative_address):
+def validate_plan_object(expected_value, plan_value, relative_path,
+                         relative_address):
   """
   Validate that plan object matches inventory
 
@@ -233,7 +243,8 @@ def validate_plan_object(expected_value, plan_value, relative_path, relative_add
     for k, v in expected_value.items():
       assert k in plan_value, \
         f'{relative_path}: {k} is not a valid address in the plan'
-      validate_plan_object(v, plan_value[k], relative_path, f'{relative_address}.{k}')
+      validate_plan_object(v, plan_value[k], relative_path,
+                           f'{relative_address}.{k}')
 
   # lists
   elif isinstance(expected_value, list) and isinstance(plan_value, list):
@@ -241,7 +252,8 @@ def validate_plan_object(expected_value, plan_value, relative_path, relative_add
       f'{relative_path}: {relative_address} has different length. Got {plan_value}, expected {expected_value}'
 
     for i, (exp, actual) in enumerate(zip(expected_value, plan_value)):
-      validate_plan_object(exp, actual, relative_path, f'{relative_address}[{i}]')
+      validate_plan_object(exp, actual, relative_path,
+                           f'{relative_address}[{i}]')
 
   # all other objects
   else:
@@ -269,6 +281,107 @@ def plan_validator_fixture(request):
   return inner
 
 
+def e2e_validator(module_path, extra_files, tf_var_files, basedir=None):
+  """Function running apply, plan and destroy to verify the case end to end
+
+  1. Tests whether apply does not return errors
+  2. Tests whether plan after apply is empty
+  3. Tests whether destroy does not return errors
+  """
+  module_path = _REPO_ROOT / module_path
+  with _prepare_root_module(module_path) as test_path:
+    binary = os.environ.get('TERRAFORM', 'terraform')
+    tf = tftest.TerraformTest(test_path, binary=binary)
+    extra_files = [(module_path / filename).resolve()
+                   for x in extra_files or []
+                   for filename in glob.glob(x, root_dir=module_path)]
+    tf.setup(extra_files=extra_files, upgrade=True)
+    tf_var_files = [(basedir / x).resolve() for x in tf_var_files or []]
+
+    try:
+      apply = tf.apply(tf_var_file=tf_var_files)
+      plan = tf.plan(output=True, tf_var_file=tf_var_files)
+      changes = {}
+      for resource_name, value in plan.resource_changes.items():
+        if value.get('change', {}).get('actions') != ['no-op']:
+          changes[resource_name] = value
+
+      # compare before with after to raise more meaningful failure to the user, i.e one
+      # that shows how resource will change
+      plan_before_state = {k: v['before'] for k, v in changes.items()}
+      plan_after_state = {k: v['after'] for k, v in changes.items()}
+
+      assert plan_before_state == plan_after_state, f'Plan not empty after apply for values'
+
+      plan_before_sensitive_state = {
+          k: v['before_sensitive'] for k, v in changes.items()
+      }
+      plan_after_sensitive_state = {
+          k: v['after_sensitive'] for k, v in changes.items()
+      }
+      assert plan_before_sensitive_state == plan_after_sensitive_state, f'Plan not empty after apply for sensitive values'
+
+      # If above did not fail, this should not either, but left as a safety check
+      assert changes == {}, f'Plan not empty for following resources: {", ".join(changes.keys())}'
+    finally:
+      destroy = tf.destroy(tf_var_file=tf_var_files)
+
+
+@pytest.fixture(name='e2e_validator')
+def e2e_validator_fixture(request):
+  """Return a function to run end-to-end test
+
+  In the returned function `basedir` becomes optional and it defaults
+  to the directory of the calling test
+
+  """
+
+  def inner(module_path: str, extra_files: list, tf_var_files: list,
+            basedir: os.PathLike = None):
+    if basedir is None:
+      basedir = Path(request.fspath).parent
+    return e2e_validator(module_path, extra_files, tf_var_files, basedir)
+
+  return inner
+
+
+@pytest.fixture(scope='session', name='e2e_tfvars_path')
+def e2e_tfvars_path():
+  """Fixture preparing end-to-end test environment
+
+  If TFTEST_E2E_TFVARS_PATH is set in the environment, then assume the environment is already provisioned
+  and necessary variables are set in the file to which variable is pointing to.
+
+  Otherwise, create a unique test environment (in case of multiple workers - as many environments as
+  there are workers), that will be injected into each example test instead of `tests/examples/variables.tf`.
+
+  Returns path to tfvars file that contains information about envrionment to use for the tests.
+  """
+  if tfvars_path := os.environ.get('TFTEST_E2E_TFVARS_PATH'):
+    # no need to set up the project
+    if int(os.environ.get('PYTEST_XDIST_WORKER_COUNT', '0')) > 1:
+      raise RuntimeError(
+          'Setting TFTEST_E2E_TFVARS_PATH is not compatible with running tests in parallel'
+      )
+    yield tfvars_path
+  else:
+    with _prepare_root_module(_REPO_ROOT / 'tests' / 'examples_e2e' /
+                              'setup_module') as test_path:
+      binary = os.environ.get('TERRAFORM', 'terraform')
+      tf = tftest.TerraformTest(test_path, binary=binary)
+      tf_vars_file = None
+      tf_vars = {
+          'suffix': os.environ.get("PYTEST_XDIST_WORKER", "0"),
+          'timestamp': str(int(time.time()))
+      }
+      if 'TFTEST_E2E_SETUP_TFVARS_PATH' in os.environ:
+        tf_vars_file = os.environ["TFTEST_E2E_SETUP_TFVARS_PATH"]
+      tf.setup(upgrade=True)
+      tf.apply(tf_vars=tf_vars, tf_var_file=tf_vars_file)
+      yield test_path / "e2e_tests.tfvars"
+      tf.destroy(tf_vars=tf_vars, tf_var_file=tf_vars_file)
+
+
 # @pytest.fixture
 # def repo_root():
 #   'Return a pathlib.Path to the root of the repository'

tests/modules/cloud_function_v1/assets/sample-function/main.py

@@ -0,0 +1,20 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functions_framework
+
+
+@functions_framework.http
+def main(request):
+  return "Hello World!"

tests/modules/cloud_run/examples/simple.yaml

@@ -19,7 +19,7 @@ values:
     metadata:
     - {}
     name: hello
-    project: my-project
+    project: project-id
     template:
     - metadata:
       - {}
@@ -45,7 +45,7 @@ values:
     location: europe-west1
     members:
     - allUsers
-    project: my-project
+    project: project-id
     role: roles/run.invoker
     service: hello