Secret manager e2etests (#2280)

secret-manager e2e tests

CONTRIBUTING.md

@@ -1107,6 +1107,7 @@ export TFTEST_E2E_organization_id="1234567890" # your organization id
 export TFTEST_E2E_parent="folders/1234567890"  # folder under which test resources will be created
 export TFTEST_E2E_prefix="your-unique-prefix"  # unique prefix for projects, no longer than 7 characters
 export TFTEST_E2E_region="europe-west4"  # region to use
+export TFTEST_E2E_region_secondary="europe-west5" # secondary region to use
 ```
 
 To use Service Account Impersonation, use provider environment variable
@@ -1136,6 +1137,7 @@ organization_id = "1234567890"  # your organization id
 parent          = "folders/1234567890"  # folder under which test resources will be created
 prefix          = "your-unique-prefix"  # unique prefix for projects
 region          = "europe-west4"  # region to use
+region_secondary = "europe-west5" # secondary region to use
 timestamp       = "1696444185" # generate your own timestamp - will be used as a part of prefix for globally unique resources
 ```
 

modules/secret-manager/README.md

@@ -15,15 +15,15 @@ The secret replication policy is automatically managed if no location is set, or
 ```hcl
 module "secret-manager" {
   source     = "./fabric/modules/secret-manager"
-  project_id = "my-project"
+  project_id = var.project_id
   secrets = {
     test-auto = {}
     test-manual = {
-      locations = ["europe-west1", "europe-west4"]
+      locations = [var.regions.primary, var.regions.secondary]
     }
   }
 }
-# tftest modules=1 resources=2
+# tftest modules=1 resources=2 inventory=secret.yaml e2e
 ```
 
 ### Secret IAM bindings
@@ -33,23 +33,23 @@ IAM bindings can be set per secret in the same way as for most other modules sup
 ```hcl
 module "secret-manager" {
   source     = "./fabric/modules/secret-manager"
-  project_id = "my-project"
+  project_id = var.project_id
   secrets = {
     test-auto = {}
     test-manual = {
-      locations = ["europe-west1", "europe-west4"]
+      locations = [var.regions.primary, var.regions.secondary]
     }
   }
   iam = {
     test-auto = {
-      "roles/secretmanager.secretAccessor" = ["group:auto-readers@example.com"]
+      "roles/secretmanager.secretAccessor" = ["group:${var.group_email}"]
     }
     test-manual = {
-      "roles/secretmanager.secretAccessor" = ["group:manual-readers@example.com"]
+      "roles/secretmanager.secretAccessor" = ["group:${var.group_email}"]
     }
   }
 }
-# tftest modules=1 resources=4 inventory=iam.yaml
+# tftest modules=1 resources=4 inventory=iam.yaml e2e
 ```
 
 ### Secret versions
@@ -59,11 +59,11 @@ As mentioned above, please be aware that **version data will be stored in state
 ```hcl
 module "secret-manager" {
   source     = "./fabric/modules/secret-manager"
-  project_id = "my-project"
+  project_id = var.project_id
   secrets = {
     test-auto = {}
     test-manual = {
-      locations = ["europe-west1", "europe-west4"]
+      locations = [var.regions.primary, var.regions.secondary]
     }
   }
   versions = {
@@ -76,7 +76,7 @@ module "secret-manager" {
     }
   }
 }
-# tftest modules=1 resources=5 inventory=versions.yaml
+# tftest modules=1 resources=5 inventory=versions.yaml e2e
 ```
 
 ### Secret with customer managed encryption key
@@ -86,24 +86,24 @@ CMEK will be used if an encryption key is set in the `keys` field of `secrets` o
 ```hcl
 module "secret-manager" {
   source     = "./fabric/modules/secret-manager"
-  project_id = "my-project"
+  project_id = var.project_id
   secrets = {
     test-auto = {
       keys = {
-        global = "projects/PROJECT_ID/locations/global/keyRings/KEYRING/cryptoKeys/KEY"
+        global = module.kms_global.keys.key-gl.id
       }
     }
     test-auto-nokeys = {}
     test-manual = {
-      locations = ["europe-west1", "europe-west4"]
+      locations = [var.regions.primary, var.regions.secondary]
       keys = {
-        europe-west1 = "projects/PROJECT_ID/locations/europe-west1/keyRings/KEYRING/cryptoKeys/KEY"
+        "${var.regions.primary}"   = module.kms_regional_primary.keys.key-a.id
-        europe-west4 = "projects/PROJECT_ID/locations/europe-west4/keyRings/KEYRING/cryptoKeys/KEY"
+        "${var.regions.secondary}" = module.kms_regional_secondary.keys.key-b.id
       }
     }
   }
 }
-# tftest modules=1 resources=3
+# tftest modules=4 resources=11 fixtures=fixtures/kms-global-regional-keys.tf inventory=secret-cmek.yaml e2e
 ```
 <!-- BEGIN TFDOC -->
 ## Variables
@@ -125,6 +125,10 @@ module "secret-manager" {
 | [version_ids](outputs.tf#L29) | Version ids keyed by secret name : version name. |  |
 | [version_versions](outputs.tf#L36) | Version versions keyed by secret name : version name. |  |
 | [versions](outputs.tf#L43) | Secret versions. | ✓ |
+
+## Fixtures
+
+- [kms-global-regional-keys.tf](../../tests/fixtures/kms-global-regional-keys.tf)
 <!-- END TFDOC -->
 ## Requirements
 

tests/examples/variables.tf

@@ -62,6 +62,13 @@ variable "region" {
   default = "europe-west8"
 }
 
+variable "regions" {
+  default = {
+    primary   = "europe-west8"
+    secondary = "europe-west9"
+  }
+}
+
 variable "service_account" {
   default = {
     id        = "service_account_id"

tests/examples_e2e/setup_module/e2e_tests.tfvars.tftpl

@@ -26,11 +26,16 @@ folder_id       = "folders/${folder_id}"
 project_id      = "${project_id}"
 project_number  = "${project_number}"
 region          = "${region}"
+regions         = {
+  primary       = "${regions.primary}"
+  secondary     = "${regions.secondary}"
+}
 service_account = {
     id        = "${service_account.id}"
     email     = "${service_account.email}"
     iam_email = "${service_account.iam_email}"
   }
+
 subnet = {
     name      = "${subnet.name}"
     region    = "${subnet.region}"

tests/examples_e2e/setup_module/main.tf

@@ -188,14 +188,18 @@ resource "local_file" "terraform_tfvars" {
     billing_account_id = var.billing_account
     folder_id          = google_folder.folder.folder_id
     group_email        = var.group_email
-    kms_key_id         = google_kms_crypto_key.key.id
     keyring = {
       name = google_kms_key_ring.keyring.name
     }
+    kms_key_id      = google_kms_crypto_key.key.id
     organization_id = var.organization_id
     project_id      = google_project.project.project_id
     project_number  = google_project.project.number
     region          = var.region
+    regions = {
+      primary   = var.region
+      secondary = var.region_secondary
+    }
     service_account = {
       id        = google_service_account.service_account.id
       email     = google_service_account.service_account.email

tests/examples_e2e/setup_module/variables.tf

@@ -30,6 +30,9 @@ variable "prefix" {
 variable "region" {
   type = string
 }
+variable "region_secondary" {
+  type = string
+}
 variable "suffix" {
   type    = string
   default = "0"

tests/fixtures.py

@@ -289,7 +289,7 @@ def plan_validator_fixture(request):
 def get_tfvars_for_e2e():
   _variables = [
       'billing_account', 'group_email', 'organization_id', 'parent', 'prefix',
-      'region'
+      'region', 'region_secondary'
   ]
   missing_vars = set([f'TFTEST_E2E_{k}' for k in _variables]) - set(
       os.environ.keys())
@@ -300,6 +300,8 @@ def get_tfvars_for_e2e():
         f'If you want to skip E2E tests add -k "not examples_e2e" to your pytest call'
     )
   tf_vars = {k: os.environ.get(f'TFTEST_E2E_{k}') for k in _variables}
+  if tf_vars['region'] == tf_vars['region_secondary']:
+        raise ValueError("E2E tests require distinct primary and secondary regions.")
   return tf_vars
 
 

tests/fixtures/kms-global-regional-keys.tf

@@ -0,0 +1,67 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+resource "google_project_service_identity" "secretmanager" {
+  provider = google-beta
+  project  = var.project_id
+  service  = "secretmanager.googleapis.com"
+}
+
+resource "google_project_iam_binding" "bindings" {
+  project = var.project_id
+  role    = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
+  members = ["serviceAccount:${resource.google_project_service_identity.secretmanager.email}"]
+}
+
+module "kms_regional_primary" {
+  source     = "./fabric/modules/kms"
+  project_id = var.project_id
+  keyring = {
+    location = var.regions.primary
+    name     = "keyring-primary"
+  }
+  keys = {
+    "key-a" = {
+    }
+  }
+  depends_on = [google_project_iam_binding.bindings]
+}
+
+module "kms_regional_secondary" {
+  source     = "./fabric/modules/kms"
+  project_id = var.project_id
+  keyring = {
+    location = var.regions.secondary
+    name     = "keyring-secondary"
+  }
+  keys = {
+    "key-b" = {
+    }
+  }
+  depends_on = [google_project_iam_binding.bindings]
+}
+
+module "kms_global" {
+  source     = "./fabric/modules/kms"
+  project_id = var.project_id
+  keyring = {
+    location = "global"
+    name     = "keyring-gl"
+  }
+  keys = {
+    "key-gl" = {
+    }
+  }
+  depends_on = [google_project_iam_binding.bindings]
+}

tests/modules/secret_manager/examples/iam.yaml

@@ -14,34 +14,36 @@
 
 values:
   module.secret-manager.google_secret_manager_secret.default["test-auto"]:
-    project: my-project
+    project: project-id
     replication:
     - auto:
       - customer_managed_encryption: []
       user_managed: []
     secret_id: test-auto
   module.secret-manager.google_secret_manager_secret.default["test-manual"]:
-    project: my-project
+    project: project-id
     replication:
     - auto: []
       user_managed:
       - replicas:
         - customer_managed_encryption: []
-          location: europe-west1
+          location: europe-west8
         - customer_managed_encryption: []
-          location: europe-west4
+          location: europe-west9
     secret_id: test-manual
   module.secret-manager.google_secret_manager_secret_iam_binding.default["test-auto.roles/secretmanager.secretAccessor"]:
     condition: []
     members:
-    - group:auto-readers@example.com
+    - group:organization-admins@example.org
     role: roles/secretmanager.secretAccessor
   module.secret-manager.google_secret_manager_secret_iam_binding.default["test-manual.roles/secretmanager.secretAccessor"]:
     condition: []
     members:
-    - group:manual-readers@example.com
+    - group:organization-admins@example.org
     role: roles/secretmanager.secretAccessor
 
 counts:
   google_secret_manager_secret: 2
   google_secret_manager_secret_iam_binding: 2
+
+outputs: {}

tests/modules/secret_manager/examples/secret-cmek.yaml

@@ -0,0 +1,75 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+  module.secret-manager.google_secret_manager_secret.default["test-auto"]:
+    annotations: null
+    labels: null
+    project: project-id
+    replication:
+    - auto:
+      - {}
+      user_managed: []
+    rotation: []
+    secret_id: test-auto
+    timeouts: null
+    topics: []
+    ttl: null
+    version_aliases: null
+    version_destroy_ttl: null
+  module.secret-manager.google_secret_manager_secret.default["test-auto-nokeys"]:
+    annotations: null
+    labels: null
+    project: project-id
+    replication:
+    - auto:
+      - customer_managed_encryption: []
+      user_managed: []
+    rotation: []
+    secret_id: test-auto-nokeys
+    timeouts: null
+    topics: []
+    ttl: null
+    version_aliases: null
+    version_destroy_ttl: null
+  module.secret-manager.google_secret_manager_secret.default["test-manual"]:
+    annotations: null
+    labels: null
+    project: project-id
+    replication:
+    - auto: []
+      user_managed:
+      - replicas:
+        - location: europe-west8
+        - location: europe-west9
+    rotation: []
+    secret_id: test-manual
+    timeouts: null
+    topics: []
+    ttl: null
+    version_aliases: null
+    version_destroy_ttl: null
+
+counts:
+  google_kms_crypto_key: 3
+  google_kms_key_ring: 3
+  google_project_iam_binding: 1
+  google_project_service_identity: 1
+  google_secret_manager_secret: 3
+  modules: 4
+  resources: 11
+
+outputs: {}
+
+outputs: {}

tests/modules/secret_manager/examples/secret.yaml

@@ -0,0 +1,54 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+  module.secret-manager.google_secret_manager_secret.default["test-auto"]:
+    annotations: null
+    labels: null
+    project: project-id
+    replication:
+    - auto:
+      - customer_managed_encryption: []
+      user_managed: []
+    rotation: []
+    secret_id: test-auto
+    timeouts: null
+    topics: []
+    ttl: null
+    version_aliases: null
+  module.secret-manager.google_secret_manager_secret.default["test-manual"]:
+    annotations: null
+    labels: null
+    project: project-id
+    replication:
+    - auto: []
+      user_managed:
+      - replicas:
+        - customer_managed_encryption: []
+          location: europe-west8
+        - customer_managed_encryption: []
+          location: europe-west9
+    rotation: []
+    secret_id: test-manual
+    timeouts: null
+    topics: []
+    ttl: null
+    version_aliases: null
+
+counts:
+  google_secret_manager_secret: 2
+  modules: 1
+  resources: 2
+
+outputs: {}

tests/modules/secret_manager/examples/versions.yaml

@@ -26,3 +26,5 @@ values:
 counts:
   google_secret_manager_secret: 2
   google_secret_manager_secret_version: 3
+
+outputs: {}