feat: add e2e test for pubsub module (#2163)

* feat: add e2e test for pubsub module

modules/pubsub/README.md

@@ -9,14 +9,14 @@ This module allows managing a single Pub/Sub topic, including multiple subscript
 ```hcl
 module "pubsub" {
   source     = "./fabric/modules/pubsub"
-  project_id = "my-project"
+  project_id = var.project_id
   name       = "my-topic"
   iam = {
-    "roles/pubsub.viewer"     = ["group:foo@example.com"]
+    "roles/pubsub.viewer"     = ["group:${var.group_email}"]
-    "roles/pubsub.subscriber" = ["user:user1@example.com"]
+    "roles/pubsub.subscriber" = ["serviceAccount:${var.service_account.email}"]
   }
 }
-# tftest modules=1 resources=3 inventory=simple.yaml
+# tftest modules=1 resources=3  inventory=simple.yaml e2e
 ```
 
 ### Topic with schema
@@ -24,7 +24,7 @@ module "pubsub" {
 ```hcl
 module "topic_with_schema" {
   source     = "./fabric/modules/pubsub"
-  project_id = "my-project"
+  project_id = var.project_id
   name       = "my-topic"
   schema = {
     msg_encoding = "JSON"
@@ -48,7 +48,7 @@ module "topic_with_schema" {
     })
   }
 }
-# tftest modules=1 resources=2 inventory=schema.yaml
+# tftest modules=1 resources=2 inventory=schema.yaml e2e
 ```
 
 ### Subscriptions
@@ -58,7 +58,7 @@ Subscriptions are defined with the `subscriptions` variable, allowing optional c
 ```hcl
 module "pubsub" {
   source     = "./fabric/modules/pubsub"
-  project_id = "my-project"
+  project_id = var.project_id
   name       = "my-topic"
   subscriptions = {
     test-pull = {}
@@ -68,7 +68,7 @@ module "pubsub" {
     }
   }
 }
-# tftest modules=1 resources=3 inventory=subscriptions.yaml
+# tftest modules=1 resources=3 inventory=subscriptions.yaml e2e
 ```
 
 ### Push subscriptions
@@ -78,7 +78,7 @@ Push subscriptions need extra configuration in the `push_configs` variable.
 ```hcl
 module "pubsub" {
   source     = "./fabric/modules/pubsub"
-  project_id = "my-project"
+  project_id = var.project_id
   name       = "my-topic"
   subscriptions = {
     test-push = {
@@ -88,7 +88,7 @@ module "pubsub" {
     }
   }
 }
-# tftest modules=1 resources=2
+# tftest modules=1 resources=2 inventory=push-subscription.yaml e2e
 ```
 
 ### BigQuery subscriptions
@@ -98,12 +98,12 @@ BigQuery subscriptions need extra configuration in the `bigquery_subscription_co
 ```hcl
 module "pubsub" {
   source     = "./fabric/modules/pubsub"
-  project_id = "my-project"
+  project_id = var.project_id
   name       = "my-topic"
   subscriptions = {
     test-bigquery = {
       bigquery = {
-        table               = "my_project_id:my_dataset.my_table"
+        table               = "${module.bigquery-dataset.tables["my_table"].project}:${module.bigquery-dataset.tables["my_table"].dataset_id}.${module.bigquery-dataset.tables["my_table"].table_id}"
         use_topic_schema    = true
         write_metadata      = false
         drop_unknown_fields = true
@@ -111,7 +111,7 @@ module "pubsub" {
     }
   }
 }
-# tftest modules=1 resources=2
+# tftest modules=2 resources=5 fixtures=fixtures/bigquery-dataset.tf inventory=bigquery-subscription.yaml e2e
 ```
 
 ### Cloud Storage subscriptions
@@ -121,13 +121,13 @@ Cloud Storage subscriptions need extra configuration in the `cloud_storage_subsc
 ```hcl
 module "pubsub" {
   source     = "./fabric/modules/pubsub"
-  project_id = "my-project"
+  project_id = var.project_id
   name       = "my-topic"
   subscriptions = {
     test-cloudstorage = {
       cloud_storage = {
-        bucket          = "my-bucket"
+        bucket          = module.gcs.name
-        filename_prefix = "test_prefix"
+        filename_prefix = var.prefix
         filename_suffix = "test_suffix"
         max_duration    = "100s"
         max_bytes       = 1000
@@ -138,24 +138,24 @@ module "pubsub" {
     }
   }
 }
-# tftest modules=1 resources=2
+# tftest modules=2 resources=4 fixtures=fixtures/gcs.tf inventory=cloud-storage-subscription.yaml e2e
 ```
 ### Subscriptions with IAM
 
 ```hcl
 module "pubsub" {
   source     = "./fabric/modules/pubsub"
-  project_id = "my-project"
+  project_id = var.project_id
   name       = "my-topic"
   subscriptions = {
     test-1 = {
       iam = {
-        "roles/pubsub.subscriber" = ["user:user1@example.com"]
+        "roles/pubsub.subscriber" = ["serviceAccount:${var.service_account.email}"]
       }
     }
   }
 }
-# tftest modules=1 resources=3
+# tftest modules=1 resources=3 inventory=subscription-iam.yaml e2e
 ```
 <!-- BEGIN TFDOC -->
 ## Variables
@@ -184,4 +184,9 @@ module "pubsub" {
 | [subscription_id](outputs.tf#L37) | Subscription ids. |  |
 | [subscriptions](outputs.tf#L48) | Subscription resources. |  |
 | [topic](outputs.tf#L57) | Topic resource. |  |
+
+## Fixtures
+
+- [bigquery-dataset.tf](../../tests/fixtures/bigquery-dataset.tf)
+- [gcs.tf](../../tests/fixtures/gcs.tf)
 <!-- END TFDOC -->

tests/fixtures/bigquery-dataset.tf

@@ -0,0 +1,38 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+locals {
+  schema = jsonencode([
+    { name = "data", type = "STRING" },
+    { name = "freq", type = "INT64" }
+  ])
+}
+
+module "bigquery-dataset" {
+  source     = "./fabric/modules/bigquery-dataset"
+  project_id = var.project_id
+  id         = "my_dataset"
+  tables = {
+    my_table = {
+      deletion_protection = false
+      schema              = local.schema
+      partitioning = {
+        time = { type = "DAY", expiration_ms = null }
+      }
+    }
+  }
+  iam = {
+    "roles/bigquery.dataEditor" = ["serviceAccount:service-${var.project_number}@gcp-sa-pubsub.iam.gserviceaccount.com"]
+  }
+}

tests/fixtures/gcs.tf

@@ -0,0 +1,23 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module "gcs" {
+  source     = "./fabric/modules/gcs"
+  project_id = var.project_id
+  prefix     = var.prefix
+  name       = "my-bucket"
+  iam = {
+    "roles/storage.admin" = ["serviceAccount:service-${var.project_number}@gcp-sa-pubsub.iam.gserviceaccount.com"]
+  }
+}

tests/modules/pubsub/examples/bigquery-subscription.yaml

@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,16 +16,21 @@ values:
   module.pubsub.google_pubsub_subscription.default["test-bigquery"]:
     bigquery_config:
     - drop_unknown_fields: true
-      table: my_project_id:my_dataset.my_table
+      table: project-id:my_dataset.my_table
       use_topic_schema: true
       write_metadata: false
     name: test-bigquery
-    project: my-project
+    project: project-id
     topic: my-topic
   module.pubsub.google_pubsub_topic.default:
     name: my-topic
-    project: my-project
+    project: project-id
 
 counts:
+  google_bigquery_dataset: 1
+  google_bigquery_dataset_iam_binding: 1
+  google_bigquery_table: 1
   google_pubsub_subscription: 1
   google_pubsub_topic: 1
+  modules: 2
+  resources: 5

tests/modules/pubsub/examples/cloud-storage-subscription.yaml

@@ -0,0 +1,53 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+  module.pubsub.google_pubsub_subscription.default["test-cloudstorage"]:
+    bigquery_config: []
+    cloud_storage_config:
+    - avro_config:
+      - write_metadata: true
+      bucket: test-my-bucket
+      filename_prefix: test
+      filename_suffix: test_suffix
+      max_bytes: 1000
+      max_duration: 100s
+    dead_letter_policy: []
+    enable_exactly_once_delivery: false
+    enable_message_ordering: false
+    filter: null
+    labels: null
+    message_retention_duration: 604800s
+    name: test-cloudstorage
+    project: project-id
+    push_config: []
+    retain_acked_messages: false
+    retry_policy: []
+    timeouts: null
+    topic: my-topic
+  module.pubsub.google_pubsub_topic.default:
+    kms_key_name: null
+    labels: null
+    message_retention_duration: null
+    name: my-topic
+    project: project-id
+    timeouts: null
+
+counts:
+  google_pubsub_subscription: 1
+  google_pubsub_topic: 1
+  google_storage_bucket: 1
+  google_storage_bucket_iam_binding: 1
+  modules: 2
+  resources: 4

tests/modules/pubsub/examples/push-subscription.yaml

@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,19 +12,38 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 values:
   module.pubsub.google_pubsub_subscription.default["test-push"]:
+    bigquery_config: []
+    cloud_storage_config: []
+    dead_letter_policy: []
+    enable_exactly_once_delivery: false
+    enable_message_ordering: false
+    filter: null
+    labels: null
+    message_retention_duration: 604800s
     name: test-push
-    project: my-project
+    project: project-id
     push_config:
     - attributes: null
+      no_wrapper: []
       oidc_token: []
       push_endpoint: https://example.com/foo
+    retain_acked_messages: false
+    retry_policy: []
+    timeouts: null
     topic: my-topic
   module.pubsub.google_pubsub_topic.default:
+    kms_key_name: null
+    labels: null
+    message_retention_duration: null
     name: my-topic
-    project: my-project
+    project: project-id
+    timeouts: null
 
 counts:
   google_pubsub_subscription: 1
   google_pubsub_topic: 1
+  modules: 1
+  resources: 2

tests/modules/pubsub/examples/schema.yaml

@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,14 +16,23 @@ values:
   module.topic_with_schema.google_pubsub_schema.default[0]:
     definition: '{"fields":[{"name":"StringField","type":"string"},{"name":"FloatField","type":"float"},{"name":"BooleanField","type":"boolean"}],"name":"Avro","type":"record"}'
     name: my-topic-schema
-    project: my-project
+    project: project-id
+    timeouts: null
     type: AVRO
   module.topic_with_schema.google_pubsub_topic.default:
+    kms_key_name: null
+    labels: null
+    message_retention_duration: null
     name: my-topic
-    project: my-project
+    project: project-id
     schema_settings:
     - encoding: JSON
+    timeouts: null
 
 counts:
   google_pubsub_schema: 1
   google_pubsub_topic: 1
+  modules: 1
+  resources: 2
+
+outputs: {}

tests/modules/pubsub/examples/simple.yaml

@@ -14,23 +14,29 @@
 
 values:
   module.pubsub.google_pubsub_topic.default:
+    kms_key_name: null
+    labels: null
+    message_retention_duration: null
     name: my-topic
-    project: my-project
+    project: project-id
+    timeouts: null
   module.pubsub.google_pubsub_topic_iam_binding.authoritative["roles/pubsub.subscriber"]:
     condition: []
     members:
-    - user:user1@example.com
+    - serviceAccount:service_account_email
-    project: my-project
+    project: project-id
     role: roles/pubsub.subscriber
     topic: my-topic
   module.pubsub.google_pubsub_topic_iam_binding.authoritative["roles/pubsub.viewer"]:
     condition: []
     members:
-    - group:foo@example.com
+    - group:organization-admins@example.org
-    project: my-project
+    project: project-id
     role: roles/pubsub.viewer
     topic: my-topic
 
 counts:
   google_pubsub_topic: 1
   google_pubsub_topic_iam_binding: 2
+  modules: 1
+  resources: 3

tests/modules/pubsub/examples/subscription-iam.yaml

@@ -12,16 +12,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 values:
+  module.pubsub.google_pubsub_subscription.default["test-1"]:
+    bigquery_config: []
+    cloud_storage_config: []
+    dead_letter_policy: []
+    enable_exactly_once_delivery: false
+    enable_message_ordering: false
+    filter: null
+    labels: null
+    message_retention_duration: 604800s
+    name: test-1
+    project: project-id
+    push_config: []
+    retain_acked_messages: false
+    retry_policy: []
+    timeouts: null
+    topic: my-topic
   module.pubsub.google_pubsub_subscription_iam_binding.authoritative["test-1.roles/pubsub.subscriber"]:
     condition: []
     members:
-    - user:user1@example.com
+    - serviceAccount:service_account_email
-    project: my-project
+    project: project-id
     role: roles/pubsub.subscriber
     subscription: test-1
+  module.pubsub.google_pubsub_topic.default:
+    kms_key_name: null
+    labels: null
+    message_retention_duration: null
+    name: my-topic
+    project: project-id
+    timeouts: null
 
 counts:
   google_pubsub_subscription: 1
   google_pubsub_subscription_iam_binding: 1
   google_pubsub_topic: 1
+  modules: 1
+  resources: 3

tests/modules/pubsub/examples/subscriptions.yaml

@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,37 +15,51 @@
 values:
   module.pubsub.google_pubsub_subscription.default["test-pull"]:
     bigquery_config: []
+    cloud_storage_config: []
     dead_letter_policy: []
-    enable_exactly_once_delivery: False
+    enable_exactly_once_delivery: false
-    enable_message_ordering: False
+    enable_message_ordering: false
     filter: null
     labels: null
     message_retention_duration: 604800s
     name: test-pull
-    project: my-project
+    project: project-id
     push_config: []
-    retain_acked_messages: False
+    retain_acked_messages: false
     retry_policy: []
+    timeouts: null
     topic: my-topic
   module.pubsub.google_pubsub_subscription.default["test-pull-override"]:
     bigquery_config: []
+    cloud_storage_config: []
     dead_letter_policy: []
-    enable_exactly_once_delivery: False
+    effective_labels:
-    enable_message_ordering: False
+      test: override
+    enable_exactly_once_delivery: false
+    enable_message_ordering: false
     filter: null
     labels:
       test: override
     message_retention_duration: 604800s
     name: test-pull-override
-    project: my-project
+    project: project-id
     push_config: []
-    retain_acked_messages: True
+    retain_acked_messages: true
     retry_policy: []
+    terraform_labels:
+      test: override
+    timeouts: null
     topic: my-topic
   module.pubsub.google_pubsub_topic.default:
+    kms_key_name: null
+    labels: null
+    message_retention_duration: null
     name: my-topic
-    project: my-project
+    project: project-id
+    timeouts: null
 
 counts:
   google_pubsub_subscription: 2
   google_pubsub_topic: 1
+  modules: 1
+  resources: 3