244 lines
12 KiB
Markdown
244 lines
12 KiB
Markdown
# Google Cloud Pub/Sub Module
|
|
|
|
This module allows managing a single Pub/Sub topic, including multiple subscriptions and IAM bindings at the topic and subscriptions levels, as well as schemas.
|
|
|
|
<!-- BEGIN TOC -->
|
|
- [Simple topic with IAM](#simple-topic-with-iam)
|
|
- [Topic with schema](#topic-with-schema)
|
|
- [Subscriptions](#subscriptions)
|
|
- [Push subscriptions](#push-subscriptions)
|
|
- [BigQuery subscriptions](#bigquery-subscriptions)
|
|
- [BigQuery Subscription with service account email](#bigquery-subscription-with-service-account-email)
|
|
- [Cloud Storage subscriptions](#cloud-storage-subscriptions)
|
|
- [Subscriptions with IAM](#subscriptions-with-iam)
|
|
- [Variables](#variables)
|
|
- [Outputs](#outputs)
|
|
- [Fixtures](#fixtures)
|
|
<!-- END TOC -->
|
|
|
|
## Simple topic with IAM
|
|
|
|
```hcl
|
|
module "pubsub" {
|
|
source = "./fabric/modules/pubsub"
|
|
project_id = var.project_id
|
|
name = "my-topic"
|
|
iam = {
|
|
"roles/pubsub.viewer" = ["group:${var.group_email}"]
|
|
"roles/pubsub.subscriber" = ["serviceAccount:${var.service_account.email}"]
|
|
}
|
|
}
|
|
# tftest modules=1 resources=3 inventory=simple.yaml e2e
|
|
```
|
|
|
|
## Topic with schema
|
|
|
|
```hcl
|
|
module "topic_with_schema" {
|
|
source = "./fabric/modules/pubsub"
|
|
project_id = var.project_id
|
|
name = "my-topic"
|
|
schema = {
|
|
msg_encoding = "JSON"
|
|
schema_type = "AVRO"
|
|
definition = jsonencode({
|
|
"type" = "record",
|
|
"name" = "Avro",
|
|
"fields" = [{
|
|
"name" = "StringField",
|
|
"type" = "string"
|
|
},
|
|
{
|
|
"name" = "FloatField",
|
|
"type" = "float"
|
|
},
|
|
{
|
|
"name" = "BooleanField",
|
|
"type" = "boolean"
|
|
},
|
|
]
|
|
})
|
|
}
|
|
}
|
|
# tftest modules=1 resources=2 inventory=schema.yaml e2e
|
|
```
|
|
|
|
## Subscriptions
|
|
|
|
Subscriptions are defined with the `subscriptions` variable, allowing optional configuration of per-subscription defaults. Push subscriptions need extra configuration, shown in the following example.
|
|
|
|
```hcl
|
|
module "pubsub" {
|
|
source = "./fabric/modules/pubsub"
|
|
project_id = var.project_id
|
|
name = "my-topic"
|
|
labels = { test = "default" }
|
|
subscriptions = {
|
|
test-pull = {}
|
|
test-pull-override = {
|
|
labels = { test = "override" }
|
|
retain_acked_messages = true
|
|
}
|
|
}
|
|
}
|
|
# tftest modules=1 resources=3 inventory=subscriptions.yaml e2e
|
|
```
|
|
|
|
## Push subscriptions
|
|
|
|
Push subscriptions need extra configuration in the `push_configs` variable.
|
|
|
|
```hcl
|
|
module "pubsub" {
|
|
source = "./fabric/modules/pubsub"
|
|
project_id = var.project_id
|
|
name = "my-topic"
|
|
subscriptions = {
|
|
test-push = {
|
|
push = {
|
|
endpoint = "https://example.com/foo"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
# tftest modules=1 resources=2 inventory=push-subscription.yaml e2e
|
|
```
|
|
|
|
## BigQuery subscriptions
|
|
|
|
BigQuery subscriptions need extra configuration in the `bigquery_subscription_configs` variable.
|
|
|
|
```hcl
|
|
module "pubsub" {
|
|
source = "./fabric/modules/pubsub"
|
|
project_id = var.project_id
|
|
name = "my-topic"
|
|
subscriptions = {
|
|
test-bigquery = {
|
|
bigquery = {
|
|
table = "${module.bigquery-dataset.tables["my_table"].project}:${module.bigquery-dataset.tables["my_table"].dataset_id}.${module.bigquery-dataset.tables["my_table"].table_id}"
|
|
use_topic_schema = true
|
|
write_metadata = false
|
|
drop_unknown_fields = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
# tftest modules=2 resources=5 fixtures=fixtures/bigquery-dataset.tf inventory=bigquery-subscription.yaml e2e
|
|
```
|
|
|
|
## BigQuery Subscription with service account email
|
|
|
|
BigQuery subscription example configuration with service account email.
|
|
|
|
```hcl
|
|
module "iam-service-account" {
|
|
source = "./fabric/modules/iam-service-account"
|
|
project_id = var.project_id
|
|
name = "fixture-service-account"
|
|
iam_project_roles = {
|
|
"${var.project_id}" = [
|
|
"roles/bigquery.dataEditor",
|
|
]
|
|
}
|
|
}
|
|
|
|
module "pubsub" {
|
|
source = "./fabric/modules/pubsub"
|
|
project_id = var.project_id
|
|
name = "my-topic"
|
|
subscriptions = {
|
|
test-bigquery-with-service-account = {
|
|
bigquery = {
|
|
table = "${module.bigquery-dataset.tables["my_table"].project}:${module.bigquery-dataset.tables["my_table"].dataset_id}.${module.bigquery-dataset.tables["my_table"].table_id}"
|
|
use_table_schema = true
|
|
write_metadata = false
|
|
service_account_email = module.iam-service-account.email
|
|
}
|
|
}
|
|
}
|
|
depends_on = [
|
|
module.iam-service-account # wait for IAM grants to finish
|
|
]
|
|
}
|
|
# tftest fixtures=fixtures/bigquery-dataset.tf inventory=bigquery-subscription-with-service-account.yaml e2e
|
|
```
|
|
|
|
## Cloud Storage subscriptions
|
|
|
|
Cloud Storage subscriptions need extra configuration in the `cloud_storage_subscription_configs` variable.
|
|
|
|
```hcl
|
|
module "pubsub" {
|
|
source = "./fabric/modules/pubsub"
|
|
project_id = var.project_id
|
|
name = "my-topic"
|
|
subscriptions = {
|
|
test-cloudstorage = {
|
|
cloud_storage = {
|
|
bucket = module.gcs.name
|
|
filename_prefix = var.prefix
|
|
filename_suffix = "test_suffix"
|
|
max_duration = "100s"
|
|
max_bytes = 1000
|
|
avro_config = {
|
|
write_metadata = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
# tftest modules=2 resources=4 fixtures=fixtures/gcs.tf inventory=cloud-storage-subscription.yaml e2e
|
|
```
|
|
|
|
## Subscriptions with IAM
|
|
|
|
```hcl
|
|
module "pubsub" {
|
|
source = "./fabric/modules/pubsub"
|
|
project_id = var.project_id
|
|
name = "my-topic"
|
|
subscriptions = {
|
|
test-1 = {
|
|
iam = {
|
|
"roles/pubsub.subscriber" = ["serviceAccount:${var.service_account.email}"]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
# tftest modules=1 resources=3 inventory=subscription-iam.yaml e2e
|
|
```
|
|
<!-- BEGIN TFDOC -->
|
|
## Variables
|
|
|
|
| name | description | type | required | default |
|
|
|---|---|:---:|:---:|:---:|
|
|
| [name](variables.tf#L73) | PubSub topic name. | <code>string</code> | ✓ | |
|
|
| [project_id](variables.tf#L78) | Project used for resources. | <code>string</code> | ✓ | |
|
|
| [iam](variables.tf#L17) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
|
| [iam_bindings](variables.tf#L24) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | <code title="map(object({ members = list(string) role = string condition = optional(object({ expression = string title = string description = optional(string) })) }))">map(object({…}))</code> | | <code>{}</code> |
|
|
| [iam_bindings_additive](variables.tf#L39) | Keyring individual additive IAM bindings. Keys are arbitrary. | <code title="map(object({ member = string role = string condition = optional(object({ expression = string title = string description = optional(string) })) }))">map(object({…}))</code> | | <code>{}</code> |
|
|
| [kms_key](variables.tf#L54) | KMS customer managed encryption key. | <code>string</code> | | <code>null</code> |
|
|
| [labels](variables.tf#L60) | Labels. | <code>map(string)</code> | | <code>{}</code> |
|
|
| [message_retention_duration](variables.tf#L67) | Minimum duration to retain a message after it is published to the topic. | <code>string</code> | | <code>null</code> |
|
|
| [regions](variables.tf#L83) | List of regions used to set persistence policy. | <code>list(string)</code> | | <code>[]</code> |
|
|
| [schema](variables.tf#L90) | Topic schema. If set, all messages in this topic should follow this schema. | <code title="object({ definition = string msg_encoding = optional(string, "ENCODING_UNSPECIFIED") schema_type = string })">object({…})</code> | | <code>null</code> |
|
|
| [subscriptions](variables.tf#L100) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | <code title="map(object({ ack_deadline_seconds = optional(number) enable_exactly_once_delivery = optional(bool, false) enable_message_ordering = optional(bool, false) expiration_policy_ttl = optional(string) filter = optional(string) iam = optional(map(list(string)), {}) labels = optional(map(string)) message_retention_duration = optional(string) retain_acked_messages = optional(bool, false) bigquery = optional(object({ table = string drop_unknown_fields = optional(bool, false) service_account_email = optional(string) use_table_schema = optional(bool, false) use_topic_schema = optional(bool, false) write_metadata = optional(bool, false) })) cloud_storage = optional(object({ bucket = string filename_prefix = optional(string) filename_suffix = optional(string) max_duration = optional(string) max_bytes = optional(number) avro_config = optional(object({ write_metadata = optional(bool, false) })) })) dead_letter_policy = optional(object({ topic = string max_delivery_attempts = optional(number) })) iam_bindings = optional(map(object({ members = list(string) role = string condition = optional(object({ expression = string title = string description = optional(string) })) })), {}) iam_bindings_additive = optional(map(object({ member = string role = string condition = optional(object({ expression = string title = string description = optional(string) })) })), {}) push = optional(object({ endpoint = string attributes = optional(map(string)) no_wrapper = optional(bool, false) oidc_token = optional(object({ audience = optional(string) service_account_email = string })) })) retry_policy = optional(object({ minimum_backoff = optional(number) maximum_backoff = optional(number) })) }))">map(object({…}))</code> | | <code>{}</code> |
|
|
|
|
## Outputs
|
|
|
|
| name | description | sensitive |
|
|
|---|---|:---:|
|
|
| [id](outputs.tf#L17) | Fully qualified topic id. | |
|
|
| [schema](outputs.tf#L28) | Schema resource. | |
|
|
| [schema_id](outputs.tf#L33) | Schema resource id. | |
|
|
| [subscription_id](outputs.tf#L38) | Subscription ids. | |
|
|
| [subscriptions](outputs.tf#L50) | Subscription resources. | |
|
|
| [topic](outputs.tf#L60) | Topic resource. | |
|
|
|
|
## Fixtures
|
|
|
|
- [bigquery-dataset.tf](../../tests/fixtures/bigquery-dataset.tf)
|
|
- [gcs.tf](../../tests/fixtures/gcs.tf)
|
|
<!-- END TFDOC -->
|