diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md
index e14fbca42..ecbbe8f12 100644
--- a/modules/bigquery-dataset/README.md
+++ b/modules/bigquery-dataset/README.md
@@ -4,7 +4,6 @@ This module allows managing a single BigQuery dataset, including access configur
## TODO
-- [ ] add support for dynamic values in access identities
- [ ] check for dynamic values in tables and views
- [ ] add support for external tables
@@ -12,17 +11,24 @@ This module allows managing a single BigQuery dataset, including access configur
### Simple dataset with access configuration
-Access configuration defaults to using incremental accesses, which add to the default ones set at dataset creation. You can use the `access_authoritative` variable to switch to authoritative mode and have full control over dataset-level access. Be sure to always have at least one `OWNER` access and to avoid duplicating accesses, or `terraform apply` will fail.
+Access configuration defaults to using the separate `google_bigquery_dataset_access` resource, so as to leave the default dataset access rules untouched.
+
+You can choose to manage the `google_bigquery_dataset` access rules instead via the `dataset_access` variable, but be sure to always have at least one `OWNER` access and to avoid duplicating accesses, or `terraform apply` will fail.
+
+The access variables are split into `access_roles` and `access_identities` variables, so that dynamic values can be passed in for identities (eg a service account email generated by a different module or resource). The `access_views` variable is separate, so as to allow proper type constraints.
```hcl
module "bigquery-dataset" {
source = "./modules/bigquery-dataset"
project_id = "my-project
- id = "my-dataset"
- access = {
- "OWNER" = [
- { identity_type = "group_by_email", identity = "dataset-owners@example.com" }
- ]
+ id = "my-dataset"
+ access_roles = {
+ reader-group = { role = "READER", type = "group_by_email" }
+ owner = { role = "OWNER", type = "user_by_email" }
+ }
+ access_identities = {
+ reader-group = "playground-test@ludomagno.net"
+ owner = "ludo@ludomagno.net"
}
}
```
@@ -126,9 +132,10 @@ module "bigquery-dataset" {
|---|---|:---: |:---:|:---:|
| id | Dataset id. | string | ✓ | |
| project_id | Id of the project where datasets will be created. | string | ✓ | |
-| *access* | Dataset access rules keyed by role, valid identity types are `domain`, `group_by_email`, `special_group` and `user_by_email`. Mode can be controlled via the `access_authoritative` variable. | map(list(object({...}))) | | {} |
-| *access_authoritative* | Use authoritative access instead of additive. | bool | | false |
-| *access_views* | Dataset access rules for views. Mode can be controlled via the `access_authoritative` variable. | list(object({...})) | | [] |
+| *access_identities* | Map of access identities used for access roles with type different from `view`. A separate variable is needed as identities can be set to dynamic values. | map(string) | | {} |
+| *access_roles* | Map of access rules with role and identity type. Keys are arbitrary and only used to combine identities with each role. Valid types are `domain`, `group_by_email`, `special_group`, `user_by_email`, `view`. | map(object({...})) | | {} |
+| *access_views* | Map of view data for access roles with identity type equal to `view`. A separate variable is needed as identities can be set to dynamic values. | map(object({...})) | | {} |
+| *dataset_access* | Set access in the dataset resource instead of using separate resources. | bool | | false |
| *encryption_key* | Self link of the KMS key that will be used to protect destination table. | string | | null |
| *friendly_name* | Dataset friendly name. | string | | null |
| *labels* | Dataset labels. | map(string) | | {} |
@@ -142,10 +149,12 @@ module "bigquery-dataset" {
| name | description | sensitive |
|---|---|:---:|
| dataset | Dataset resource. | |
-| dataset_id | Dataset full id. | |
-| id | Dataset id. | |
+| dataset_id | Dataset id. | |
+| id | Fully qualified dataset id. | |
| self_link | Dataset self link. | |
+| table_ids | Map of fully qualified table ids keyed by table ids. | |
| tables | Table resources. | |
+| view_ids | Map of fully qualified view ids keyed by view ids. | |
| views | View resources. | |
diff --git a/modules/bigquery-dataset/main.tf b/modules/bigquery-dataset/main.tf
index 2e8f6cec2..4df526f77 100644
--- a/modules/bigquery-dataset/main.tf
+++ b/modules/bigquery-dataset/main.tf
@@ -15,18 +15,20 @@
*/
locals {
- access_list = flatten([
- for role, accesses in var.access : [
- for access in accesses : merge(access, { role = role })
- ]
- ])
- access_default = {
- for access in local.access_list :
- "${access.role}-${access.identity_type}-${access.identity}" => access
+ access_domain = {
+ for k, v in var.access_roles : k => v if v.type == "domain"
}
- access_views = {
- for access in var.access_views :
- "${access.project_id}-${access.dataset_id}-${access.table_id}" => access
+ access_group = {
+ for k, v in var.access_roles : k => v if v.type == "group_by_email"
+ }
+ access_special = {
+ for k, v in var.access_roles : k => v if v.type == "special_group"
+ }
+ access_user = {
+ for k, v in var.access_roles : k => v if v.type == "user_by_email"
+ }
+ access_view = {
+ for k, v in var.access_roles : k => v if v.type == "view"
}
}
@@ -43,31 +45,44 @@ resource "google_bigquery_dataset" "default" {
default_partition_expiration_ms = var.options.default_partition_expiration_ms
dynamic access {
- for_each = var.access_authoritative ? local.access_default : {}
+ for_each = var.dataset_access ? local.access_domain : {}
content {
- role = access.value.role
- domain = (
- access.value.identity_type == "domain" ? access.value.identity : null
- )
- group_by_email = (
- access.value.identity_type == "group_by_email" ? access.value.identity : null
- )
- special_group = (
- access.value.identity_type == "special_group" ? access.value.identity : null
- )
- user_by_email = (
- access.value.identity_type == "user_by_email" ? access.value.identity : null
- )
+ role = access.value.role
+ domain = try(var.access_identities[access.key])
}
}
dynamic access {
- for_each = var.access_authoritative ? local.access_views : {}
+ for_each = var.dataset_access ? local.access_group : {}
+ content {
+ role = access.value.role
+ group_by_email = try(var.access_identities[access.key])
+ }
+ }
+
+ dynamic access {
+ for_each = var.dataset_access ? local.access_special : {}
+ content {
+ role = access.value.role
+ special_group = try(var.access_identities[access.key])
+ }
+ }
+
+ dynamic access {
+ for_each = var.dataset_access ? local.access_user : {}
+ content {
+ role = access.value.role
+ user_by_email = try(var.access_identities[access.key])
+ }
+ }
+
+ dynamic access {
+ for_each = var.dataset_access ? local.access_view : {}
content {
view {
- project_id = access.value.project_id
- dataset_id = access.value.dataset_id
- table_id = access.value.table_id
+ project_id = try(var.access.views[access.key].project_id, null)
+ dataset_id = try(var.access.views[access.key].dataset_id, null)
+ table_id = try(var.access.views[access.key].table_id, null)
}
}
}
@@ -81,34 +96,46 @@ resource "google_bigquery_dataset" "default" {
}
-resource "google_bigquery_dataset_access" "default" {
- for_each = var.access_authoritative ? {} : local.access_default
+resource "google_bigquery_dataset_access" "domain" {
+ for_each = var.dataset_access ? {} : local.access_domain
project = var.project_id
dataset_id = google_bigquery_dataset.default.dataset_id
role = each.value.role
- domain = (
- each.value.identity_type == "domain" ? each.value.identity : null
- )
- group_by_email = (
- each.value.identity_type == "group_by_email" ? each.value.identity : null
- )
- special_group = (
- each.value.identity_type == "special_group" ? each.value.identity : null
- )
- user_by_email = (
- each.value.identity_type == "user_by_email" ? each.value.identity : null
- )
+ domain = try(var.access_identities[each.key])
}
+resource "google_bigquery_dataset_access" "group_by_email" {
+ for_each = var.dataset_access ? {} : local.access_group
+ project = var.project_id
+ dataset_id = google_bigquery_dataset.default.dataset_id
+ role = each.value.role
+ group_by_email = try(var.access_identities[each.key])
+}
+
+resource "google_bigquery_dataset_access" "special_group" {
+ for_each = var.dataset_access ? {} : local.access_special
+ project = var.project_id
+ dataset_id = google_bigquery_dataset.default.dataset_id
+ role = each.value.role
+ special_group = try(var.access_identities[each.key])
+}
+
+resource "google_bigquery_dataset_access" "user_by_email" {
+ for_each = var.dataset_access ? {} : local.access_user
+ project = var.project_id
+ dataset_id = google_bigquery_dataset.default.dataset_id
+ role = each.value.role
+ user_by_email = try(var.access_identities[each.key])
+}
resource "google_bigquery_dataset_access" "views" {
- for_each = var.access_authoritative ? {} : local.access_views
+ for_each = var.dataset_access ? {} : local.access_view
project = var.project_id
dataset_id = google_bigquery_dataset.default.dataset_id
view {
- project_id = each.value.project
- dataset_id = each.value.dataset_id
- table_id = each.value.table_id
+ project_id = try(var.access_views[each.key].project_id, null)
+ dataset_id = try(var.access_views[each.key].dataset_id, null)
+ table_id = try(var.access_views[each.key].table_id, null)
}
}
diff --git a/modules/bigquery-dataset/outputs.tf b/modules/bigquery-dataset/outputs.tf
index ede802fbc..44753fadb 100644
--- a/modules/bigquery-dataset/outputs.tf
+++ b/modules/bigquery-dataset/outputs.tf
@@ -20,18 +20,39 @@ output "dataset" {
}
output "dataset_id" {
- description = "Dataset full id."
+ description = "Dataset id."
value = google_bigquery_dataset.default.dataset_id
+ depends_on = [
+ google_bigquery_dataset_access.domain,
+ google_bigquery_dataset_access.group_by_email,
+ google_bigquery_dataset_access.special_group,
+ google_bigquery_dataset_access.user_by_email,
+ google_bigquery_dataset_access.views
+ ]
}
output "id" {
- description = "Dataset id."
+ description = "Fully qualified dataset id."
value = google_bigquery_dataset.default.id
+ depends_on = [
+ google_bigquery_dataset_access.domain,
+ google_bigquery_dataset_access.group_by_email,
+ google_bigquery_dataset_access.special_group,
+ google_bigquery_dataset_access.user_by_email,
+ google_bigquery_dataset_access.views
+ ]
}
output "self_link" {
description = "Dataset self link."
value = google_bigquery_dataset.default.self_link
+ depends_on = [
+ google_bigquery_dataset_access.domain,
+ google_bigquery_dataset_access.group_by_email,
+ google_bigquery_dataset_access.special_group,
+ google_bigquery_dataset_access.user_by_email,
+ google_bigquery_dataset_access.views
+ ]
}
output "tables" {
@@ -39,7 +60,17 @@ output "tables" {
value = google_bigquery_table.default
}
+output "table_ids" {
+ description = "Map of fully qualified table ids keyed by table ids."
+ value = { for k, v in google_bigquery_table.default : v.table_id => v.id }
+}
+
output "views" {
description = "View resources."
value = google_bigquery_table.views
}
+
+output "view_ids" {
+ description = "Map of fully qualified view ids keyed by view ids."
+ value = { for k, v in google_bigquery_table.views : v.table_id => v.id }
+}
diff --git a/modules/bigquery-dataset/variables.tf b/modules/bigquery-dataset/variables.tf
index fca9378fa..54555b54b 100644
--- a/modules/bigquery-dataset/variables.tf
+++ b/modules/bigquery-dataset/variables.tf
@@ -14,29 +14,35 @@
* limitations under the License.
*/
-variable "access" {
- description = "Dataset access rules keyed by role, valid identity types are `domain`, `group_by_email`, `special_group` and `user_by_email`. Mode can be controlled via the `access_authoritative` variable."
- type = map(list(object({
- identity_type = string
- identity = string
- })))
+variable "access_identities" {
+ description = "Map of access identities used for access roles with type different from `view`. A separate variable is needed as identities can be set to dynamic values."
+ type = map(string)
+ default = {}
+}
+
+variable "access_roles" {
+ description = "Map of access rules with role and identity type. Keys are arbitrary and only used to combine identities with each role. Valid types are `domain`, `group_by_email`, `special_group`, `user_by_email`, `view`."
+ type = map(object({
+ role = string
+ type = string
+ }))
default = {}
}
-variable "access_authoritative" {
- description = "Use authoritative access instead of additive."
- type = bool
- default = false
-}
-
variable "access_views" {
- description = "Dataset access rules for views. Mode can be controlled via the `access_authoritative` variable."
- type = list(object({
+ description = "Map of view data for access roles with identity type equal to `view`. A separate variable is needed as identities can be set to dynamic values."
+ type = map(object({
project_id = string
dataset_id = string
table_id = string
}))
- default = []
+ default = {}
+}
+
+variable "dataset_access" {
+ description = "Set access in the dataset resource instead of using separate resources."
+ type = bool
+ default = false
}
variable "encryption_key" {