New bigquery-dataset module (#66)
* new bigquery-dataset module * update README * update top-level READMEs * update CHANGELOG
This commit is contained in:
Ludovico Magnocavallo
committed by
GitHub
GitHub
parent
a9d6bd1f1c
commit
c7bffbbac1
150
modules/bigquery-dataset/README.md
Normal file
150
modules/bigquery-dataset/README.md
Normal file
@@ -0,0 +1,150 @@
|
||||
# Google Cloud Bigquery Module
|
||||
|
||||
This module allows managing a single BigQuery dataset, including access configuration, tables and views.
|
||||
|
||||
## Examples
|
||||
|
||||
### Simple dataset with access configuration
|
||||
|
||||
Access configuration defaults to using incremental accesses, which add to the default ones set at dataset creation. You can use the `access_authoritative` variable to switch to authoritative mode and have full control over dataset-level access. Be sure to always have at least one `OWNER` access and to avoid duplicating accesses, or `terraform apply` will fail.
|
||||
|
||||
```hcl
|
||||
module "bigquery-dataset" {
|
||||
source = "./modules/bigquery-dataset"
|
||||
project_id = "my-project
|
||||
id = "my-dataset"
|
||||
access = [
|
||||
{
|
||||
role = "OWNER"
|
||||
identity_type = "group_by_email"
|
||||
identity = "dataset-owners@example.com"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Dataset options
|
||||
|
||||
Dataset options are set via the `options` variable. all options must be specified, but a `null` value can be set to options that need to use defaults.
|
||||
|
||||
```hcl
|
||||
module "bigquery-dataset" {
|
||||
source = "./modules/bigquery-dataset"
|
||||
project_id = "my-project
|
||||
id = "my-dataset"
|
||||
options = {
|
||||
default_table_expiration_ms = 3600000
|
||||
default_partition_expiration_ms = null
|
||||
delete_contents_on_destroy = false
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Tables and views
|
||||
|
||||
Tables are created via the `tables` variable, or the `view` variable for views. Support for external tables will be added in a future release.
|
||||
|
||||
```hcl
|
||||
module "bigquery-dataset" {
|
||||
source = "./modules/bigquery-dataset"
|
||||
project_id = "my-project
|
||||
id = "my-dataset"
|
||||
tables = {
|
||||
table_a = {
|
||||
friendly_name = "Table a"
|
||||
labels = {}
|
||||
options = null
|
||||
partitioning = null
|
||||
schema = file("table-a.json")
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
If partitioning is needed, populate the `partitioning` variable using either the `time` or `range` attribute.
|
||||
|
||||
```hcl
|
||||
module "bigquery-dataset" {
|
||||
source = "./modules/bigquery-dataset"
|
||||
project_id = "my-project
|
||||
id = "my-dataset"
|
||||
tables = {
|
||||
table_a = {
|
||||
friendly_name = "Table a"
|
||||
labels = {}
|
||||
options = null
|
||||
partitioning = {
|
||||
field = null
|
||||
range = null # use start/end/interval for range
|
||||
time = { type = "DAY", expiration_ms = null }
|
||||
}
|
||||
schema = file("table-a.json")
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
To create views use the `view` variable. If you're querying a table created by the same module `terraform apply` will initially fail and eventually succeed once the underlying table has been created. You can probably also use the module's output in the view's query to create a dependency on the table.
|
||||
|
||||
```hcl
|
||||
module "bigquery-dataset" {
|
||||
source = "./modules/bigquery-dataset"
|
||||
project_id = "my-project
|
||||
id = "my-dataset"
|
||||
tables = {
|
||||
table_a = {
|
||||
friendly_name = "Table a"
|
||||
labels = {}
|
||||
options = null
|
||||
partitioning = {
|
||||
field = null
|
||||
range = null # use start/end/interval for range
|
||||
time = { type = "DAY", expiration_ms = null }
|
||||
}
|
||||
schema = file("table-a.json")
|
||||
}
|
||||
}
|
||||
views = {
|
||||
view_a = {
|
||||
friendly_name = "View a"
|
||||
labels = {}
|
||||
query = "SELECT * from `my-project.my-dataset.table_a`"
|
||||
use_legacy_sql = false
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
<!-- BEGIN TFDOC -->
|
||||
## Variables
|
||||
|
||||
| name | description | type | required | default |
|
||||
|---|---|:---: |:---:|:---:|
|
||||
| id | Dataset id. | <code title="">string</code> | ✓ | |
|
||||
| project_id | Id of the project where datasets will be created. | <code title="">string</code> | ✓ | |
|
||||
| *access* | Dataset access rules keyed by role, valid identity types are `domain`, `group_by_email`, `special_group` and `user_by_email`. Mode can be controlled via the `access_authoritative` variable. | <code title="map(list(object({ identity_type = string identity = string })))">map(list(object({...})))</code> | | <code title="">{}</code> |
|
||||
| *access_authoritative* | Use authoritative access instead of additive. | <code title="">bool</code> | | <code title="">false</code> |
|
||||
| *access_views* | Dataset access rules for views. Mode can be controlled via the `access_authoritative` variable. | <code title="list(object({ project_id = string dataset_id = string table_id = string }))">list(object({...}))</code> | | <code title="">[]</code> |
|
||||
| *encryption_key* | Self link of the KMS key that will be used to protect destination table. | <code title="">string</code> | | <code title="">null</code> |
|
||||
| *friendly_name* | Dataset friendly name. | <code title="">string</code> | | <code title="">null</code> |
|
||||
| *labels* | Dataset labels. | <code title="map(string)">map(string)</code> | | <code title="">{}</code> |
|
||||
| *location* | Dataset location. | <code title="">string</code> | | <code title="">EU</code> |
|
||||
| *options* | Dataset options. | <code title="object({ default_table_expiration_ms = number default_partition_expiration_ms = number delete_contents_on_destroy = bool })">object({...})</code> | | <code title="{ default_table_expiration_ms = null default_partition_expiration_ms = null delete_contents_on_destroy = false }">...</code> |
|
||||
| *tables* | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | <code title="map(object({ friendly_name = string labels = map(string) options = object({ clustering = list(string) encryption_key = string expiration_time = number }) partitioning = object({ field = string range = object({ end = number interval = number start = number }) time = object({ expiration_ms = number type = string }) }) schema = string }))">map(object({...}))</code> | | <code title="">{}</code> |
|
||||
| *views* | View definitions. | <code title="map(object({ friendly_name = string labels = map(string) query = string use_legacy_sql = bool }))">map(object({...}))</code> | | <code title="">{}</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
| name | description | sensitive |
|
||||
|---|---|:---:|
|
||||
| dataset | Dataset resource. | |
|
||||
| dataset_id | Dataset full id. | |
|
||||
| id | Dataset id. | |
|
||||
| self_link | Dataset self link. | |
|
||||
| tables | Table resources. | |
|
||||
| views | View resources. | |
|
||||
<!-- END TFDOC -->
|
||||
|
||||
## TODO
|
||||
|
||||
- [ ] add support for tables
|
||||
173
modules/bigquery-dataset/main.tf
Normal file
173
modules/bigquery-dataset/main.tf
Normal file
@@ -0,0 +1,173 @@
|
||||
/**
|
||||
* Copyright 2019 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
locals {
|
||||
access_list = flatten([
|
||||
for role, accesses in var.access : [
|
||||
for access in accesses : merge(access, { role = role })
|
||||
]
|
||||
])
|
||||
access_default = {
|
||||
for access in local.access_list :
|
||||
"${access.role}-${access.identity_type}-${access.identity}" => access
|
||||
}
|
||||
access_views = {
|
||||
for access in var.access_views :
|
||||
"${access.project_id}-${access.dataset_id}-${access.table_id}" => access
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_bigquery_dataset" "default" {
|
||||
project = var.project_id
|
||||
dataset_id = var.id
|
||||
friendly_name = var.friendly_name
|
||||
description = "Terraform managed."
|
||||
labels = var.labels
|
||||
location = var.location
|
||||
|
||||
delete_contents_on_destroy = var.options.delete_contents_on_destroy
|
||||
default_table_expiration_ms = var.options.default_table_expiration_ms
|
||||
default_partition_expiration_ms = var.options.default_partition_expiration_ms
|
||||
|
||||
dynamic access {
|
||||
for_each = var.access_authoritative ? local.access_default : {}
|
||||
content {
|
||||
role = access.value.role
|
||||
domain = (
|
||||
access.value.identity_type == "domain" ? access.value.identity : null
|
||||
)
|
||||
group_by_email = (
|
||||
access.value.identity_type == "group_by_email" ? access.value.identity : null
|
||||
)
|
||||
special_group = (
|
||||
access.value.identity_type == "special_group" ? access.value.identity : null
|
||||
)
|
||||
user_by_email = (
|
||||
access.value.identity_type == "user_by_email" ? access.value.identity : null
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
dynamic access {
|
||||
for_each = var.access_authoritative ? local.access_views : {}
|
||||
content {
|
||||
view {
|
||||
project_id = access.value.project_id
|
||||
dataset_id = access.value.dataset_id
|
||||
table_id = access.value.table_id
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dynamic default_encryption_configuration {
|
||||
for_each = var.encryption_key == null ? [] : [""]
|
||||
content {
|
||||
kms_key_name = var.encryption_key
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
resource "google_bigquery_dataset_access" "default" {
|
||||
for_each = var.access_authoritative ? {} : local.access_default
|
||||
project = var.project_id
|
||||
dataset_id = google_bigquery_dataset.default.dataset_id
|
||||
role = each.value.role
|
||||
domain = (
|
||||
each.value.identity_type == "domain" ? each.value.identity : null
|
||||
)
|
||||
group_by_email = (
|
||||
each.value.identity_type == "group_by_email" ? each.value.identity : null
|
||||
)
|
||||
special_group = (
|
||||
each.value.identity_type == "special_group" ? each.value.identity : null
|
||||
)
|
||||
user_by_email = (
|
||||
each.value.identity_type == "user_by_email" ? each.value.identity : null
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
resource "google_bigquery_dataset_access" "views" {
|
||||
for_each = var.access_authoritative ? {} : local.access_views
|
||||
project = var.project_id
|
||||
dataset_id = google_bigquery_dataset.default.dataset_id
|
||||
view {
|
||||
project_id = each.value.project
|
||||
dataset_id = each.value.dataset_id
|
||||
table_id = each.value.table_id
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_bigquery_table" "default" {
|
||||
provider = google-beta
|
||||
for_each = var.tables
|
||||
project = var.project_id
|
||||
dataset_id = google_bigquery_dataset.default.dataset_id
|
||||
table_id = each.key
|
||||
friendly_name = each.value.friendly_name
|
||||
description = "Terraform managed."
|
||||
clustering = try(each.value.options.clustering, null)
|
||||
expiration_time = try(each.value.options.expiration_time, null)
|
||||
labels = each.value.labels
|
||||
schema = each.value.schema
|
||||
|
||||
dynamic encryption_configuration {
|
||||
for_each = try(each.value.options.encryption_key, null) != null ? [""] : []
|
||||
content {
|
||||
kms_key_name = each.value.options.encryption_key
|
||||
}
|
||||
}
|
||||
|
||||
dynamic range_partitioning {
|
||||
for_each = try(each.value.partitioning.range, null) != null ? [""] : []
|
||||
content {
|
||||
field = each.value.partitioning.field
|
||||
range {
|
||||
start = each.value.partitioning.range.start
|
||||
end = each.value.partitioning.range.end
|
||||
interval = each.value.partitioning.range.interval
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dynamic time_partitioning {
|
||||
for_each = try(each.value.partitioning.time, null) != null ? [""] : []
|
||||
content {
|
||||
expiration_ms = each.value.partitioning.time.expiration_ms
|
||||
field = each.value.partitioning.field
|
||||
type = each.value.partitioning.time.type
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
resource "google_bigquery_table" "views" {
|
||||
for_each = var.views
|
||||
project = var.project_id
|
||||
dataset_id = google_bigquery_dataset.default.dataset_id
|
||||
table_id = each.key
|
||||
friendly_name = each.value.friendly_name
|
||||
description = "Terraform managed."
|
||||
labels = each.value.labels
|
||||
|
||||
view {
|
||||
query = each.value.query
|
||||
use_legacy_sql = each.value.use_legacy_sql
|
||||
}
|
||||
|
||||
}
|
||||
45
modules/bigquery-dataset/outputs.tf
Normal file
45
modules/bigquery-dataset/outputs.tf
Normal file
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* Copyright 2019 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
output "dataset" {
|
||||
description = "Dataset resource."
|
||||
value = google_bigquery_dataset.default
|
||||
}
|
||||
|
||||
output "dataset_id" {
|
||||
description = "Dataset full id."
|
||||
value = google_bigquery_dataset.default.dataset_id
|
||||
}
|
||||
|
||||
output "id" {
|
||||
description = "Dataset id."
|
||||
value = google_bigquery_dataset.default.id
|
||||
}
|
||||
|
||||
output "self_link" {
|
||||
description = "Dataset self link."
|
||||
value = google_bigquery_dataset.default.self_link
|
||||
}
|
||||
|
||||
output "tables" {
|
||||
description = "Table resources."
|
||||
value = google_bigquery_table.default
|
||||
}
|
||||
|
||||
output "views" {
|
||||
description = "View resources."
|
||||
value = google_bigquery_table.views
|
||||
}
|
||||
126
modules/bigquery-dataset/variables.tf
Normal file
126
modules/bigquery-dataset/variables.tf
Normal file
@@ -0,0 +1,126 @@
|
||||
/**
|
||||
* Copyright 2019 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
variable "access" {
|
||||
description = "Dataset access rules keyed by role, valid identity types are `domain`, `group_by_email`, `special_group` and `user_by_email`. Mode can be controlled via the `access_authoritative` variable."
|
||||
type = map(list(object({
|
||||
identity_type = string
|
||||
identity = string
|
||||
})))
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "access_authoritative" {
|
||||
description = "Use authoritative access instead of additive."
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "access_views" {
|
||||
description = "Dataset access rules for views. Mode can be controlled via the `access_authoritative` variable."
|
||||
type = list(object({
|
||||
project_id = string
|
||||
dataset_id = string
|
||||
table_id = string
|
||||
}))
|
||||
default = []
|
||||
}
|
||||
|
||||
variable "encryption_key" {
|
||||
description = "Self link of the KMS key that will be used to protect destination table."
|
||||
type = string
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "labels" {
|
||||
description = "Dataset labels."
|
||||
type = map(string)
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "location" {
|
||||
description = "Dataset location."
|
||||
type = string
|
||||
default = "EU"
|
||||
}
|
||||
|
||||
variable "friendly_name" {
|
||||
description = "Dataset friendly name."
|
||||
type = string
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "id" {
|
||||
description = "Dataset id."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "options" {
|
||||
description = "Dataset options."
|
||||
type = object({
|
||||
default_table_expiration_ms = number
|
||||
default_partition_expiration_ms = number
|
||||
delete_contents_on_destroy = bool
|
||||
})
|
||||
default = {
|
||||
default_table_expiration_ms = null
|
||||
default_partition_expiration_ms = null
|
||||
delete_contents_on_destroy = false
|
||||
}
|
||||
}
|
||||
|
||||
variable "project_id" {
|
||||
description = "Id of the project where datasets will be created."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "tables" {
|
||||
description = "Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null."
|
||||
type = map(object({
|
||||
friendly_name = string
|
||||
labels = map(string)
|
||||
options = object({
|
||||
clustering = list(string)
|
||||
encryption_key = string
|
||||
expiration_time = number
|
||||
})
|
||||
partitioning = object({
|
||||
field = string
|
||||
range = object({
|
||||
end = number
|
||||
interval = number
|
||||
start = number
|
||||
})
|
||||
time = object({
|
||||
expiration_ms = number
|
||||
type = string
|
||||
})
|
||||
})
|
||||
schema = string
|
||||
}))
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "views" {
|
||||
description = "View definitions."
|
||||
type = map(object({
|
||||
friendly_name = string
|
||||
labels = map(string)
|
||||
query = string
|
||||
use_legacy_sql = bool
|
||||
}))
|
||||
default = {}
|
||||
}
|
||||
19
modules/bigquery-dataset/versions.tf
Normal file
19
modules/bigquery-dataset/versions.tf
Normal file
@@ -0,0 +1,19 @@
|
||||
/**
|
||||
* Copyright 2019 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
terraform {
|
||||
required_version = ">= 0.12.6"
|
||||
}
|
||||
Reference in New Issue
Block a user