diff --git a/modules/cloud-dataplex/README.md b/modules/cloud-dataplex/README.md new file mode 100644 index 000000000..8a46900d0 --- /dev/null +++ b/modules/cloud-dataplex/README.md @@ -0,0 +1,74 @@ +# Cloud Dataplex instance with lake, zone & assests + +This module manages the creation of Cloud Dataplex instance along with lake, zone & assets in single regions. + + +## Simple example + +This example shows how to setup a Cloud Dataplex instance, lake, zone & asset creation in GCP project. + +```hcl + +module "dataplex" { + source = "./fabric/modules/cloud-dataplex" + name = "terraform-lake" + prefix = "test" + project_id = "myproject" + region = "europe-west2" + zones = { + zone_1 = { + type = "RAW" + discovery = true + assets = { + asset_1 = { + bucket_name = "asset_1" + cron_schedule = "15 15 * * *" + discovery_spec_enabled = true + resource_spec_type = "STORAGE_BUCKET" + } + } + }, + zone_2 = { + type = "CURATED" + discovery = true + assets = { + asset_2 = { + bucket_name = "asset_2" + cron_schedule = "15 15 * * *" + discovery_spec_enabled = true + resource_spec_type = "STORAGE_BUCKET" + } + } + } + } +} + +# tftest modules=1 resources=5 +``` +## TODO + +- [ ] Add IAM support +- [ ] support different type of assets +- [ ] support multi-regions + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L23) | Name of Dataplex Lake. | string | ✓ | | +| [prefix](variables.tf#L28) | Optional prefix used to generate Dataplex Lake. | string | ✓ | | +| [project_id](variables.tf#L33) | The ID of the project where this Dataplex Lake will be created. | string | ✓ | | +| [region](variables.tf#L38) | Region of the Dataplax Lake. | string | ✓ | | +| [zones](variables.tf#L43) | Dataplex lake zones, such as `RAW` and `CURATED`. | map(object({…})) | ✓ | | +| [location_type](variables.tf#L17) | The location type of the Dataplax Lake. | string | | "SINGLE_REGION" | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [assets](outputs.tf#L17) | Assets attached to the lake of Dataplex Lake. | | +| [lake](outputs.tf#L22) | The lake name of Dataplex Lake. | | +| [zones](outputs.tf#L27) | The zone name of Dataplex Lake. | | + + diff --git a/modules/cloud-dataplex/main.tf b/modules/cloud-dataplex/main.tf new file mode 100644 index 000000000..af5ef018f --- /dev/null +++ b/modules/cloud-dataplex/main.tf @@ -0,0 +1,80 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + prefix = var.prefix == null ? "" : "${var.prefix}-" + zone_assets = flatten([ + for zone, zones_info in var.zones : [ + for asset, asset_data in zones_info.assets : { + zone_name = zone + asset_name = asset + bucket_name = asset_data.bucket_name + cron_schedule = asset_data.cron_schedule + discovery_spec_enabled = asset_data.discovery_spec_enabled + resource_spec_type = asset_data.resource_spec_type + } + ] + ]) +} + +resource "google_dataplex_lake" "basic_lake" { + name = "${local.prefix}${var.name}" + location = var.region + provider = google-beta + project = var.project_id +} + +resource "google_dataplex_zone" "basic_zone" { + for_each = var.zones + name = each.key + location = var.region + provider = google-beta + lake = google_dataplex_lake.basic_lake.name + type = each.value.type + + discovery_spec { + enabled = each.value.discovery + } + + resource_spec { + location_type = var.location_type + } + + project = var.project_id +} + +resource "google_dataplex_asset" "primary" { + for_each = { + for tm in local.zone_assets : "${tm.zone_name}-${tm.asset_name}" => tm + } + name = each.value.asset_name + location = var.region + provider = google-beta + + lake = google_dataplex_lake.basic_lake.name + dataplex_zone = google_dataplex_zone.basic_zone[each.value.zone_name].name + + discovery_spec { + enabled = each.value.discovery_spec_enabled + schedule = each.value.cron_schedule + } + + resource_spec { + name = "projects/${var.project_id}/buckets/${each.value.bucket_name}" + type = each.value.resource_spec_type + } + project = var.project_id +} diff --git a/modules/cloud-dataplex/outputs.tf b/modules/cloud-dataplex/outputs.tf new file mode 100644 index 000000000..bdc5562ea --- /dev/null +++ b/modules/cloud-dataplex/outputs.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "assets" { + description = "Assets attached to the lake of Dataplex Lake." + value = local.zone_assets[*]["asset_name"] +} + +output "lake" { + description = "The lake name of Dataplex Lake." + value = google_dataplex_lake.basic_lake.name +} + +output "zones" { + description = "The zone name of Dataplex Lake." + value = local.zone_assets[*]["zone_name"] +} + diff --git a/modules/cloud-dataplex/variables.tf b/modules/cloud-dataplex/variables.tf new file mode 100644 index 000000000..bbbba1c79 --- /dev/null +++ b/modules/cloud-dataplex/variables.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "location_type" { + description = "The location type of the Dataplax Lake." + type = string + default = "SINGLE_REGION" +} + +variable "name" { + description = "Name of Dataplex Lake." + type = string +} + +variable "prefix" { + description = "Optional prefix used to generate Dataplex Lake." + type = string +} + +variable "project_id" { + description = "The ID of the project where this Dataplex Lake will be created." + type = string +} + +variable "region" { + description = "Region of the Dataplax Lake." + type = string +} + +variable "zones" { + description = "Dataplex lake zones, such as `RAW` and `CURATED`." + type = map(object({ + type = string + discovery = optional(bool, true) + assets = map(object({ + bucket_name = string + cron_schedule = optional(string, "15 15 * * *") + discovery_spec_enabled = optional(bool, true) + resource_spec_type = optional(string, "STORAGE_BUCKET") + })) + })) +} diff --git a/modules/cloud-dataplex/versions.tf b/modules/cloud-dataplex/versions.tf new file mode 100644 index 000000000..111b4bf75 --- /dev/null +++ b/modules/cloud-dataplex/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.0.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.47.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.47.0" # tftest + } + } +} + + diff --git a/tests/modules/cloud_dataplex/examples/dataplex_lake.tfvars b/tests/modules/cloud_dataplex/examples/dataplex_lake.tfvars new file mode 100644 index 000000000..f47c23067 --- /dev/null +++ b/tests/modules/cloud_dataplex/examples/dataplex_lake.tfvars @@ -0,0 +1,30 @@ +name = "terraform-lake" +prefix = "test" +project_id = "myproject" +region = "europe-west2" +zones = { + zone_1 = { + type = "RAW" + discovery = true + assets = { + asset_1 = { + bucket_name = "asset_1" + cron_schedule = "15 15 * * *" + discovery_spec_enabled = true + resource_spec_type = "STORAGE_BUCKET" + } + } + }, + zone_2 = { + type = "CURATED" + discovery = true + assets = { + asset_2 = { + bucket_name = "asset_2" + cron_schedule = "15 15 * * *" + discovery_spec_enabled = true + resource_spec_type = "STORAGE_BUCKET" + } + } + } +} diff --git a/tests/modules/cloud_dataplex/examples/dataplex_lake.yaml b/tests/modules/cloud_dataplex/examples/dataplex_lake.yaml new file mode 100644 index 000000000..386551016 --- /dev/null +++ b/tests/modules/cloud_dataplex/examples/dataplex_lake.yaml @@ -0,0 +1,107 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +values: + google_dataplex_asset.primary["zone_1-asset_1"]: + dataplex_zone: zone_1 + description: null + discovery_spec: + - enabled: true + exclude_patterns: null + include_patterns: null + schedule: 15 15 * * * + display_name: null + labels: null + lake: test-terraform-lake + location: europe-west2 + name: asset_1 + project: myproject + resource_spec: + - name: projects/myproject/buckets/asset_1 + type: STORAGE_BUCKET + timeouts: null + google_dataplex_asset.primary["zone_2-asset_2"]: + dataplex_zone: zone_2 + description: null + discovery_spec: + - enabled: true + exclude_patterns: null + include_patterns: null + schedule: 15 15 * * * + display_name: null + labels: null + lake: test-terraform-lake + location: europe-west2 + name: asset_2 + project: myproject + resource_spec: + - name: projects/myproject/buckets/asset_2 + type: STORAGE_BUCKET + timeouts: null + google_dataplex_lake.basic_lake: + description: null + display_name: null + labels: null + location: europe-west2 + metastore: [] + name: test-terraform-lake + project: myproject + timeouts: null + google_dataplex_zone.basic_zone["zone_1"]: + description: null + discovery_spec: + - enabled: true + exclude_patterns: null + include_patterns: null + display_name: null + labels: null + lake: test-terraform-lake + location: europe-west2 + name: zone_1 + project: myproject + resource_spec: + - location_type: SINGLE_REGION + timeouts: null + type: RAW + google_dataplex_zone.basic_zone["zone_2"]: + description: null + discovery_spec: + - enabled: true + exclude_patterns: null + include_patterns: null + display_name: null + labels: null + lake: test-terraform-lake + location: europe-west2 + name: zone_2 + project: myproject + resource_spec: + - location_type: SINGLE_REGION + timeouts: null + type: CURATED + +counts: + google_dataplex_asset: 2 + google_dataplex_lake: 1 + google_dataplex_zone: 2 + modules: 0 + resources: 5 + +outputs: + assets: + - asset_1 + - asset_2 + lake: test-terraform-lake + zones: + - zone_1 + - zone_2