kovagoadi/hunfabric
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add cloud dataplex module (#1308)

Browse Source 
* Add dataplex module

* fix dataplex test

* resolve comments

* python test removed

* Change variable desc

* refactor variables

* fix typos

* fix assets & zones resources

* fix linting error

* fix tests

* fix typo
This commit is contained in:
Prabha Arya
2023-05-05 08:26:46 +01:00
committed by GitHub
parent 47bd0987fa
commit 23299f7bd7
7 changed files with 406 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
modules/cloud-dataplex/README.md Normal file
View File

@@ -0,0 +1,74 @@
# Cloud Dataplex instance with lake, zone & assests
This module manages the creation of Cloud Dataplex instance along with lake, zone & assets in single regions.
## Simple example
This example shows how to setup a Cloud Dataplex instance, lake, zone & asset creation in GCP project.
```hcl
module "dataplex" {
source = "./fabric/modules/cloud-dataplex"
name = "terraform-lake"
prefix = "test"
project_id = "myproject"
region = "europe-west2"
zones = {
zone_1 = {
type = "RAW"
discovery = true
assets = {
asset_1 = {
bucket_name = "asset_1"
cron_schedule = "15 15 * * *"
discovery_spec_enabled = true
resource_spec_type = "STORAGE_BUCKET"
}
}
},
zone_2 = {
type = "CURATED"
discovery = true
assets = {
asset_2 = {
bucket_name = "asset_2"
cron_schedule = "15 15 * * *"
discovery_spec_enabled = true
resource_spec_type = "STORAGE_BUCKET"
}
}
}
}
}
# tftest modules=1 resources=5
```
## TODO
- [ ] Add IAM support
- [ ] support different type of assets
- [ ] support multi-regions
<!-- BEGIN TFDOC -->
## Variables
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
| [name](variables.tf#L23) | Name of Dataplex Lake. | <code>string</code> | ✓ | |
| [prefix](variables.tf#L28) | Optional prefix used to generate Dataplex Lake. | <code>string</code> | ✓ | |
| [project_id](variables.tf#L33) | The ID of the project where this Dataplex Lake will be created. | <code>string</code> | ✓ | |
| [region](variables.tf#L38) | Region of the Dataplax Lake. | <code>string</code> | ✓ | |
| [zones](variables.tf#L43) | Dataplex lake zones, such as `RAW` and `CURATED`. | <code title="map&#40;object&#40;&#123;&#10; type &#61; string&#10; discovery &#61; optional&#40;bool, true&#41;&#10; assets &#61; map&#40;object&#40;&#123;&#10; bucket_name &#61; string&#10; cron_schedule &#61; optional&#40;string, &#34;15 15 &#42; &#42; &#42;&#34;&#41;&#10; discovery_spec_enabled &#61; optional&#40;bool, true&#41;&#10; resource_spec_type &#61; optional&#40;string, &#34;STORAGE_BUCKET&#34;&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | ✓ | |
| [location_type](variables.tf#L17) | The location type of the Dataplax Lake. | <code>string</code> | | <code>&#34;SINGLE_REGION&#34;</code> |
## Outputs
| name | description | sensitive |
|---|---|:---:|
| [assets](outputs.tf#L17) | Assets attached to the lake of Dataplex Lake. | |
| [lake](outputs.tf#L22) | The lake name of Dataplex Lake. | |
| [zones](outputs.tf#L27) | The zone name of Dataplex Lake. | |
<!-- END TFDOC -->

80
modules/cloud-dataplex/main.tf Normal file
View File

@@ -0,0 +1,80 @@
/**
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
locals {
prefix = var.prefix == null ? "" : "${var.prefix}-"
zone_assets = flatten([
for zone, zones_info in var.zones : [
for asset, asset_data in zones_info.assets : {
zone_name = zone
asset_name = asset
bucket_name = asset_data.bucket_name
cron_schedule = asset_data.cron_schedule
discovery_spec_enabled = asset_data.discovery_spec_enabled
resource_spec_type = asset_data.resource_spec_type
}
]
])
}
resource "google_dataplex_lake" "basic_lake" {
name = "${local.prefix}${var.name}"
location = var.region
provider = google-beta
project = var.project_id
}
resource "google_dataplex_zone" "basic_zone" {
for_each = var.zones
name = each.key
location = var.region
provider = google-beta
lake = google_dataplex_lake.basic_lake.name
type = each.value.type
discovery_spec {
enabled = each.value.discovery
}
resource_spec {
location_type = var.location_type
}
project = var.project_id
}
resource "google_dataplex_asset" "primary" {
for_each = {
for tm in local.zone_assets : "${tm.zone_name}-${tm.asset_name}" => tm
}
name = each.value.asset_name
location = var.region
provider = google-beta
lake = google_dataplex_lake.basic_lake.name
dataplex_zone = google_dataplex_zone.basic_zone[each.value.zone_name].name
discovery_spec {
enabled = each.value.discovery_spec_enabled
schedule = each.value.cron_schedule
}
resource_spec {
name = "projects/${var.project_id}/buckets/${each.value.bucket_name}"
type = each.value.resource_spec_type
}
project = var.project_id
}

31
modules/cloud-dataplex/outputs.tf Normal file
View File

@@ -0,0 +1,31 @@
/**
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
output "assets" {
description = "Assets attached to the lake of Dataplex Lake."
value = local.zone_assets[*]["asset_name"]
}
output "lake" {
description = "The lake name of Dataplex Lake."
value = google_dataplex_lake.basic_lake.name
}
output "zones" {
description = "The zone name of Dataplex Lake."
value = local.zone_assets[*]["zone_name"]
}

55
modules/cloud-dataplex/variables.tf Normal file
View File

@@ -0,0 +1,55 @@
/**
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
variable "location_type" {
description = "The location type of the Dataplax Lake."
type = string
default = "SINGLE_REGION"
}
variable "name" {
description = "Name of Dataplex Lake."
type = string
}
variable "prefix" {
description = "Optional prefix used to generate Dataplex Lake."
type = string
}
variable "project_id" {
description = "The ID of the project where this Dataplex Lake will be created."
type = string
}
variable "region" {
description = "Region of the Dataplax Lake."
type = string
}
variable "zones" {
description = "Dataplex lake zones, such as `RAW` and `CURATED`."
type = map(object({
type = string
discovery = optional(bool, true)
assets = map(object({
bucket_name = string
cron_schedule = optional(string, "15 15 * * *")
discovery_spec_enabled = optional(bool, true)
resource_spec_type = optional(string, "STORAGE_BUCKET")
}))
}))
}

29
modules/cloud-dataplex/versions.tf Normal file
View File

@@ -0,0 +1,29 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
terraform {
required_version = ">= 1.0.0"
required_providers {
google = {
source = "hashicorp/google"
version = ">= 4.47.0" # tftest
}
google-beta = {
source = "hashicorp/google-beta"
version = ">= 4.47.0" # tftest
}
}
}

30
tests/modules/cloud_dataplex/examples/dataplex_lake.tfvars Normal file
View File

@@ -0,0 +1,30 @@
name = "terraform-lake"
prefix = "test"
project_id = "myproject"
region = "europe-west2"
zones = {
zone_1 = {
type = "RAW"
discovery = true
assets = {
asset_1 = {
bucket_name = "asset_1"
cron_schedule = "15 15 * * *"
discovery_spec_enabled = true
resource_spec_type = "STORAGE_BUCKET"
}
}
},
zone_2 = {
type = "CURATED"
discovery = true
assets = {
asset_2 = {
bucket_name = "asset_2"
cron_schedule = "15 15 * * *"
discovery_spec_enabled = true
resource_spec_type = "STORAGE_BUCKET"
}
}
}
}

107
tests/modules/cloud_dataplex/examples/dataplex_lake.yaml Normal file
View File

@@ -0,0 +1,107 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
values:
google_dataplex_asset.primary["zone_1-asset_1"]:
dataplex_zone: zone_1
description: null
discovery_spec:
- enabled: true
exclude_patterns: null
include_patterns: null
schedule: 15 15 * * *
display_name: null
labels: null
lake: test-terraform-lake
location: europe-west2
name: asset_1
project: myproject
resource_spec:
- name: projects/myproject/buckets/asset_1
type: STORAGE_BUCKET
timeouts: null
google_dataplex_asset.primary["zone_2-asset_2"]:
dataplex_zone: zone_2
description: null
discovery_spec:
- enabled: true
exclude_patterns: null
include_patterns: null
schedule: 15 15 * * *
display_name: null
labels: null
lake: test-terraform-lake
location: europe-west2
name: asset_2
project: myproject
resource_spec:
- name: projects/myproject/buckets/asset_2
type: STORAGE_BUCKET
timeouts: null
google_dataplex_lake.basic_lake:
description: null
display_name: null
labels: null
location: europe-west2
metastore: []
name: test-terraform-lake
project: myproject
timeouts: null
google_dataplex_zone.basic_zone["zone_1"]:
description: null
discovery_spec:
- enabled: true
exclude_patterns: null
include_patterns: null
display_name: null
labels: null
lake: test-terraform-lake
location: europe-west2
name: zone_1
project: myproject
resource_spec:
- location_type: SINGLE_REGION
timeouts: null
type: RAW
google_dataplex_zone.basic_zone["zone_2"]:
description: null
discovery_spec:
- enabled: true
exclude_patterns: null
include_patterns: null
display_name: null
labels: null
lake: test-terraform-lake
location: europe-west2
name: zone_2
project: myproject
resource_spec:
- location_type: SINGLE_REGION
timeouts: null
type: CURATED
counts:
google_dataplex_asset: 2
google_dataplex_lake: 1
google_dataplex_zone: 2
modules: 0
resources: 5
outputs:
assets:
- asset_1
- asset_2
lake: test-terraform-lake
zones:
- zone_1
- zone_2