From 461662ad1db9719ccb987c3eaec171ee67ae97c4 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Fri, 22 May 2020 12:30:24 +0200 Subject: [PATCH] First commit for Data Fusion module (#77) * First commit for Data Fusion module * Fix comments and improve ip_allocation logic. * Add example * generate variables and outputs table, fix example headings * update year in boilerplates * rename firewall variable, remove region from resource names Co-authored-by: Ludovico Magnocavallo --- modules/datafusion/README.md | 63 +++++++++++++++++++++ modules/datafusion/main.tf | 79 ++++++++++++++++++++++++++ modules/datafusion/outputs.tf | 45 +++++++++++++++ modules/datafusion/variables.tf | 99 +++++++++++++++++++++++++++++++++ modules/datafusion/versions.tf | 19 +++++++ 5 files changed, 305 insertions(+) create mode 100644 modules/datafusion/README.md create mode 100644 modules/datafusion/main.tf create mode 100644 modules/datafusion/outputs.tf create mode 100644 modules/datafusion/variables.tf create mode 100644 modules/datafusion/versions.tf diff --git a/modules/datafusion/README.md b/modules/datafusion/README.md new file mode 100644 index 000000000..75fddc654 --- /dev/null +++ b/modules/datafusion/README.md @@ -0,0 +1,63 @@ +# Google Cloud Data Fusion Module + +This module allows simple management of ['Google Data Fusion'](https://cloud.google.com/data-fusion) instances. It supports creating Basic or Enterprise, public or private instances. + +## Examples + +## Auto-managed IP allocation + +```hcl +module "datafusion" { + source = "./modules/datafusion" + name = "my-datafusion" + region = "europe-west1" + project_id = "my-project" + network = "my-network-name" +} +``` + +### Externally managed IP allocation + +```hcl +module "datafusion" { + source = "./modules/datafusion" + name = "my-datafusion" + region = "europe-west1" + project_id = "my-project" + network = "my-network-name" + ip_allocation_create = false + ip_allocation = "10.0.0.0/22" +} +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| name | Name of the DataFusion instance. | string | ✓ | | +| network | Name of the network in the project with which the tenant project will be peered for executing pipelines in the form of projects/{project-id}/global/networks/{network} | string | ✓ | | +| project_id | Project ID. | string | ✓ | | +| region | DataFusion region. | string | ✓ | | +| *description* | DataFuzion instance description. | string | | Terraform managed. | +| *enable_stackdriver_logging* | Option to enable Stackdriver Logging. | bool | | false | +| *enable_stackdriver_monitoring* | Option to enable Stackdriver Monitorig. | bool | | false | +| *firewall_create* | Create Network firewall rules to enable SSH. | bool | | true | +| *ip_allocation* | Ip allocated for datafusion instance when not using the auto created one and created outside of the module. | string | | null | +| *ip_allocation_create* | Create Ip range for datafusion instance. | bool | | true | +| *labels* | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string) | | {} | +| *network_peering* | Create Network peering between project and DataFusion tenant project. | bool | | true | +| *private_instance* | Create private instance. | bool | | true | +| *type* | Datafusion Instance type. It can be BASIC or ENTERPRISE (default value). | string | | ENTERPRISE | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| id | DataFusion instance ID. | | +| ip_allocation | IP range reserved for Data Fusion instance in case of a private instance. | | +| resource | DataFusion resource. | | +| service_account | DataFusion Service Account. | | +| service_endpoint | DataFusion Service Endpoint. | | +| version | DataFusion version. | | + diff --git a/modules/datafusion/main.tf b/modules/datafusion/main.tf new file mode 100644 index 000000000..9161c0136 --- /dev/null +++ b/modules/datafusion/main.tf @@ -0,0 +1,79 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + prefix_length = 22 + ip_allocation = ( + var.ip_allocation_create + ? "${google_compute_global_address.default[0].address}/${local.prefix_length}" + : var.ip_allocation + ) + tenant_project = regex( + "cloud-datafusion-management-sa@([\\w-]+).iam.gserviceaccount.com", + google_data_fusion_instance.default.service_account + )[0] +} + +resource "google_compute_global_address" "default" { + count = var.ip_allocation_create ? 1 : 0 + project = var.project_id + name = "cdf-${var.name}" + address_type = "INTERNAL" + purpose = "VPC_PEERING" + prefix_length = local.prefix_length + network = var.network +} + +resource "google_compute_network_peering" "default" { + count = var.network_peering == true ? 1 : 0 + name = "cdf-${var.name}" + network = "projects/${var.project_id}/global/networks/${var.network}" + peer_network = "projects/${local.tenant_project}/global/networks/${var.region}-${google_data_fusion_instance.default.name}" + export_custom_routes = true + import_custom_routes = true +} + +resource "google_compute_firewall" "default" { + count = var.firewall_create == true ? 1 : 0 + name = "${var.name}-allow-ssh" + project = var.project_id + network = var.network + source_ranges = [local.ip_allocation] + target_tags = ["${var.name}-allow-ssh"] + + allow { + protocol = "tcp" + ports = ["22"] + } +} + +resource "google_data_fusion_instance" "default" { + provider = google-beta + project = var.project_id + name = var.name + type = var.type + description = var.description + labels = var.labels + region = var.region + private_instance = var.private_instance + enable_stackdriver_logging = var.enable_stackdriver_logging + enable_stackdriver_monitoring = var.enable_stackdriver_monitoring + network_config { + network = var.network + ip_allocation = local.ip_allocation + } +} + diff --git a/modules/datafusion/outputs.tf b/modules/datafusion/outputs.tf new file mode 100644 index 000000000..92adf5e46 --- /dev/null +++ b/modules/datafusion/outputs.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "DataFusion instance ID." + value = google_data_fusion_instance.default.id +} + +output "ip_allocation" { + description = "IP range reserved for Data Fusion instance in case of a private instance." + value = "${local.ip_allocation}" +} + +output "resource" { + description = "DataFusion resource." + value = google_data_fusion_instance.default +} + +output "service_account" { + description = "DataFusion Service Account." + value = google_data_fusion_instance.default.service_account +} + +output "service_endpoint" { + description = "DataFusion Service Endpoint." + value = google_data_fusion_instance.default.service_endpoint +} + +output "version" { + description = "DataFusion version." + value = google_data_fusion_instance.default.version +} diff --git a/modules/datafusion/variables.tf b/modules/datafusion/variables.tf new file mode 100644 index 000000000..f10a6e3f1 --- /dev/null +++ b/modules/datafusion/variables.tf @@ -0,0 +1,99 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +############################################################################### +# DtaFusion variables # +############################################################################### + +variable "description" { + description = "DataFuzion instance description." + type = string + default = "Terraform managed." +} + +variable "enable_stackdriver_logging" { + description = "Option to enable Stackdriver Logging." + type = bool + default = false +} + +variable "enable_stackdriver_monitoring" { + description = "Option to enable Stackdriver Monitorig." + type = bool + default = false +} + +variable "labels" { + description = "The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs." + type = map(string) + default = {} +} + +variable "name" { + description = "Name of the DataFusion instance." + type = string +} + +variable "network" { + description = "Name of the network in the project with which the tenant project will be peered for executing pipelines in the form of projects/{project-id}/global/networks/{network}" + type = string +} + +variable "firewall_create" { + description = "Create Network firewall rules to enable SSH." + type = bool + default = true +} + +variable "network_peering" { + description = "Create Network peering between project and DataFusion tenant project." + type = bool + default = true +} + +variable "private_instance" { + description = "Create private instance." + type = bool + default = true +} + +variable "project_id" { + description = "Project ID." + type = string +} + +variable "region" { + description = "DataFusion region." + type = string +} + +variable "ip_allocation_create" { + description = "Create Ip range for datafusion instance." + type = bool + default = true +} + +variable "ip_allocation" { + description = "Ip allocated for datafusion instance when not using the auto created one and created outside of the module." + type = string + default = null +} + +variable "type" { + description = "Datafusion Instance type. It can be BASIC or ENTERPRISE (default value)." + type = string + default = "ENTERPRISE" +} diff --git a/modules/datafusion/versions.tf b/modules/datafusion/versions.tf new file mode 100644 index 000000000..bc4c2a9d7 --- /dev/null +++ b/modules/datafusion/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +}