kovagoadi/hunfabric
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

E2E tests for dataproc + cleanup

Browse Source
This commit is contained in:
Wiktor Niesiobędzki
2024-03-08 18:51:53 +00:00
committed by Wiktor Niesiobędzki
parent de917b722d
commit 1a235cbcec
4 changed files with 156 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
modules/dataproc/README.md
View File

File diff suppressed because one or more lines are too long

6
modules/dataproc/main.tf
View File

@@ -16,12 +16,8 @@
# tfdoc:file:description Cloud Dataproc resource definition.
locals {
prefix = var.prefix == null ? "" : "${var.prefix}-"
}
resource "google_dataproc_cluster" "cluster" {
name = "${local.prefix}${var.name}"
name = var.name
project = var.project_id
region = var.region
graceful_decommission_timeout = var.dataproc_config.graceful_decommission_timeout

36
modules/dataproc/outputs.tf
View File

@@ -16,29 +16,31 @@
# tfdoc:file:description Cloud Dataproc module output.
output "bucket_names" {
description = "List of bucket names which have been assigned to the cluster."
value = google_dataproc_cluster.cluster.cluster_config.0.bucket
}
output "http_ports" {
description = "The map of port descriptions to URLs."
value = google_dataproc_cluster.cluster.cluster_config.0.endpoint_config.0.http_ports
}
# FIXME: 2024-03-08: broken in provider
#output "bucket_names" {
# description = "List of bucket names which have been assigned to the cluster."
# value = google_dataproc_cluster.cluster.cluster_config.0.bucket
#}
#
#output "http_ports" {
# description = "The map of port descriptions to URLs."
# value = google_dataproc_cluster.cluster.cluster_config.0.endpoint_config.0.http_ports
#}
output "id" {
description = "Fully qualified cluster id."
value = google_dataproc_cluster.cluster.id
}
output "instance_names" {
description = "List of instance names which have been assigned to the cluster."
value = {
master = google_dataproc_cluster.cluster.cluster_config.0.master_config.0.instance_names
worker = google_dataproc_cluster.cluster.cluster_config.0.worker_config.0.instance_names
preemptible_worker = google_dataproc_cluster.cluster.cluster_config.0.preemptible_worker_config.0.instance_names
}
}
# FIXME: 2024-03-08: broken in provider
#output "instance_names" {
# description = "List of instance names which have been assigned to the cluster."
# value = {
# master = google_dataproc_cluster.cluster.cluster_config.0.master_config.0.instance_names
# worker = google_dataproc_cluster.cluster.cluster_config.0.worker_config.0.instance_names
# preemptible_worker = google_dataproc_cluster.cluster.cluster_config.0.preemptible_worker_config.0.instance_names
# }
#}
output "name" {
description = "The name of the cluster."

16
modules/dataproc/variables.tf
View File

@@ -193,16 +193,6 @@ variable "name" {
type = string
}
variable "prefix" {
description = "Optional prefix used to generate project id and name."
type = string
default = null
validation {
condition = var.prefix != ""
error_message = "Prefix cannot be empty, please use null instead."
}
}
variable "project_id" {
description = "Project ID."
type = string
@@ -212,9 +202,3 @@ variable "region" {
description = "Dataproc region."
type = string
}
variable "service_account" {
description = "Service account to set on the Dataproc cluster."
type = string
default = null
}