From d86b8d565c2b586e060cf68090f27db5d056026f Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 12 Dec 2024 11:02:24 +0100 Subject: [PATCH] Refactor GKE cluster modules access configurations, add support for DNS endpoint (#2761) * stub * gke standard module and tests * blueprints * tfdoc * autopilot * blueprints * tfdoc * gke hub module examples * dataproc and gke fixture --- blueprints/apigee/hybrid-gke/gke.tf | 19 +- blueprints/gke/autopilot/cluster.tf | 15 +- blueprints/gke/binauthz/main.tf | 4 - .../multi-cluster-mesh-gke-fleet-api/gke.tf | 22 +- .../gke/patterns/autopilot-cluster/cluster.tf | 17 +- .../networking/shared-vpc-gke/README.md | 7 +- blueprints/networking/shared-vpc-gke/main.tf | 13 +- .../networking/shared-vpc-gke/variables.tf | 8 - .../secops/secops-gke-forwarder/main.tf | 13 +- fast/stages/3-gke-dev/README.md | 14 +- fast/stages/3-gke-dev/gke-clusters.tf | 2 +- fast/stages/3-gke-dev/variables.tf | 19 +- modules/dataproc/README.md | 10 +- modules/dataproc/variables.tf | 5 - modules/gke-cluster-autopilot/README.md | 99 ++++---- modules/gke-cluster-autopilot/main.tf | 47 ++-- modules/gke-cluster-autopilot/outputs.tf | 8 + modules/gke-cluster-autopilot/variables.tf | 52 +++-- modules/gke-cluster-standard/README.md | 215 ++++++++++++------ modules/gke-cluster-standard/main.tf | 48 ++-- modules/gke-cluster-standard/outputs.tf | 14 +- modules/gke-cluster-standard/variables.tf | 51 +++-- modules/gke-hub/README.md | 50 ++-- tests/fixtures/gke-cluster-standard.tf | 15 +- .../gke_cluster_autopilot/examples/basic.yaml | 100 +++++++- .../examples/access-private.yaml | 158 +++++++++++++ .../examples/access-public.yaml | 150 ++++++++++++ .../gke_cluster_standard/examples/basic.yaml | 29 --- .../examples/dataplane-v2.yaml | 126 +++++++++- .../examples/regional.yaml | 160 +++++++++++++ tests/modules/gke_hub/examples/full.yaml | 34 ++- 31 files changed, 1143 insertions(+), 381 deletions(-) create mode 100644 tests/modules/gke_cluster_standard/examples/access-private.yaml create mode 100644 tests/modules/gke_cluster_standard/examples/access-public.yaml delete mode 100644 tests/modules/gke_cluster_standard/examples/basic.yaml create mode 100644 tests/modules/gke_cluster_standard/examples/regional.yaml diff --git a/blueprints/apigee/hybrid-gke/gke.tf b/blueprints/apigee/hybrid-gke/gke.tf index 1baa7b4a4..05df5ebbf 100644 --- a/blueprints/apigee/hybrid-gke/gke.tf +++ b/blueprints/apigee/hybrid-gke/gke.tf @@ -19,18 +19,19 @@ module "cluster" { project_id = module.project.project_id name = "cluster" location = var.region + access_config = { + ip_access = { + authorized_ranges = ( + var.cluster_network_config.master_authorized_cidr_blocks + ) + } + } vpc_config = { - network = module.vpc.self_link - subnetwork = module.vpc.subnet_self_links["${var.region}/subnet-apigee"] - secondary_range_names = {} - master_authorized_ranges = var.cluster_network_config.master_authorized_cidr_blocks - master_ipv4_cidr_block = var.cluster_network_config.master_cidr_block + network = module.vpc.self_link + subnetwork = module.vpc.subnet_self_links["${var.region}/subnet-apigee"] + secondary_range_names = {} } max_pods_per_node = 32 - private_cluster_config = { - enable_private_endpoint = true - master_global_access = false - } enable_features = { workload_identity = true } diff --git a/blueprints/gke/autopilot/cluster.tf b/blueprints/gke/autopilot/cluster.tf index d18de5f4f..1e0cd0cd4 100644 --- a/blueprints/gke/autopilot/cluster.tf +++ b/blueprints/gke/autopilot/cluster.tf @@ -19,12 +19,17 @@ module "cluster" { project_id = module.project.project_id name = "cluster" location = var.region + access_config = { + ip_access = { + authorized_ranges = ( + var.cluster_network_config.master_authorized_cidr_blocks + ) + } + } vpc_config = { - network = module.vpc.self_link - subnetwork = module.vpc.subnet_self_links["${var.region}/subnet-cluster"] - secondary_range_names = {} - master_authorized_ranges = var.cluster_network_config.master_authorized_cidr_blocks - master_ipv4_cidr_block = var.cluster_network_config.master_cidr_block + network = module.vpc.self_link + subnetwork = module.vpc.subnet_self_links["${var.region}/subnet-cluster"] + secondary_range_names = {} } # enable_features = { # autopilot = true diff --git a/blueprints/gke/binauthz/main.tf b/blueprints/gke/binauthz/main.tf index 521c6351b..5ee0b59c0 100644 --- a/blueprints/gke/binauthz/main.tf +++ b/blueprints/gke/binauthz/main.tf @@ -92,10 +92,6 @@ module "cluster" { network = module.vpc.self_link subnetwork = module.vpc.subnet_self_links["${var.region}/subnet"] } - private_cluster_config = { - enable_private_endpoint = false - master_global_access = false - } deletion_protection = var.deletion_protection } diff --git a/blueprints/gke/multi-cluster-mesh-gke-fleet-api/gke.tf b/blueprints/gke/multi-cluster-mesh-gke-fleet-api/gke.tf index 3eef990f3..94b280afb 100644 --- a/blueprints/gke/multi-cluster-mesh-gke-fleet-api/gke.tf +++ b/blueprints/gke/multi-cluster-mesh-gke-fleet-api/gke.tf @@ -22,20 +22,20 @@ module "clusters" { project_id = module.fleet_project.project_id name = each.key location = var.region + access_config = { + ip_access = { + authorized_ranges = merge( + { mgmt : var.mgmt_subnet_cidr_block }, + { + for key, config in var.clusters_config : + "pods-${key}" => config.pods_cidr_block if key != each.key + } + ) + } + } vpc_config = { network = module.svpc.self_link subnetwork = module.svpc.subnet_self_links["${var.region}/subnet-${each.key}"] - master_authorized_ranges = merge({ - mgmt : var.mgmt_subnet_cidr_block - }, - { for key, config in var.clusters_config : - "pods-${key}" => config.pods_cidr_block if key != each.key - }) - master_ipv4_cidr_block = each.value.master_cidr_block - } - private_cluster_config = { - enable_private_endpoint = true - master_global_access = true } release_channel = "REGULAR" labels = { diff --git a/blueprints/gke/patterns/autopilot-cluster/cluster.tf b/blueprints/gke/patterns/autopilot-cluster/cluster.tf index 91f08c3f8..69285b2bc 100644 --- a/blueprints/gke/patterns/autopilot-cluster/cluster.tf +++ b/blueprints/gke/patterns/autopilot-cluster/cluster.tf @@ -76,16 +76,15 @@ module "cluster" { deletion_protection = var.cluster_create.deletion_protection name = var.cluster_name location = var.region - vpc_config = { - network = local.cluster_vpc.network - subnetwork = local.cluster_vpc.subnet - secondary_range_names = local.cluster_vpc.secondary_range_names - master_authorized_ranges = var.cluster_create.master_authorized_ranges - master_ipv4_cidr_block = var.cluster_create.master_ipv4_cidr_block + access_config = { + ip_access = { + authorized_ranges = var.cluster_create.master_authorized_ranges + } } - private_cluster_config = { - enable_private_endpoint = true - master_global_access = true + vpc_config = { + network = local.cluster_vpc.network + subnetwork = local.cluster_vpc.subnet + secondary_range_names = local.cluster_vpc.secondary_range_names } node_config = { service_account = module.cluster-service-account[0].email diff --git a/blueprints/networking/shared-vpc-gke/README.md b/blueprints/networking/shared-vpc-gke/README.md index f13677674..fa48b5b5d 100644 --- a/blueprints/networking/shared-vpc-gke/README.md +++ b/blueprints/networking/shared-vpc-gke/README.md @@ -48,7 +48,7 @@ There's a minor glitch that can surface running `terraform destroy`, where the s |---|---|:---:|:---:|:---:| | [billing_account_id](variables.tf#L15) | Billing account id used as default for new projects. | string | ✓ | | | [prefix](variables.tf#L69) | Prefix used for resource names. | string | ✓ | | -| [root_node](variables.tf#L101) | Hierarchy node where projects will be created, 'organizations/org_id' or 'folders/folder_id'. | string | ✓ | | +| [root_node](variables.tf#L93) | Hierarchy node where projects will be created, 'organizations/org_id' or 'folders/folder_id'. | string | ✓ | | | [cluster_create](variables.tf#L20) | Create GKE cluster and nodepool. | bool | | true | | [deletion_protection](variables.tf#L26) | Prevent Terraform from destroying data storage resources (storage buckets, GKE clusters, CloudSQL instances) in this blueprint. When this field is set in Terraform state, a terraform destroy or terraform apply that would delete data storage resources will fail. | bool | | false | | [ip_ranges](variables.tf#L33) | Subnet IP CIDR ranges. | map(string) | | {…} | @@ -56,9 +56,8 @@ There's a minor glitch that can surface running `terraform destroy`, where the s | [owners_gce](variables.tf#L51) | GCE project owners, in IAM format. | list(string) | | [] | | [owners_gke](variables.tf#L57) | GKE project owners, in IAM format. | list(string) | | [] | | [owners_host](variables.tf#L63) | Host project owners, in IAM format. | list(string) | | [] | -| [private_service_ranges](variables.tf#L78) | Private service IP CIDR ranges. | map(string) | | {…} | -| [project_services](variables.tf#L86) | Service APIs enabled by default in new projects. | list(string) | | […] | -| [region](variables.tf#L95) | Region used. | string | | "europe-west1" | +| [project_services](variables.tf#L78) | Service APIs enabled by default in new projects. | list(string) | | […] | +| [region](variables.tf#L87) | Region used. | string | | "europe-west1" | ## Outputs diff --git a/blueprints/networking/shared-vpc-gke/main.tf b/blueprints/networking/shared-vpc-gke/main.tf index f97f2bbcf..3497542da 100644 --- a/blueprints/networking/shared-vpc-gke/main.tf +++ b/blueprints/networking/shared-vpc-gke/main.tf @@ -207,19 +207,16 @@ module "cluster-1" { name = "cluster-1" project_id = module.project-svc-gke.project_id location = "${var.region}-b" + access_config = { + ip_access = { + authorized_ranges = { internal-vms = var.ip_ranges.gce } + } + } vpc_config = { network = module.vpc-shared.self_link subnetwork = module.vpc-shared.subnet_self_links["${var.region}/gke"] - master_authorized_ranges = { - internal-vms = var.ip_ranges.gce - } - master_ipv4_cidr_block = var.private_service_ranges.cluster-1 } max_pods_per_node = 32 - private_cluster_config = { - enable_private_endpoint = true - master_global_access = true - } labels = { environment = "test" } diff --git a/blueprints/networking/shared-vpc-gke/variables.tf b/blueprints/networking/shared-vpc-gke/variables.tf index 77dfa0249..07980148b 100644 --- a/blueprints/networking/shared-vpc-gke/variables.tf +++ b/blueprints/networking/shared-vpc-gke/variables.tf @@ -75,14 +75,6 @@ variable "prefix" { } } -variable "private_service_ranges" { - description = "Private service IP CIDR ranges." - type = map(string) - default = { - cluster-1 = "192.168.0.0/28" - } -} - variable "project_services" { description = "Service APIs enabled by default in new projects." type = list(string) diff --git a/blueprints/secops/secops-gke-forwarder/main.tf b/blueprints/secops/secops-gke-forwarder/main.tf index cbf256cb7..bc3c8e870 100644 --- a/blueprints/secops/secops-gke-forwarder/main.tf +++ b/blueprints/secops/secops-gke-forwarder/main.tf @@ -61,6 +61,11 @@ module "chronicle-forwarder" { name = var.chronicle_forwarder.cluster_name location = var.region deletion_protection = false + access_config = { + ip_access = { + authorized_ranges = var.chronicle_forwarder.master_authorized_ranges + } + } vpc_config = { network = var.network_config.network_self_link subnetwork = var.network_config.subnet_self_link @@ -68,12 +73,6 @@ module "chronicle-forwarder" { pods = "pods" services = "services" } - master_ipv4_cidr_block = var.network_config.ip_range_gke_master - master_authorized_ranges = var.chronicle_forwarder.master_authorized_ranges - } - private_cluster_config = { - enable_private_endpoint = true - master_global_access = true } enable_features = { gateway_api = true @@ -100,4 +99,4 @@ module "chronicle-forwarder-deployment" { source = "./secops-forwarder-deployment" depends_on = [module.chronicle-forwarder] tenants = var.tenants -} \ No newline at end of file +} diff --git a/fast/stages/3-gke-dev/README.md b/fast/stages/3-gke-dev/README.md index 2fb479da6..790fecd88 100644 --- a/fast/stages/3-gke-dev/README.md +++ b/fast/stages/3-gke-dev/README.md @@ -192,18 +192,18 @@ Clusters can then be configured for fleet registration and one of the config man | [billing_account](variables-fast.tf#L17) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap | | [environments](variables-fast.tf#L25) | Long environment names. | object({…}) | ✓ | | 1-resman | | [prefix](variables-fast.tf#L51) | Prefix used for resources that need unique names. Use a maximum of 9 chars for organizations, and 11 chars for tenants. | string | ✓ | | 0-bootstrap | -| [clusters](variables.tf#L17) | Clusters configuration. Refer to the gke-cluster module for type details. | map(object({…})) | | {} | | -| [deletion_protection](variables.tf#L88) | Prevent Terraform from destroying data resources. | bool | | false | | +| [clusters](variables.tf#L17) | Clusters configuration. Refer to the gke-cluster module for type details. | map(object({…})) | | {} | | +| [deletion_protection](variables.tf#L97) | Prevent Terraform from destroying data resources. | bool | | false | | | [fleet_config](variables-fleet.tf#L19) | Fleet configuration. | object({…}) | | null | | | [fleet_configmanagement_templates](variables-fleet.tf#L35) | Sets of fleet configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | | [folder_ids](variables-fast.tf#L35) | Folder name => id mappings. | map(string) | | {} | 1-resman | | [host_project_ids](variables-fast.tf#L43) | Shared VPC host project name => id mappings. | map(string) | | {} | 2-networking | -| [iam](variables.tf#L95) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | -| [iam_by_principals](variables.tf#L102) | Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid cycle errors. Merged internally with the `iam` variable. | map(list(string)) | | {} | | -| [nodepools](variables.tf#L109) | Nodepools configuration. Refer to the gke-nodepool module for type details. | map(map(object({…}))) | | {} | | -| [stage_config](variables.tf#L142) | FAST stage configuration used to find resource ids. Must match name defined for the stage in resource management. | object({…}) | | {…} | | +| [iam](variables.tf#L104) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | +| [iam_by_principals](variables.tf#L111) | Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid cycle errors. Merged internally with the `iam` variable. | map(list(string)) | | {} | | +| [nodepools](variables.tf#L118) | Nodepools configuration. Refer to the gke-nodepool module for type details. | map(map(object({…}))) | | {} | | +| [stage_config](variables.tf#L151) | FAST stage configuration used to find resource ids. Must match name defined for the stage in resource management. | object({…}) | | {…} | | | [subnet_self_links](variables-fast.tf#L61) | Subnet VPC name => { name => self link } mappings. | map(map(string)) | | {} | 2-networking | -| [vpc_config](variables.tf#L154) | VPC-level configuration for project and clusters. | object({…}) | | {…} | | +| [vpc_config](variables.tf#L163) | VPC-level configuration for project and clusters. | object({…}) | | {…} | | | [vpc_self_links](variables-fast.tf#L69) | Shared VPC name => self link mappings. | map(string) | | {} | 2-networking | ## Outputs diff --git a/fast/stages/3-gke-dev/gke-clusters.tf b/fast/stages/3-gke-dev/gke-clusters.tf index d88171167..b23368f61 100644 --- a/fast/stages/3-gke-dev/gke-clusters.tf +++ b/fast/stages/3-gke-dev/gke-clusters.tf @@ -41,6 +41,7 @@ module "gke-cluster" { for_each = var.clusters name = each.key project_id = module.gke-project-0.project_id + access_config = each.value.access_config cluster_autoscaling = each.value.cluster_autoscaling description = each.value.description enable_features = each.value.enable_features @@ -54,7 +55,6 @@ module "gke-cluster" { min_master_version = each.value.min_master_version monitoring_config = each.value.monitoring_config node_locations = each.value.node_locations - private_cluster_config = each.value.private_cluster_config release_channel = each.value.release_channel vpc_config = merge(each.value.vpc_config, { network = try( diff --git a/fast/stages/3-gke-dev/variables.tf b/fast/stages/3-gke-dev/variables.tf index 496bb954b..a7189ebe9 100644 --- a/fast/stages/3-gke-dev/variables.tf +++ b/fast/stages/3-gke-dev/variables.tf @@ -17,6 +17,18 @@ variable "clusters" { description = "Clusters configuration. Refer to the gke-cluster module for type details." type = map(object({ + access_config = optional(object({ + dns_access = optional(bool, true) + ip_access = optional(object({ + authorized_ranges = optional(map(string), {}) + disable_public_endpoint = optional(bool, true) + private_endpoint_config = optional(object({ + endpoint_subnetwork = optional(string) + global_access = optional(bool, true) + }), {}) + }), {}) + private_nodes = optional(bool, true) + }), {}) cluster_autoscaling = optional(any) description = optional(string) enable_addons = optional(any, { @@ -63,9 +75,8 @@ variable "clusters" { # Google Cloud Managed Service for Prometheus enable_managed_prometheus = optional(bool, true) }), {}) - node_locations = optional(list(string)) - private_cluster_config = optional(any) - release_channel = optional(string) + node_locations = optional(list(string)) + release_channel = optional(string) vpc_config = object({ subnetwork = string network = optional(string) @@ -77,8 +88,6 @@ variable "clusters" { pods = string services = string }), { pods = "pods", services = "services" }) - master_authorized_ranges = optional(map(string)) - master_ipv4_cidr_block = optional(string) }) })) default = {} diff --git a/modules/dataproc/README.md b/modules/dataproc/README.md index 5e11ac372..344fedc6f 100644 --- a/modules/dataproc/README.md +++ b/modules/dataproc/README.md @@ -301,15 +301,15 @@ module "processing-dp-cluster" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L194) | Cluster name. | string | ✓ | | -| [project_id](variables.tf#L199) | Project ID. | string | ✓ | | -| [region](variables.tf#L204) | Dataproc region. | string | ✓ | | -| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…}) | | {} | +| [name](variables.tf#L189) | Cluster name. | string | ✓ | | +| [project_id](variables.tf#L194) | Project ID. | string | ✓ | | +| [region](variables.tf#L199) | Dataproc region. | string | ✓ | | +| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…}) | | {} | | [iam](variables-iam.tf#L24) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_bindings](variables-iam.tf#L31) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | | [iam_bindings_additive](variables-iam.tf#L46) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | | [iam_by_principals](variables-iam.tf#L17) | Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid cycle errors. Merged internally with the `iam` variable. | map(list(string)) | | {} | -| [labels](variables.tf#L188) | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string) | | {} | +| [labels](variables.tf#L183) | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string) | | {} | ## Outputs diff --git a/modules/dataproc/variables.tf b/modules/dataproc/variables.tf index ef6004d89..3d39da00a 100644 --- a/modules/dataproc/variables.tf +++ b/modules/dataproc/variables.tf @@ -38,7 +38,6 @@ variable "dataproc_config" { node_group_affinity = optional(object({ node_group_uri = string })) - shielded_instance_config = optional(object({ enable_secure_boot = bool enable_vtpm = bool @@ -138,7 +137,6 @@ variable "dataproc_config" { dataproc_metastore_service = string })) })) - virtual_cluster_config = optional(object({ staging_bucket = optional(string) auxiliary_services_config = optional(object({ @@ -155,7 +153,6 @@ variable "dataproc_config" { component_version = map(string) properties = optional(map(string)) }) - gke_cluster_config = object({ gke_cluster_target = optional(string) node_pool_target = optional(object({ @@ -166,7 +163,6 @@ variable "dataproc_config" { min_node_count = optional(number) max_node_count = optional(number) })) - config = object({ machine_type = optional(string) preemptible = optional(bool) @@ -174,7 +170,6 @@ variable "dataproc_config" { min_cpu_platform = optional(string) spot = optional(bool) }) - locations = optional(list(string)) })) })) diff --git a/modules/gke-cluster-autopilot/README.md b/modules/gke-cluster-autopilot/README.md index 00f6842c8..e73a0b6c3 100644 --- a/modules/gke-cluster-autopilot/README.md +++ b/modules/gke-cluster-autopilot/README.md @@ -3,19 +3,18 @@ This module offers a way to create and manage Google Kubernetes Engine (GKE) [Autopilot clusters](https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-overview). With its sensible default settings based on best practices and authors' experience as Google Cloud practitioners, the module accommodates for many common use cases out-of-the-box, without having to rely on verbose configuration. -- [Examples](#examples) - - [GKE Autopilot cluster](#gke-autopilot-cluster) - - [Cloud DNS](#cloud-dns) - - [Logging configuration](#logging-configuration) - - [Monitoring configuration](#monitoring-configuration) - - [Backup for GKE](#backup-for-gke) +- [GKE Autopilot cluster](#gke-autopilot-cluster) +- [Cloud DNS](#cloud-dns) +- [Logging configuration](#logging-configuration) +- [Monitoring configuration](#monitoring-configuration) +- [Backup for GKE](#backup-for-gke) - [Variables](#variables) - [Outputs](#outputs) -## Examples +For an explanation of cluster access configurations, please refer to the [GKE cluster standard](../gke-cluster-standard/README.md) module. -### GKE Autopilot cluster +## GKE Autopilot cluster This example shows how to [create a GKE cluster in Autopilot mode](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-an-autopilot-cluster). @@ -25,6 +24,13 @@ module "cluster-1" { project_id = "myproject" name = "cluster-1" location = "europe-west1" + access_config = { + ip_access = { + authorized_ranges = { + internal-vms = "10.0.0.0/8" + } + } + } vpc_config = { network = var.vpc.self_link subnetwork = var.subnet.self_link @@ -32,14 +38,6 @@ module "cluster-1" { pods = "pods" services = "services" } - master_authorized_ranges = { - internal-vms = "10.0.0.0/8" - } - master_ipv4_cidr_block = "192.168.0.0/28" - } - private_cluster_config = { - enable_private_endpoint = true - master_global_access = false } labels = { environment = "dev" @@ -48,7 +46,7 @@ module "cluster-1" { # tftest modules=1 resources=1 inventory=basic.yaml ``` -### Cloud DNS +## Cloud DNS > [!WARNING] > [Cloud DNS is the only DNS provider for Autopilot clusters](https://cloud.google.com/kubernetes-engine/docs/concepts/service-discovery#cloud_dns) running version `1.25.9-gke.400` and later, and version `1.26.4-gke.500` and later. It is [pre-configured](https://cloud.google.com/kubernetes-engine/docs/resources/autopilot-standard-feature-comparison#feature-comparison) for those clusters. The following example *only* applies to Autopilot clusters running *earlier* versions. @@ -77,12 +75,12 @@ module "cluster-1" { # tftest modules=1 resources=1 inventory=dns.yaml ``` -### Logging configuration +## Logging configuration > [!NOTE] > System and workload logs collection is pre-configured for Autopilot clusters and cannot be disabled. -This example shows how to [collect logs for the Kubernetes control plane components](https://cloud.google.com/stackdriver/docs/solutions/gke/installing). The logs for these components are not collected by default. +This example shows how to [collect logs for the Kubernetes control plane components](https://cloud.google.com/stackdriver/docs/solutions/gke/installing). The logs for these components are not collected by default. ```hcl module "cluster-1" { @@ -104,7 +102,7 @@ module "cluster-1" { # tftest modules=1 resources=1 inventory=logging-config.yaml ``` -### Monitoring configuration +## Monitoring configuration > [!NOTE] > [System metrics](https://cloud.google.com/stackdriver/docs/solutions/gke/managing-metrics#enable-system-metrics) collection is pre-configured for Autopilot clusters and cannot be disabled. @@ -164,15 +162,15 @@ module "cluster-1" { The *control plane metrics* and *kube state metrics* collection can be configured in a single `monitoring_config` block. -### Backup for GKE +## Backup for GKE > [!NOTE] > Although Backup for GKE can be enabled as an add-on when configuring your GKE clusters, it is a separate service from GKE. [Backup for GKE](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/concepts/backup-for-gke) is a service for backing up and restoring workloads in GKE clusters. It has two components: -* A [Google Cloud API](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/reference/rest) that serves as the control plane for the service. -* A GKE add-on (the [Backup for GKE agent](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/concepts/backup-for-gke#agent_overview)) that must be enabled in each cluster for which you wish to perform backup and restore operations. +- A [Google Cloud API](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/reference/rest) that serves as the control plane for the service. +- A GKE add-on (the [Backup for GKE agent](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/concepts/backup-for-gke#agent_overview)) that must be enabled in each cluster for which you wish to perform backup and restore operations. Backup for GKE is supported in GKE Autopilot clusters with [some restrictions](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/concepts/about-autopilot). @@ -206,25 +204,25 @@ module "cluster-1" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [location](variables.tf#L118) | Autopilot clusters are always regional. | string | ✓ | | -| [name](variables.tf#L195) | Cluster name. | string | ✓ | | -| [project_id](variables.tf#L240) | Cluster project ID. | string | ✓ | | -| [vpc_config](variables.tf#L256) | VPC-level configuration. | object({…}) | ✓ | | -| [backup_configs](variables.tf#L17) | Configuration for Backup for GKE. | object({…}) | | {} | -| [deletion_protection](variables.tf#L38) | Whether or not to allow Terraform to destroy the cluster. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the cluster will fail. | bool | | true | -| [description](variables.tf#L45) | Cluster description. | string | | null | -| [enable_addons](variables.tf#L51) | Addons enabled in the cluster (true means enabled). | object({…}) | | {} | -| [enable_features](variables.tf#L65) | Enable cluster-level features. Certain features allow configuration. | object({…}) | | {} | -| [issue_client_certificate](variables.tf#L106) | Enable issuing client certificate. | bool | | false | -| [labels](variables.tf#L112) | Cluster resource labels. | map(string) | | null | -| [logging_config](variables.tf#L123) | Logging configuration. | object({…}) | | {} | -| [maintenance_config](variables.tf#L134) | Maintenance window configuration. | object({…}) | | {…} | -| [min_master_version](variables.tf#L157) | Minimum version of the master, defaults to the version of the most recent official release. | string | | null | -| [monitoring_config](variables.tf#L163) | Monitoring configuration. System metrics collection cannot be disabled. Control plane metrics are optional. Kube state metrics are optional. Google Cloud Managed Service for Prometheus is enabled by default. | object({…}) | | {} | -| [node_config](variables.tf#L200) | Configuration for nodes and nodepools. | object({…}) | | {} | -| [node_locations](variables.tf#L219) | Zones in which the cluster's nodes are located. | list(string) | | [] | -| [private_cluster_config](variables.tf#L226) | Private cluster configuration. | object({…}) | | null | -| [release_channel](variables.tf#L245) | Release channel for GKE upgrades. Clusters created in the Autopilot mode must use a release channel. Choose between \"RAPID\", \"REGULAR\", and \"STABLE\". | string | | "REGULAR" | +| [location](variables.tf#L143) | Autopilot clusters are always regional. | string | ✓ | | +| [name](variables.tf#L220) | Cluster name. | string | ✓ | | +| [project_id](variables.tf#L251) | Cluster project ID. | string | ✓ | | +| [vpc_config](variables.tf#L267) | VPC-level configuration. | object({…}) | ✓ | | +| [access_config](variables.tf#L17) | Control plane endpoint and nodes access configurations. | object({…}) | | {} | +| [backup_configs](variables.tf#L42) | Configuration for Backup for GKE. | object({…}) | | {} | +| [deletion_protection](variables.tf#L63) | Whether or not to allow Terraform to destroy the cluster. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the cluster will fail. | bool | | true | +| [description](variables.tf#L70) | Cluster description. | string | | null | +| [enable_addons](variables.tf#L76) | Addons enabled in the cluster (true means enabled). | object({…}) | | {} | +| [enable_features](variables.tf#L90) | Enable cluster-level features. Certain features allow configuration. | object({…}) | | {} | +| [issue_client_certificate](variables.tf#L131) | Enable issuing client certificate. | bool | | false | +| [labels](variables.tf#L137) | Cluster resource labels. | map(string) | | null | +| [logging_config](variables.tf#L148) | Logging configuration. | object({…}) | | {} | +| [maintenance_config](variables.tf#L159) | Maintenance window configuration. | object({…}) | | {…} | +| [min_master_version](variables.tf#L182) | Minimum version of the master, defaults to the version of the most recent official release. | string | | null | +| [monitoring_config](variables.tf#L188) | Monitoring configuration. System metrics collection cannot be disabled. Control plane metrics are optional. Kube state metrics are optional. Google Cloud Managed Service for Prometheus is enabled by default. | object({…}) | | {} | +| [node_config](variables.tf#L225) | Configuration for nodes and nodepools. | object({…}) | | {} | +| [node_locations](variables.tf#L244) | Zones in which the cluster's nodes are located. | list(string) | | [] | +| [release_channel](variables.tf#L256) | Release channel for GKE upgrades. Clusters created in the Autopilot mode must use a release channel. Choose between \"RAPID\", \"REGULAR\", and \"STABLE\". | string | | "REGULAR" | ## Outputs @@ -232,12 +230,13 @@ module "cluster-1" { |---|---|:---:| | [ca_certificate](outputs.tf#L17) | Public certificate of the cluster (base64-encoded). | ✓ | | [cluster](outputs.tf#L23) | Cluster resource. | ✓ | -| [endpoint](outputs.tf#L29) | Cluster endpoint. | | -| [id](outputs.tf#L34) | Fully qualified cluster ID. | | -| [location](outputs.tf#L39) | Cluster location. | | -| [master_version](outputs.tf#L44) | Master version. | | -| [name](outputs.tf#L49) | Cluster name. | | -| [notifications](outputs.tf#L54) | GKE Pub/Sub notifications topic. | | -| [self_link](outputs.tf#L59) | Cluster self link. | ✓ | -| [workload_identity_pool](outputs.tf#L65) | Workload identity pool. | | +| [dns_endpoint](outputs.tf#L29) | Control plane DNS endpoint. | | +| [endpoint](outputs.tf#L37) | Cluster endpoint. | | +| [id](outputs.tf#L42) | Fully qualified cluster ID. | | +| [location](outputs.tf#L47) | Cluster location. | | +| [master_version](outputs.tf#L52) | Master version. | | +| [name](outputs.tf#L57) | Cluster name. | | +| [notifications](outputs.tf#L62) | GKE Pub/Sub notifications topic. | | +| [self_link](outputs.tf#L67) | Cluster self link. | ✓ | +| [workload_identity_pool](outputs.tf#L73) | Workload identity pool. | | diff --git a/modules/gke-cluster-autopilot/main.tf b/modules/gke-cluster-autopilot/main.tf index f9c4a85f2..af93d684a 100644 --- a/modules/gke-cluster-autopilot/main.tf +++ b/modules/gke-cluster-autopilot/main.tf @@ -81,6 +81,14 @@ resource "google_container_cluster" "cluster" { service_account = var.node_config.service_account } } + dynamic "control_plane_endpoints_config" { + for_each = var.access_config.dns_access == true ? [""] : [] + content { + dns_endpoint_config { + allow_external_traffic = true + } + } + } dynamic "database_encryption" { for_each = var.enable_features.database_encryption != null ? [""] : [] content { @@ -200,10 +208,10 @@ resource "google_container_cluster" "cluster" { } } dynamic "master_authorized_networks_config" { - for_each = var.vpc_config.master_authorized_ranges != null ? [""] : [] + for_each = try(var.access_config.ip_access.authorized_ranges, null) != null ? [""] : [] content { dynamic "cidr_blocks" { - for_each = var.vpc_config.master_authorized_ranges + for_each = var.access_config.ip_access.authorized_ranges iterator = range content { cidr_block = range.value @@ -260,16 +268,21 @@ resource "google_container_cluster" "cluster" { } } dynamic "private_cluster_config" { - for_each = ( - var.private_cluster_config != null ? [""] : [] - ) + for_each = var.access_config.private_nodes == true ? [""] : [] content { - enable_private_nodes = true - enable_private_endpoint = var.private_cluster_config.enable_private_endpoint - private_endpoint_subnetwork = try(var.vpc_config.master_endpoint_subnetwork, null) - master_ipv4_cidr_block = try(var.vpc_config.master_ipv4_cidr_block, null) + enable_private_nodes = true + enable_private_endpoint = ( + var.access_config.ip_access.disable_public_endpoint + ) + private_endpoint_subnetwork = try( + var.access_config.ip_access.private_endpoint_config.endpoint_subnetwork, + null + ) master_global_access_config { - enabled = var.private_cluster_config.master_global_access + enabled = try( + var.access_config.ip_access.private_endpoint_config.global_access, + null + ) } } } @@ -364,20 +377,6 @@ resource "google_gke_backup_backup_plan" "backup_plan" { } } -resource "google_compute_network_peering_routes_config" "gke_master" { - count = ( - try(var.private_cluster_config.peering_config, null) != null ? 1 : 0 - ) - project = coalesce(var.private_cluster_config.peering_config.project_id, var.project_id) - peering = try( - google_container_cluster.cluster.private_cluster_config[0].peering_name, - null - ) - network = element(reverse(split("/", var.vpc_config.network)), 0) - import_custom_routes = var.private_cluster_config.peering_config.import_routes - export_custom_routes = var.private_cluster_config.peering_config.export_routes -} - resource "google_pubsub_topic" "notifications" { count = ( try(var.enable_features.upgrade_notifications, null) != null && diff --git a/modules/gke-cluster-autopilot/outputs.tf b/modules/gke-cluster-autopilot/outputs.tf index c5f524b15..81bdc2a0c 100644 --- a/modules/gke-cluster-autopilot/outputs.tf +++ b/modules/gke-cluster-autopilot/outputs.tf @@ -26,6 +26,14 @@ output "cluster" { value = google_container_cluster.cluster } +output "dns_endpoint" { + description = "Control plane DNS endpoint." + value = try( + google_container_cluster.cluster.control_plane_endpoints_config[0].dns_endpoint_config[0].endpoint, + null + ) +} + output "endpoint" { description = "Cluster endpoint." value = google_container_cluster.cluster.endpoint diff --git a/modules/gke-cluster-autopilot/variables.tf b/modules/gke-cluster-autopilot/variables.tf index 3bc87f283..53122c945 100644 --- a/modules/gke-cluster-autopilot/variables.tf +++ b/modules/gke-cluster-autopilot/variables.tf @@ -14,6 +14,31 @@ * limitations under the License. */ +variable "access_config" { + description = "Control plane endpoint and nodes access configurations." + type = object({ + dns_access = optional(bool, true) + ip_access = optional(object({ + authorized_ranges = optional(map(string), {}) + disable_public_endpoint = optional(bool, true) + private_endpoint_config = optional(object({ + endpoint_subnetwork = optional(string) + global_access = optional(bool, true) + }), {}) + }), {}) + private_nodes = optional(bool, true) + }) + nullable = false + default = {} + validation { + condition = ( + try(var.access_config.ip_access.disable_public_endpoint, null) != true || + var.access_config.private_nodes == true + ) + error_message = "Private endpoint can only be enabled with private nodes." + } +} + variable "backup_configs" { description = "Configuration for Backup for GKE." type = object({ @@ -223,20 +248,6 @@ variable "node_locations" { nullable = false } -variable "private_cluster_config" { - description = "Private cluster configuration." - type = object({ - enable_private_endpoint = optional(bool) - master_global_access = optional(bool) - peering_config = optional(object({ - export_routes = optional(bool) - import_routes = optional(bool) - project_id = optional(string) - })) - }) - default = null -} - variable "project_id" { description = "Cluster project ID." type = string @@ -256,11 +267,9 @@ variable "release_channel" { variable "vpc_config" { description = "VPC-level configuration." type = object({ - disable_default_snat = optional(bool) - network = string - subnetwork = string - master_ipv4_cidr_block = optional(string) - master_endpoint_subnetwork = optional(string) + disable_default_snat = optional(bool) + network = string + subnetwork = string secondary_range_blocks = optional(object({ pods = string services = string @@ -269,9 +278,8 @@ variable "vpc_config" { pods = optional(string) services = optional(string) })) - additional_ranges = optional(list(string)) - master_authorized_ranges = optional(map(string)) - stack_type = optional(string) + additional_ranges = optional(list(string)) + stack_type = optional(string) }) nullable = false } diff --git a/modules/gke-cluster-standard/README.md b/modules/gke-cluster-standard/README.md index 24e2fb721..c73a1555b 100644 --- a/modules/gke-cluster-standard/README.md +++ b/modules/gke-cluster-standard/README.md @@ -6,25 +6,31 @@ This module offers a way to create and manage Google Kubernetes Engine (GKE) [St > This module should be used together with the [`gke-nodepool`](../gke-nodepool/) module because the default node pool is deleted upon cluster creation by default. -- [Example](#example) - - [GKE Standard cluster](#gke-standard-cluster) - - [Enable Dataplane V2](#enable-dataplane-v2) - - [Managing GKE logs](#managing-gke-logs) - - [Monitoring configuration](#monitoring-configuration) - - [Disable GKE logs or metrics collection](#disable-gke-logs-or-metrics-collection) - - [Cloud DNS](#cloud-dns) - - [Backup for GKE](#backup-for-gke) - - [Automatic creation of new secondary ranges](#automatic-creation-of-new-secondary-ranges) - - [Node auto-provisioning with GPUs and TPUs](#node-auto-provisioning-with-gpus-and-tpus) +- [Cluster access configurations](#cluster-access-configurations) + - [Private cluster with DNS endpoint enabled](#private-cluster-with-dns-endpoint-enabled) + - [Public cluster](#public-cluster) +- [Regional cluster](#regional-cluster) +- [Enable Dataplane V2](#enable-dataplane-v2) +- [Managing GKE logs](#managing-gke-logs) +- [Monitoring configuration](#monitoring-configuration) +- [Disable GKE logs or metrics collection](#disable-gke-logs-or-metrics-collection) +- [Cloud DNS](#cloud-dns) +- [Backup for GKE](#backup-for-gke) +- [Automatic creation of new secondary ranges](#automatic-creation-of-new-secondary-ranges) +- [Node auto-provisioning with GPUs and TPUs](#node-auto-provisioning-with-gpus-and-tpus) - [Variables](#variables) - [Outputs](#outputs) -## Example +## Cluster access configurations -### GKE Standard cluster +The `access_config` variable can be used to configure access to the control plane, and nodes public access. The following examples illustrate different possible configurations. -This example shows how to [create a zonal GKE cluster in Standard mode](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-zonal-cluster). +### Private cluster with DNS endpoint enabled + +The default module configuration creates a cluster with private nodes, no public endpoint, and access via the DNS endpoint enabled. The default variable configuration is shown in comments. + +Master authorized ranges can be set via the `access_config.ip_access.authorized_ranges` attribute. ```hcl module "cluster-1" { @@ -32,6 +38,20 @@ module "cluster-1" { project_id = "myproject" name = "cluster-1" location = "europe-west1-b" + # access_config can be omitted if master authorized ranges are not needed + access_config = { + # dns_access = true + ip_access = { + authorized_ranges = { + internal-vms = "10.0.0.0/8" + } + # disable_public_endpoint = true + # private_endpoint_config = { + # global_access = true + # } + } + # private_nodes = true + } vpc_config = { network = var.vpc.self_link subnetwork = var.subnet.self_link @@ -39,24 +59,86 @@ module "cluster-1" { pods = "pods" services = "services" } - master_authorized_ranges = { - internal-vms = "10.0.0.0/8" - } - master_ipv4_cidr_block = "192.168.0.0/28" } max_pods_per_node = 32 - private_cluster_config = { - enable_private_endpoint = true - master_global_access = false - } labels = { environment = "dev" } } -# tftest modules=1 resources=1 inventory=basic.yaml +# tftest modules=1 resources=1 inventory=access-private.yaml ``` -### Enable Dataplane V2 +### Public cluster + +To configure a public cluster, turn off `access_config.ip_access.disable_public_endpoint`. Nodes can be left as private or made public if needed, like in the example below. DNS endpoint is turned off here as it's probably redundant for a public cluster. + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-standard" + project_id = "myproject" + name = "cluster-1" + location = "europe-west1-b" + access_config = { + dns_access = false + ip_access = { + authorized_ranges = { + "corporate proxy" = "8.8.8.8/32" + } + disable_public_endpoint = false + } + private_nodes = false + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_names = { + pods = "pods" + services = "services" + } + } + max_pods_per_node = 32 + labels = { + environment = "dev" + } +} +# tftest modules=1 resources=1 inventory=access-public.yaml +``` + +## Regional cluster + +Regional clusters are created by setting `location` to a GCP region and then configuring `node_locations`, as shown in the example below. + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-standard" + project_id = "myproject" + name = "cluster-1" + location = "europe-west1" + node_locations = ["europe-west1-b"] + access_config = { + ip_access = { + authorized_ranges = { + internal-vms = "10.0.0.0/8" + } + } + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_names = { + pods = "pods" + services = "services" + } + } + max_pods_per_node = 32 + labels = { + environment = "dev" + } +} +# tftest modules=1 resources=1 inventory=regional.yaml +``` + +## Enable Dataplane V2 This example shows how to [create a zonal GKE Cluster with Dataplane V2 enabled](https://cloud.google.com/kubernetes-engine/docs/how-to/dataplane-v2). @@ -70,14 +152,6 @@ module "cluster-1" { network = var.vpc.self_link subnetwork = var.subnet.self_link secondary_range_names = {} # use default names "pods" and "services" - master_authorized_ranges = { - internal-vms = "10.0.0.0/8" - } - master_ipv4_cidr_block = "192.168.0.0/28" - } - private_cluster_config = { - enable_private_endpoint = true - master_global_access = false } enable_features = { dataplane_v2 = true @@ -89,10 +163,10 @@ module "cluster-1" { environment = "dev" } } -# tftest modules=1 resources=1 inventory=dataplane-v2.yaml +# tftest modules=1 resources=1 ``` -### Managing GKE logs +## Managing GKE logs This example shows you how to [control which logs are sent from your GKE cluster to Cloud Logging](https://cloud.google.com/stackdriver/docs/solutions/gke/installing). @@ -119,7 +193,7 @@ module "cluster-1" { # tftest modules=1 resources=1 inventory=logging-config-enable-all.yaml ``` -### Monitoring configuration +## Monitoring configuration This example shows how to [configure collection of Kubernetes control plane metrics](https://cloud.google.com/stackdriver/docs/solutions/gke/managing-metrics#enable-control-plane-metrics). These metrics are optional and are not collected by default. @@ -173,7 +247,7 @@ module "cluster-1" { The *control plane metrics* and *kube state metrics* collection can be configured in a single `monitoring_config` block. -### Disable GKE logs or metrics collection +## Disable GKE logs or metrics collection > [!WARNING] > If you've disabled Cloud Logging or Cloud Monitoring, GKE customer support @@ -221,7 +295,7 @@ module "cluster-1" { # tftest modules=1 resources=1 inventory=monitoring-config-disable-all.yaml ``` -### Cloud DNS +## Cloud DNS This example shows how to [use Cloud DNS as a Kubernetes DNS provider](https://cloud.google.com/kubernetes-engine/docs/how-to/cloud-dns) for GKE Standard clusters. @@ -247,7 +321,7 @@ module "cluster-1" { # tftest modules=1 resources=1 inventory=dns.yaml ``` -### Backup for GKE +## Backup for GKE > [!NOTE] > Although Backup for GKE can be enabled as an add-on when configuring your GKE clusters, it is a separate service from GKE. @@ -286,7 +360,7 @@ module "cluster-1" { # tftest modules=1 resources=2 inventory=backup.yaml ``` -### Automatic creation of new secondary ranges +## Automatic creation of new secondary ranges You can use `var.vpc_config.secondary_range_blocks` to let GKE create new secondary ranges for the cluster. The example below reserves an available /14 block for pods and a /20 for services. @@ -308,7 +382,7 @@ module "cluster-1" { # tftest modules=1 resources=1 ``` -### Node auto-provisioning with GPUs and TPUs +## Node auto-provisioning with GPUs and TPUs You can use `var.cluster_autoscaling` block to configure node auto-provisioning for the GKE cluster. The example below configures limits for CPU, memory, GPUs and TPUs. @@ -353,41 +427,42 @@ module "cluster-1" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [location](variables.tf#L243) | Cluster zone or region. | string | ✓ | | -| [name](variables.tf#L355) | Cluster name. | string | ✓ | | -| [project_id](variables.tf#L402) | Cluster project id. | string | ✓ | | -| [vpc_config](variables.tf#L413) | VPC-level configuration. | object({…}) | ✓ | | -| [backup_configs](variables.tf#L18) | Configuration for Backup for GKE. | object({…}) | | {} | -| [cluster_autoscaling](variables.tf#L40) | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({…}) | | null | -| [default_nodepool](variables.tf#L119) | Enable default nodepool. | object({…}) | | {} | -| [deletion_protection](variables.tf#L137) | Whether or not to allow Terraform to destroy the cluster. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the cluster will fail. | bool | | true | -| [description](variables.tf#L144) | Cluster description. | string | | null | -| [enable_addons](variables.tf#L150) | Addons enabled in the cluster (true means enabled). | object({…}) | | {…} | -| [enable_features](variables.tf#L175) | Enable cluster-level features. Certain features allow configuration. | object({…}) | | {…} | -| [issue_client_certificate](variables.tf#L230) | Enable issuing client certificate. | bool | | false | -| [labels](variables.tf#L236) | Cluster resource labels. | map(string) | | {} | -| [logging_config](variables.tf#L248) | Logging configuration. | object({…}) | | {} | -| [maintenance_config](variables.tf#L269) | Maintenance window configuration. | object({…}) | | {…} | -| [max_pods_per_node](variables.tf#L292) | Maximum number of pods per node in this cluster. | number | | 110 | -| [min_master_version](variables.tf#L298) | Minimum version of the master, defaults to the version of the most recent official release. | string | | null | -| [monitoring_config](variables.tf#L304) | Monitoring configuration. Google Cloud Managed Service for Prometheus is enabled by default. | object({…}) | | {} | -| [node_config](variables.tf#L360) | Node-level configuration. | object({…}) | | {} | -| [node_locations](variables.tf#L381) | Zones in which the cluster's nodes are located. | list(string) | | [] | -| [private_cluster_config](variables.tf#L388) | Private cluster configuration. | object({…}) | | null | -| [release_channel](variables.tf#L407) | Release channel for GKE upgrades. | string | | null | +| [location](variables.tf#L267) | Cluster zone or region. | string | ✓ | | +| [name](variables.tf#L379) | Cluster name. | string | ✓ | | +| [project_id](variables.tf#L412) | Cluster project id. | string | ✓ | | +| [vpc_config](variables.tf#L423) | VPC-level configuration. | object({…}) | ✓ | | +| [access_config](variables.tf#L17) | Control plane endpoint and nodes access configurations. | object({…}) | | {} | +| [backup_configs](variables.tf#L42) | Configuration for Backup for GKE. | object({…}) | | {} | +| [cluster_autoscaling](variables.tf#L64) | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({…}) | | null | +| [default_nodepool](variables.tf#L143) | Enable default nodepool. | object({…}) | | {} | +| [deletion_protection](variables.tf#L161) | Whether or not to allow Terraform to destroy the cluster. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the cluster will fail. | bool | | true | +| [description](variables.tf#L168) | Cluster description. | string | | null | +| [enable_addons](variables.tf#L174) | Addons enabled in the cluster (true means enabled). | object({…}) | | {…} | +| [enable_features](variables.tf#L199) | Enable cluster-level features. Certain features allow configuration. | object({…}) | | {…} | +| [issue_client_certificate](variables.tf#L254) | Enable issuing client certificate. | bool | | false | +| [labels](variables.tf#L260) | Cluster resource labels. | map(string) | | {} | +| [logging_config](variables.tf#L272) | Logging configuration. | object({…}) | | {} | +| [maintenance_config](variables.tf#L293) | Maintenance window configuration. | object({…}) | | {…} | +| [max_pods_per_node](variables.tf#L316) | Maximum number of pods per node in this cluster. | number | | 110 | +| [min_master_version](variables.tf#L322) | Minimum version of the master, defaults to the version of the most recent official release. | string | | null | +| [monitoring_config](variables.tf#L328) | Monitoring configuration. Google Cloud Managed Service for Prometheus is enabled by default. | object({…}) | | {} | +| [node_config](variables.tf#L384) | Node-level configuration. | object({…}) | | {} | +| [node_locations](variables.tf#L405) | Zones in which the cluster's nodes are located. | list(string) | | [] | +| [release_channel](variables.tf#L417) | Release channel for GKE upgrades. | string | | null | ## Outputs | name | description | sensitive | |---|---|:---:| | [ca_certificate](outputs.tf#L17) | Public certificate of the cluster (base64-encoded). | ✓ | -| [cluster](outputs.tf#L23) | Cluster resource. | ✓ | -| [endpoint](outputs.tf#L29) | Cluster endpoint. | | -| [id](outputs.tf#L34) | FUlly qualified cluster id. | | -| [location](outputs.tf#L39) | Cluster location. | | -| [master_version](outputs.tf#L44) | Master version. | | -| [name](outputs.tf#L49) | Cluster name. | | -| [notifications](outputs.tf#L54) | GKE PubSub notifications topic. | | -| [self_link](outputs.tf#L59) | Cluster self link. | ✓ | -| [workload_identity_pool](outputs.tf#L65) | Workload identity pool. | | +| [cluster](outputs.tf#L25) | Cluster resource. | ✓ | +| [dns_endpoint](outputs.tf#L31) | Control plane DNS endpoint. | | +| [endpoint](outputs.tf#L39) | Cluster endpoint. | | +| [id](outputs.tf#L44) | FUlly qualified cluster id. | | +| [location](outputs.tf#L49) | Cluster location. | | +| [master_version](outputs.tf#L54) | Master version. | | +| [name](outputs.tf#L59) | Cluster name. | | +| [notifications](outputs.tf#L64) | GKE PubSub notifications topic. | | +| [self_link](outputs.tf#L69) | Cluster self link. | ✓ | +| [workload_identity_pool](outputs.tf#L75) | Workload identity pool. | | diff --git a/modules/gke-cluster-standard/main.tf b/modules/gke-cluster-standard/main.tf index 0ce5ab676..e0c1ece2d 100644 --- a/modules/gke-cluster-standard/main.tf +++ b/modules/gke-cluster-standard/main.tf @@ -249,6 +249,14 @@ resource "google_container_cluster" "cluster" { } } } + dynamic "control_plane_endpoints_config" { + for_each = var.access_config.dns_access == true ? [""] : [] + content { + dns_endpoint_config { + allow_external_traffic = true + } + } + } dynamic "database_encryption" { for_each = var.enable_features.database_encryption != null ? [""] : [] content { @@ -384,10 +392,10 @@ resource "google_container_cluster" "cluster" { } } dynamic "master_authorized_networks_config" { - for_each = var.vpc_config.master_authorized_ranges != null ? [""] : [] + for_each = try(var.access_config.ip_access.authorized_ranges, null) != null ? [""] : [] content { dynamic "cidr_blocks" { - for_each = var.vpc_config.master_authorized_ranges + for_each = var.access_config.ip_access.authorized_ranges iterator = range content { cidr_block = range.value @@ -464,16 +472,21 @@ resource "google_container_cluster" "cluster" { } } dynamic "private_cluster_config" { - for_each = ( - var.private_cluster_config != null ? [""] : [] - ) + for_each = var.access_config.private_nodes == true ? [""] : [] content { - enable_private_nodes = true - enable_private_endpoint = var.private_cluster_config.enable_private_endpoint - private_endpoint_subnetwork = try(var.vpc_config.master_endpoint_subnetwork, null) - master_ipv4_cidr_block = try(var.vpc_config.master_ipv4_cidr_block, null) + enable_private_nodes = true + enable_private_endpoint = ( + var.access_config.ip_access.disable_public_endpoint + ) + private_endpoint_subnetwork = try( + var.access_config.ip_access.private_endpoint_config.endpoint_subnetwork, + null + ) master_global_access_config { - enabled = var.private_cluster_config.master_global_access + enabled = try( + var.access_config.ip_access.private_endpoint_config.global_access, + null + ) } } } @@ -600,21 +613,6 @@ resource "google_gke_backup_backup_plan" "backup_plan" { } } - -resource "google_compute_network_peering_routes_config" "gke_master" { - count = ( - try(var.private_cluster_config.peering_config, null) != null ? 1 : 0 - ) - project = coalesce(var.private_cluster_config.peering_config.project_id, var.project_id) - peering = try( - google_container_cluster.cluster.private_cluster_config[0].peering_name, - null - ) - network = element(reverse(split("/", var.vpc_config.network)), 0) - import_custom_routes = var.private_cluster_config.peering_config.import_routes - export_custom_routes = var.private_cluster_config.peering_config.export_routes -} - resource "google_pubsub_topic" "notifications" { count = ( try(var.enable_features.upgrade_notifications, null) != null && diff --git a/modules/gke-cluster-standard/outputs.tf b/modules/gke-cluster-standard/outputs.tf index aa6b8052d..bc6c8e9a4 100644 --- a/modules/gke-cluster-standard/outputs.tf +++ b/modules/gke-cluster-standard/outputs.tf @@ -16,8 +16,10 @@ output "ca_certificate" { description = "Public certificate of the cluster (base64-encoded)." - value = google_container_cluster.cluster.master_auth[0].cluster_ca_certificate - sensitive = true + value = ( + google_container_cluster.cluster.master_auth[0].cluster_ca_certificate + ) + sensitive = true } output "cluster" { @@ -26,6 +28,14 @@ output "cluster" { value = google_container_cluster.cluster } +output "dns_endpoint" { + description = "Control plane DNS endpoint." + value = try( + google_container_cluster.cluster.control_plane_endpoints_config[0].dns_endpoint_config[0].endpoint, + null + ) +} + output "endpoint" { description = "Cluster endpoint." value = google_container_cluster.cluster.endpoint diff --git a/modules/gke-cluster-standard/variables.tf b/modules/gke-cluster-standard/variables.tf index 0b02e0ef5..20d3764d2 100644 --- a/modules/gke-cluster-standard/variables.tf +++ b/modules/gke-cluster-standard/variables.tf @@ -14,6 +14,30 @@ * limitations under the License. */ +variable "access_config" { + description = "Control plane endpoint and nodes access configurations." + type = object({ + dns_access = optional(bool, true) + ip_access = optional(object({ + authorized_ranges = optional(map(string), {}) + disable_public_endpoint = optional(bool, true) + private_endpoint_config = optional(object({ + endpoint_subnetwork = optional(string) + global_access = optional(bool, true) + }), {}) + }), {}) + private_nodes = optional(bool, true) + }) + nullable = false + default = {} + validation { + condition = ( + try(var.access_config.ip_access.disable_public_endpoint, null) != true || + var.access_config.private_nodes == true + ) + error_message = "Private endpoint can only be enabled with private nodes." + } +} variable "backup_configs" { description = "Configuration for Backup for GKE." @@ -385,20 +409,6 @@ variable "node_locations" { nullable = false } -variable "private_cluster_config" { - description = "Private cluster configuration." - type = object({ - enable_private_endpoint = optional(bool) - master_global_access = optional(bool) - peering_config = optional(object({ - export_routes = optional(bool) - import_routes = optional(bool) - project_id = optional(string) - })) - }) - default = null -} - variable "project_id" { description = "Cluster project id." type = string @@ -413,11 +423,9 @@ variable "release_channel" { variable "vpc_config" { description = "VPC-level configuration." type = object({ - disable_default_snat = optional(bool) - network = string - subnetwork = string - master_ipv4_cidr_block = optional(string) - master_endpoint_subnetwork = optional(string) + disable_default_snat = optional(bool) + network = string + subnetwork = string secondary_range_blocks = optional(object({ pods = string services = string @@ -426,9 +434,8 @@ variable "vpc_config" { pods = optional(string) services = optional(string) })) - additional_ranges = optional(list(string)) - master_authorized_ranges = optional(map(string)) - stack_type = optional(string) + additional_ranges = optional(list(string)) + stack_type = optional(string) }) nullable = false } diff --git a/modules/gke-hub/README.md b/modules/gke-hub/README.md index d701448ad..c5c7205f4 100644 --- a/modules/gke-hub/README.md +++ b/modules/gke-hub/README.md @@ -50,22 +50,21 @@ module "cluster_1" { project_id = module.project.project_id name = "cluster-1" location = "europe-west1" + access_config = { + ip_access = { + authorized_ranges = { + rfc1918_10_8 = "10.0.0.0/8" + } + } + } vpc_config = { network = module.vpc.self_link subnetwork = module.vpc.subnet_self_links["europe-west1/cluster-1"] - master_authorized_ranges = { - rfc1918_10_8 = "10.0.0.0/8" - } - master_ipv4_cidr_block = "192.168.0.0/28" } enable_features = { dataplane_v2 = true workload_identity = true } - private_cluster_config = { - enable_private_endpoint = true - master_global_access = false - } } module "hub" { @@ -196,20 +195,18 @@ module "cluster_1" { project_id = module.project.project_id name = "cluster-1" location = "europe-west1" + access_config = { + ip_access = { + authorized_ranges = { + mgmt = "10.0.0.0/28" + pods-cluster-1 = "10.3.0.0/16" + } + } + } vpc_config = { network = module.vpc.self_link subnetwork = module.vpc.subnet_self_links["europe-west1/subnet-cluster-1"] - master_authorized_ranges = { - mgmt = "10.0.0.0/28" - pods-cluster-1 = "10.3.0.0/16" - } - master_ipv4_cidr_block = "192.168.1.0/28" } - private_cluster_config = { - enable_private_endpoint = false - master_global_access = true - } - release_channel = "REGULAR" labels = { mesh_id = "proj-${module.project.number}" @@ -237,18 +234,17 @@ module "cluster_2" { project_id = module.project.project_id name = "cluster-2" location = "europe-west4" + access_config = { + ip_access = { + authorized_ranges = { + mgmt = "10.0.0.0/28" + pods-cluster-1 = "10.3.0.0/16" + } + } + } vpc_config = { network = module.vpc.self_link subnetwork = module.vpc.subnet_self_links["europe-west4/subnet-cluster-2"] - master_authorized_ranges = { - mgmt = "10.0.0.0/28" - pods-cluster-1 = "10.3.0.0/16" - } - master_ipv4_cidr_block = "192.168.2.0/28" - } - private_cluster_config = { - enable_private_endpoint = false - master_global_access = true } release_channel = "REGULAR" labels = { diff --git a/tests/fixtures/gke-cluster-standard.tf b/tests/fixtures/gke-cluster-standard.tf index b0c8af667..a1bfd3c61 100644 --- a/tests/fixtures/gke-cluster-standard.tf +++ b/tests/fixtures/gke-cluster-standard.tf @@ -18,18 +18,17 @@ module "gke-cluster-standard" { name = "cluster" location = "${var.region}-b" deletion_protection = false + access_config = { + ip_access = { + authorized_ranges = { + internal-vms = "10.0.0.0/8" + } + } + } vpc_config = { network = var.vpc.self_link subnetwork = var.subnet.self_link secondary_range_names = {} # use default names "pods" and "services" - master_authorized_ranges = { - internal-vms = "10.0.0.0/8" - } - master_ipv4_cidr_block = "192.168.0.0/28" - } - private_cluster_config = { - enable_private_endpoint = true - master_global_access = false } enable_features = { dataplane_v2 = true diff --git a/tests/modules/gke_cluster_autopilot/examples/basic.yaml b/tests/modules/gke_cluster_autopilot/examples/basic.yaml index decbb042c..c30eda99a 100644 --- a/tests/modules/gke_cluster_autopilot/examples/basic.yaml +++ b/tests/modules/gke_cluster_autopilot/examples/basic.yaml @@ -14,15 +14,105 @@ values: module.cluster-1.google_container_cluster.cluster: + addons_config: + - cloudrun_config: + - disabled: true + load_balancer_type: null + config_connector_config: + - enabled: false + gke_backup_agent_config: + - enabled: false + horizontal_pod_autoscaling: + - disabled: false + http_load_balancing: + - disabled: false + kalm_config: + - enabled: false + allow_net_admin: false + binary_authorization: [] + cluster_autoscaling: + - auto_provisioning_defaults: + - boot_disk_kms_key: null + disk_size: null + disk_type: null + image_type: null + min_cpu_platform: null + service_account: default + shielded_instance_config: [] + autoscaling_profile: null + resource_limits: [] + control_plane_endpoints_config: + - dns_endpoint_config: + - allow_external_traffic: true + deletion_protection: true + description: null + dns_config: [] + effective_labels: + environment: dev + goog-terraform-provisioned: 'true' + enable_autopilot: true + enable_cilium_clusterwide_network_policy: false + enable_fqdn_network_policy: false + enable_intranode_visibility: true + enable_k8s_beta_apis: [] + enable_kubernetes_alpha: false + enable_l4_ilb_subsetting: false + enable_legacy_abac: false + enable_multi_networking: false + enable_shielded_nodes: true + enable_tpu: false + fleet: [] + initial_node_count: 1 + ip_allocation_policy: + - additional_pod_ranges_config: [] + cluster_secondary_range_name: pods + services_secondary_range_name: services + stack_type: IPV4 + location: europe-west1 + logging_config: + - enable_components: + - SYSTEM_COMPONENTS + - WORKLOADS + maintenance_policy: + - daily_maintenance_window: + - start_time: 03:00 + maintenance_exclusion: [] + recurring_window: [] + master_auth: + - client_certificate_config: + - issue_client_certificate: false + master_authorized_networks_config: + - cidr_blocks: + - cidr_block: 10.0.0.0/8 + display_name: internal-vms + min_master_version: null + monitoring_config: + - enable_components: + - SYSTEM_COMPONENTS + managed_prometheus: + - enabled: true + name: cluster-1 + network: projects/xxx/global/networks/aaa + network_policy: [] + networking_mode: VPC_NATIVE + pod_security_policy_config: [] private_cluster_config: - enable_private_endpoint: true enable_private_nodes: true master_global_access_config: - - enabled: false - master_ipv4_cidr_block: 192.168.0.0/28 + - enabled: true private_endpoint_subnetwork: null + project: myproject + release_channel: + - channel: REGULAR + remove_default_node_pool: null resource_labels: environment: dev - -counts: - google_container_cluster: 1 + resource_usage_export_config: [] + secret_manager_config: [] + subnetwork: subnet_self_link + terraform_labels: + environment: dev + goog-terraform-provisioned: 'true' + timeouts: null + user_managed_keys_config: [] diff --git a/tests/modules/gke_cluster_standard/examples/access-private.yaml b/tests/modules/gke_cluster_standard/examples/access-private.yaml new file mode 100644 index 000000000..cee77a2c2 --- /dev/null +++ b/tests/modules/gke_cluster_standard/examples/access-private.yaml @@ -0,0 +1,158 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.cluster-1.google_container_cluster.cluster: + addons_config: + - cloudrun_config: + - disabled: true + load_balancer_type: null + config_connector_config: + - enabled: false + dns_cache_config: + - enabled: false + gce_persistent_disk_csi_driver_config: + - enabled: false + gcp_filestore_csi_driver_config: + - enabled: false + gcs_fuse_csi_driver_config: + - enabled: false + gke_backup_agent_config: + - enabled: false + horizontal_pod_autoscaling: + - disabled: false + http_load_balancing: + - disabled: false + istio_config: + - auth: null + disabled: true + kalm_config: + - enabled: false + network_policy_config: + - disabled: true + stateful_ha_config: + - enabled: false + allow_net_admin: null + binary_authorization: [] + control_plane_endpoints_config: + - dns_endpoint_config: + - allow_external_traffic: true + default_max_pods_per_node: 32 + deletion_protection: true + description: null + dns_config: [] + effective_labels: + environment: dev + goog-terraform-provisioned: 'true' + enable_autopilot: null + enable_cilium_clusterwide_network_policy: false + enable_fqdn_network_policy: false + enable_intranode_visibility: false + enable_k8s_beta_apis: [] + enable_kubernetes_alpha: false + enable_l4_ilb_subsetting: false + enable_legacy_abac: false + enable_multi_networking: false + enable_shielded_nodes: false + enable_tpu: false + fleet: [] + initial_node_count: 1 + ip_allocation_policy: + - additional_pod_ranges_config: [] + cluster_secondary_range_name: pods + services_secondary_range_name: services + stack_type: IPV4 + location: europe-west1-b + logging_config: + - enable_components: + - SYSTEM_COMPONENTS + maintenance_policy: + - daily_maintenance_window: + - start_time: 03:00 + maintenance_exclusion: [] + recurring_window: [] + master_auth: + - client_certificate_config: + - issue_client_certificate: false + master_authorized_networks_config: + - cidr_blocks: + - cidr_block: 10.0.0.0/8 + display_name: internal-vms + min_master_version: null + monitoring_config: + - enable_components: + - SYSTEM_COMPONENTS + managed_prometheus: + - enabled: true + name: cluster-1 + network: projects/xxx/global/networks/aaa + network_policy: [] + node_config: + - advanced_machine_features: [] + boot_disk_kms_key: null + containerd_config: [] + enable_confidential_storage: null + ephemeral_storage_config: [] + ephemeral_storage_local_ssd_config: [] + fast_socket: [] + gvnic: [] + host_maintenance_policy: [] + linux_node_config: [] + local_nvme_ssd_block_config: [] + local_ssd_encryption_mode: null + node_group: null + preemptible: false + reservation_affinity: [] + resource_labels: null + resource_manager_tags: null + sandbox_config: [] + secondary_boot_disks: [] + sole_tenant_config: [] + spot: false + storage_pools: null + tags: null + taint: [] + node_pool_defaults: + - node_config_defaults: + - containerd_config: [] + gcfs_config: + - enabled: false + pod_security_policy_config: [] + private_cluster_config: + - enable_private_endpoint: true + enable_private_nodes: true + master_global_access_config: + - enabled: true + private_endpoint_subnetwork: null + project: myproject + remove_default_node_pool: true + resource_labels: + environment: dev + resource_usage_export_config: [] + secret_manager_config: [] + subnetwork: subnet_self_link + terraform_labels: + environment: dev + goog-terraform-provisioned: 'true' + timeouts: null + user_managed_keys_config: [] + workload_identity_config: + - workload_pool: myproject.svc.id.goog + +counts: + google_container_cluster: 1 + modules: 1 + resources: 1 + +outputs: {} diff --git a/tests/modules/gke_cluster_standard/examples/access-public.yaml b/tests/modules/gke_cluster_standard/examples/access-public.yaml new file mode 100644 index 000000000..f8fbf8aa3 --- /dev/null +++ b/tests/modules/gke_cluster_standard/examples/access-public.yaml @@ -0,0 +1,150 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.cluster-1.google_container_cluster.cluster: + addons_config: + - cloudrun_config: + - disabled: true + load_balancer_type: null + config_connector_config: + - enabled: false + dns_cache_config: + - enabled: false + gce_persistent_disk_csi_driver_config: + - enabled: false + gcp_filestore_csi_driver_config: + - enabled: false + gcs_fuse_csi_driver_config: + - enabled: false + gke_backup_agent_config: + - enabled: false + horizontal_pod_autoscaling: + - disabled: false + http_load_balancing: + - disabled: false + istio_config: + - auth: null + disabled: true + kalm_config: + - enabled: false + network_policy_config: + - disabled: true + stateful_ha_config: + - enabled: false + allow_net_admin: null + binary_authorization: [] + default_max_pods_per_node: 32 + deletion_protection: true + description: null + dns_config: [] + effective_labels: + environment: dev + goog-terraform-provisioned: 'true' + enable_autopilot: null + enable_cilium_clusterwide_network_policy: false + enable_fqdn_network_policy: false + enable_intranode_visibility: false + enable_k8s_beta_apis: [] + enable_kubernetes_alpha: false + enable_l4_ilb_subsetting: false + enable_legacy_abac: false + enable_multi_networking: false + enable_shielded_nodes: false + enable_tpu: false + fleet: [] + initial_node_count: 1 + ip_allocation_policy: + - additional_pod_ranges_config: [] + cluster_secondary_range_name: pods + services_secondary_range_name: services + stack_type: IPV4 + location: europe-west1-b + logging_config: + - enable_components: + - SYSTEM_COMPONENTS + maintenance_policy: + - daily_maintenance_window: + - start_time: 03:00 + maintenance_exclusion: [] + recurring_window: [] + master_auth: + - client_certificate_config: + - issue_client_certificate: false + master_authorized_networks_config: + - cidr_blocks: + - cidr_block: 8.8.8.8/32 + display_name: corporate proxy + min_master_version: null + monitoring_config: + - enable_components: + - SYSTEM_COMPONENTS + managed_prometheus: + - enabled: true + name: cluster-1 + network: projects/xxx/global/networks/aaa + network_policy: [] + node_config: + - advanced_machine_features: [] + boot_disk_kms_key: null + containerd_config: [] + enable_confidential_storage: null + ephemeral_storage_config: [] + ephemeral_storage_local_ssd_config: [] + fast_socket: [] + gvnic: [] + host_maintenance_policy: [] + linux_node_config: [] + local_nvme_ssd_block_config: [] + local_ssd_encryption_mode: null + node_group: null + preemptible: false + reservation_affinity: [] + resource_labels: null + resource_manager_tags: null + sandbox_config: [] + secondary_boot_disks: [] + sole_tenant_config: [] + spot: false + storage_pools: null + tags: null + taint: [] + node_pool_defaults: + - node_config_defaults: + - containerd_config: [] + gcfs_config: + - enabled: false + pod_security_policy_config: [] + private_cluster_config: [] + project: myproject + remove_default_node_pool: true + resource_labels: + environment: dev + resource_usage_export_config: [] + secret_manager_config: [] + subnetwork: subnet_self_link + terraform_labels: + environment: dev + goog-terraform-provisioned: 'true' + timeouts: null + user_managed_keys_config: [] + workload_identity_config: + - workload_pool: myproject.svc.id.goog + +counts: + google_container_cluster: 1 + modules: 1 + resources: 1 + +outputs: {} diff --git a/tests/modules/gke_cluster_standard/examples/basic.yaml b/tests/modules/gke_cluster_standard/examples/basic.yaml deleted file mode 100644 index 300495bc1..000000000 --- a/tests/modules/gke_cluster_standard/examples/basic.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -values: - module.cluster-1.google_container_cluster.cluster: - private_cluster_config: - - enable_private_endpoint: true - enable_private_nodes: true - master_global_access_config: - - enabled: false - master_ipv4_cidr_block: 192.168.0.0/28 - private_endpoint_subnetwork: null - remove_default_node_pool: true - resource_labels: - environment: dev - -counts: - google_container_cluster: 1 diff --git a/tests/modules/gke_cluster_standard/examples/dataplane-v2.yaml b/tests/modules/gke_cluster_standard/examples/dataplane-v2.yaml index 2c3328faa..47bd0e8b2 100644 --- a/tests/modules/gke_cluster_standard/examples/dataplane-v2.yaml +++ b/tests/modules/gke_cluster_standard/examples/dataplane-v2.yaml @@ -14,27 +14,143 @@ values: module.cluster-1.google_container_cluster.cluster: + addons_config: + - cloudrun_config: + - disabled: true + load_balancer_type: null + config_connector_config: + - enabled: false + dns_cache_config: + - enabled: false + gce_persistent_disk_csi_driver_config: + - enabled: false + gcp_filestore_csi_driver_config: + - enabled: false + gcs_fuse_csi_driver_config: + - enabled: false + gke_backup_agent_config: + - enabled: false + horizontal_pod_autoscaling: + - disabled: false + http_load_balancing: + - disabled: false + istio_config: + - auth: null + disabled: true + kalm_config: + - enabled: false + network_policy_config: + - disabled: true + stateful_ha_config: + - enabled: false + allow_net_admin: null + binary_authorization: [] + control_plane_endpoints_config: + - dns_endpoint_config: + - allow_external_traffic: true datapath_provider: ADVANCED_DATAPATH + default_max_pods_per_node: 110 + deletion_protection: true + description: null + dns_config: [] + effective_labels: + environment: dev + goog-terraform-provisioned: 'true' + enable_autopilot: null + enable_cilium_clusterwide_network_policy: false + enable_fqdn_network_policy: true + enable_intranode_visibility: false + enable_k8s_beta_apis: [] + enable_kubernetes_alpha: false + enable_l4_ilb_subsetting: false + enable_legacy_abac: false + enable_multi_networking: false + enable_shielded_nodes: false + enable_tpu: false + fleet: [] + initial_node_count: 1 + ip_allocation_policy: + - additional_pod_ranges_config: [] + stack_type: IPV4 + location: europe-west1-b + logging_config: + - enable_components: + - SYSTEM_COMPONENTS + maintenance_policy: + - daily_maintenance_window: + - start_time: 03:00 + maintenance_exclusion: [] + recurring_window: [] + master_auth: + - client_certificate_config: + - issue_client_certificate: false master_authorized_networks_config: - - cidr_blocks: - - cidr_block: 10.0.0.0/8 - display_name: internal-vms + - cidr_blocks: [] min_master_version: null + monitoring_config: + - enable_components: + - SYSTEM_COMPONENTS + managed_prometheus: + - enabled: true name: cluster-dataplane-v2 network: projects/xxx/global/networks/aaa + network_policy: [] + node_config: + - advanced_machine_features: [] + boot_disk_kms_key: null + containerd_config: [] + enable_confidential_storage: null + ephemeral_storage_config: [] + ephemeral_storage_local_ssd_config: [] + fast_socket: [] + gvnic: [] + host_maintenance_policy: [] + linux_node_config: [] + local_nvme_ssd_block_config: [] + local_ssd_encryption_mode: null + node_group: null + preemptible: false + reservation_affinity: [] + resource_labels: null + resource_manager_tags: null + sandbox_config: [] + secondary_boot_disks: [] + sole_tenant_config: [] + spot: false + storage_pools: null + tags: null + taint: [] + node_pool_defaults: + - node_config_defaults: + - containerd_config: [] + gcfs_config: + - enabled: false + pod_security_policy_config: [] private_cluster_config: - enable_private_endpoint: true enable_private_nodes: true master_global_access_config: - - enabled: false - master_ipv4_cidr_block: 192.168.0.0/28 + - enabled: true private_endpoint_subnetwork: null project: myproject remove_default_node_pool: true resource_labels: environment: dev + resource_usage_export_config: [] + secret_manager_config: + - enabled: true + subnetwork: subnet_self_link + terraform_labels: + environment: dev + goog-terraform-provisioned: 'true' + timeouts: null + user_managed_keys_config: [] workload_identity_config: - workload_pool: myproject.svc.id.goog counts: google_container_cluster: 1 + modules: 1 + resources: 1 + +outputs: {} diff --git a/tests/modules/gke_cluster_standard/examples/regional.yaml b/tests/modules/gke_cluster_standard/examples/regional.yaml new file mode 100644 index 000000000..978f1dd49 --- /dev/null +++ b/tests/modules/gke_cluster_standard/examples/regional.yaml @@ -0,0 +1,160 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.cluster-1.google_container_cluster.cluster: + addons_config: + - cloudrun_config: + - disabled: true + load_balancer_type: null + config_connector_config: + - enabled: false + dns_cache_config: + - enabled: false + gce_persistent_disk_csi_driver_config: + - enabled: false + gcp_filestore_csi_driver_config: + - enabled: false + gcs_fuse_csi_driver_config: + - enabled: false + gke_backup_agent_config: + - enabled: false + horizontal_pod_autoscaling: + - disabled: false + http_load_balancing: + - disabled: false + istio_config: + - auth: null + disabled: true + kalm_config: + - enabled: false + network_policy_config: + - disabled: true + stateful_ha_config: + - enabled: false + allow_net_admin: null + binary_authorization: [] + control_plane_endpoints_config: + - dns_endpoint_config: + - allow_external_traffic: true + default_max_pods_per_node: 32 + deletion_protection: true + description: null + dns_config: [] + effective_labels: + environment: dev + goog-terraform-provisioned: 'true' + enable_autopilot: null + enable_cilium_clusterwide_network_policy: false + enable_fqdn_network_policy: false + enable_intranode_visibility: false + enable_k8s_beta_apis: [] + enable_kubernetes_alpha: false + enable_l4_ilb_subsetting: false + enable_legacy_abac: false + enable_multi_networking: false + enable_shielded_nodes: false + enable_tpu: false + fleet: [] + initial_node_count: 1 + ip_allocation_policy: + - additional_pod_ranges_config: [] + cluster_secondary_range_name: pods + services_secondary_range_name: services + stack_type: IPV4 + location: europe-west1 + logging_config: + - enable_components: + - SYSTEM_COMPONENTS + maintenance_policy: + - daily_maintenance_window: + - start_time: 03:00 + maintenance_exclusion: [] + recurring_window: [] + master_auth: + - client_certificate_config: + - issue_client_certificate: false + master_authorized_networks_config: + - cidr_blocks: + - cidr_block: 10.0.0.0/8 + display_name: internal-vms + min_master_version: null + monitoring_config: + - enable_components: + - SYSTEM_COMPONENTS + managed_prometheus: + - enabled: true + name: cluster-1 + network: projects/xxx/global/networks/aaa + network_policy: [] + node_config: + - advanced_machine_features: [] + boot_disk_kms_key: null + containerd_config: [] + enable_confidential_storage: null + ephemeral_storage_config: [] + ephemeral_storage_local_ssd_config: [] + fast_socket: [] + gvnic: [] + host_maintenance_policy: [] + linux_node_config: [] + local_nvme_ssd_block_config: [] + local_ssd_encryption_mode: null + node_group: null + preemptible: false + reservation_affinity: [] + resource_labels: null + resource_manager_tags: null + sandbox_config: [] + secondary_boot_disks: [] + sole_tenant_config: [] + spot: false + storage_pools: null + tags: null + taint: [] + node_locations: + - europe-west1-b + node_pool_defaults: + - node_config_defaults: + - containerd_config: [] + gcfs_config: + - enabled: false + pod_security_policy_config: [] + private_cluster_config: + - enable_private_endpoint: true + enable_private_nodes: true + master_global_access_config: + - enabled: true + private_endpoint_subnetwork: null + project: myproject + remove_default_node_pool: true + resource_labels: + environment: dev + resource_usage_export_config: [] + secret_manager_config: [] + subnetwork: subnet_self_link + terraform_labels: + environment: dev + goog-terraform-provisioned: 'true' + timeouts: null + user_managed_keys_config: [] + workload_identity_config: + - workload_pool: myproject.svc.id.goog + +counts: + google_container_cluster: 1 + modules: 1 + resources: 1 + +outputs: {} diff --git a/tests/modules/gke_hub/examples/full.yaml b/tests/modules/gke_hub/examples/full.yaml index fa2e0192b..3662546b6 100644 --- a/tests/modules/gke_hub/examples/full.yaml +++ b/tests/modules/gke_hub/examples/full.yaml @@ -45,11 +45,16 @@ values: - enabled: false allow_net_admin: null binary_authorization: [] + control_plane_endpoints_config: + - dns_endpoint_config: + - allow_external_traffic: true datapath_provider: ADVANCED_DATAPATH default_max_pods_per_node: 110 deletion_protection: true description: null dns_config: [] + effective_labels: + goog-terraform-provisioned: 'true' enable_autopilot: null enable_cilium_clusterwide_network_policy: false enable_fqdn_network_policy: false @@ -99,6 +104,7 @@ values: host_maintenance_policy: [] linux_node_config: [] local_nvme_ssd_block_config: [] + local_ssd_encryption_mode: null node_group: null preemptible: false reservation_affinity: [] @@ -108,6 +114,7 @@ values: secondary_boot_disks: [] sole_tenant_config: [] spot: false + storage_pools: null tags: null taint: [] node_pool_defaults: @@ -120,29 +127,36 @@ values: - enable_private_endpoint: true enable_private_nodes: true master_global_access_config: - - enabled: false - master_ipv4_cidr_block: 192.168.0.0/28 + - enabled: true private_endpoint_subnetwork: null project: gkehub-test remove_default_node_pool: true resource_labels: null resource_usage_export_config: [] secret_manager_config: [] + terraform_labels: + goog-terraform-provisioned: 'true' timeouts: null + user_managed_keys_config: [] workload_identity_config: - workload_pool: gkehub-test.svc.id.goog module.hub.google_gke_hub_feature.default["configmanagement"]: + effective_labels: + goog-terraform-provisioned: 'true' fleet_default_member_config: [] labels: null location: global name: configmanagement project: gkehub-test spec: [] + terraform_labels: + goog-terraform-provisioned: 'true' timeouts: null module.hub.google_gke_hub_feature_membership.default["cluster-1"]: configmanagement: - config_sync: - - git: + - enabled: true + git: - gcp_service_account_email: null https_proxy: null policy_dir: configsync @@ -154,6 +168,7 @@ values: metrics_gcp_service_account_email: null oci: [] source_format: hierarchy + stop_syncing: null hierarchy_controller: - enable_hierarchical_resource_quota: true enable_pod_tree_labels: true @@ -178,6 +193,8 @@ values: module.hub.google_gke_hub_membership.default["cluster-1"]: authority: [] description: null + effective_labels: + goog-terraform-provisioned: 'true' endpoint: - gke_cluster: - {} @@ -185,16 +202,23 @@ values: location: global membership_id: cluster-1 project: gkehub-test + terraform_labels: + goog-terraform-provisioned: 'true' timeouts: null module.project.google_project.project[0]: auto_create_network: false - deletion_policy: 'DELETE' billing_account: 123456-123456-123456 + deletion_policy: DELETE + effective_labels: + goog-terraform-provisioned: 'true' folder_id: '12345' labels: null name: gkehub-test org_id: null project_id: gkehub-test + tags: null + terraform_labels: + goog-terraform-provisioned: 'true' timeouts: null module.project.google_project_iam_member.service_agents["anthosconfigmanagement"]: condition: [] @@ -328,7 +352,9 @@ values: private_ip_google_access: true project: gkehub-test region: europe-west1 + reserved_internal_range: null role: null + send_secondary_ip_range_if_empty: true timeouts: null counts: