diff --git a/fast/project-templates/managed-kafka/README.md b/fast/project-templates/managed-kafka/README.md new file mode 100644 index 000000000..dec061fbc --- /dev/null +++ b/fast/project-templates/managed-kafka/README.md @@ -0,0 +1,79 @@ +# Managed Kafka Cluster with Topics + +This setup allows creating and configuring a managed Kafka cluster using [Google Cloud Managed Service for Apache Kafka](https://cloud.google.com/managed-service-for-apache-kafka), with configurable topics, networking, and labels. It is designed to be FAST-compliant and integrates seamlessly with existing Google Cloud infrastructure. + +## Prerequisites + +The [`project.yaml`](./project.yaml) file describes the project-level configuration needed in terms of API activation and IAM bindings. + +If you are deploying this inside a FAST-enabled organization, the file can be lightly edited to match your configuration and then used directly in the [project factory](../../stages/2-project-factory/). + +For non-FAST setups, use the `project.yaml` file as a reference to configure your project manually: + +- Enable the APIs listed under `services`. +- Grant the permissions listed under `iam` to the principal running Terraform, either a service account or a human user. + +## Variable Configuration + +Configuration is primarily done via the `kafka_config` and `topics` variables. Key considerations: + +- The Kafka cluster is deployed in the specified region and subnetworks. +- Topics can be configured with partition count, replication factor, and additional settings. +- Labels can be added for resource organization and management. + +Bringing up a cluster and the associated topics from scratch will require approximately 20–30 minutes. + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [kafka_config](variables.tf#L23) | Configuration for the Kafka cluster. | object({…}) | ✓ | | +| [network_project_ids](variables.tf#L46) | List of project IDs where the subnets are located. | list(string) | ✓ | | +| [project_id](variables.tf#L18) | The ID of the Google Cloud project where the Kafka cluster will be deployed. | string | ✓ | | +| [labels](variables.tf#L40) | Additional labels for the Kafka cluster. | map(string) | | {} | +| [topics](variables.tf#L51) | The list of topics to create in the Kafka cluster. | list(object({…})) | | [] | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [kafka_cluster_id](outputs.tf#L17) | The ID of the Kafka cluster. | | +| [kafka_labels](outputs.tf#L27) | Labels applied to the Kafka cluster. | | +| [kafka_region](outputs.tf#L22) | The region where the Kafka cluster is deployed. | | +| [project_number](outputs.tf#L32) | The project number of the Kafka cluster. | | + +## Test + +```hcl +module "test" { + source = "./fabric/fast/project-templates/managed-kafka" + kafka_config = { + cluster_id = "test-cluster" + region = "us-central1" + vcpu_count = 4 + memory_bytes = 21474836480 + subnetworks = [ + "projects/test-project/regions/us-central1/subnetworks/test-subnetwork" + ] + rebalance_mode = "AUTO_REBALANCE_ON_SCALE_UP" + } + project_id = "my-managed-kafka-project" + network_project_ids = [ + "test-network-project" + ] + topics = [ + { + topic_id = "test-topic-1" + partition_count = 3 + replication_factor = 2 + configs = { + "cleanup.policy" = "compact" + } + } + ] + labels = { + environment = "test" + } +} +# tftest skip +``` diff --git a/fast/project-templates/managed-kafka/main.tf b/fast/project-templates/managed-kafka/main.tf new file mode 100644 index 000000000..74d6555e0 --- /dev/null +++ b/fast/project-templates/managed-kafka/main.tf @@ -0,0 +1,88 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_project" "service_project" { + project_id = var.project_id +} + +resource "google_project_service" "managed_kafka_api" { + project = var.project_id + service = "managedkafka.googleapis.com" + disable_on_destroy = false +} + + +resource "google_project_iam_member" "managed_kafka_service_agent" { + for_each = toset(var.network_project_ids) + + project = each.value + role = "roles/managedkafka.serviceAgent" + member = "serviceAccount:service-${data.google_project.service_project.number}@gcp-sa-managedkafka.iam.gserviceaccount.com" +} + +resource "time_sleep" "wait_for_api_and_iam" { + depends_on = [ + google_project_service.managed_kafka_api, + google_project_iam_member.managed_kafka_service_agent + ] + create_duration = "60s" +} +/****************************************** + Resources configuration + *****************************************/ + + +resource "google_managed_kafka_cluster" "kafka_cluster" { + depends_on = [time_sleep.wait_for_api_and_iam] + project = var.project_id + cluster_id = var.kafka_config.cluster_id + location = var.kafka_config.region + + capacity_config { + vcpu_count = var.kafka_config.vcpu_count + memory_bytes = var.kafka_config.memory_bytes + } + + gcp_config { + access_config { + dynamic "network_configs" { + for_each = var.kafka_config.subnetworks + content { + subnet = network_configs.value + } + } + } + } + + rebalance_config { + mode = var.kafka_config.rebalance_mode + } + + labels = var.labels +} + +resource "google_managed_kafka_topic" "topics" { + for_each = tomap( + var.topics != [] ? { for topic in var.topics : topic.topic_id => topic } : {} + ) + topic_id = each.value.topic_id + cluster = google_managed_kafka_cluster.kafka_cluster.cluster_id + location = var.kafka_config.region + project = var.project_id + partition_count = each.value.partition_count + replication_factor = each.value.replication_factor + configs = each.value.configs +} \ No newline at end of file diff --git a/fast/project-templates/managed-kafka/outputs.tf b/fast/project-templates/managed-kafka/outputs.tf new file mode 100644 index 000000000..dfde27c46 --- /dev/null +++ b/fast/project-templates/managed-kafka/outputs.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +# Output variable definitions +output "kafka_cluster_id" { + description = "The ID of the Kafka cluster" + value = google_managed_kafka_cluster.kafka_cluster.cluster_id +} + +output "kafka_region" { + description = "The region where the Kafka cluster is deployed" + value = google_managed_kafka_cluster.kafka_cluster.location +} + +output "kafka_labels" { + description = "Labels applied to the Kafka cluster" + value = google_managed_kafka_cluster.kafka_cluster.labels +} + +output "project_number" { + value = data.google_project.service_project.number +} \ No newline at end of file diff --git a/fast/project-templates/managed-kafka/project.yaml b/fast/project-templates/managed-kafka/project.yaml new file mode 100644 index 000000000..d0a88bf73 --- /dev/null +++ b/fast/project-templates/managed-kafka/project.yaml @@ -0,0 +1,49 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# FAST-compliant project definition for the Managed Kafka cluster + +parent: shared +name: prod-shared-managed-kafka-0 +services: + - compute.googleapis.com + - logging.googleapis.com + - monitoring.googleapis.com + - kafka.googleapis.com + - dns.googleapis.com +# If automation resources are not used, grant these roles to the principal +# that will be used to apply this Terraform setup +iam: + roles/compute.admin: + - automation/rw + roles/servicedirectory.admin: + - automation/rw + roles/managedkafka.client: + - automation/rw +automation: + project: foo-prod-shared-iac-0 + service_accounts: + rw: + description: Read/write automation service account for Managed Kafka. + bucket: + description: Terraform state bucket for Managed Kafka. + iam: + roles/storage.objectAdmin: + - automation/rw + roles/storage.objectViewer: + - automation/rw +# Edit or comment shared VPC service host +shared_vpc_service_config: + host_project: dev-spoke-0 + network_users: + - automation/rw diff --git a/fast/project-templates/managed-kafka/variables.tf b/fast/project-templates/managed-kafka/variables.tf new file mode 100644 index 000000000..560f8db87 --- /dev/null +++ b/fast/project-templates/managed-kafka/variables.tf @@ -0,0 +1,60 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# Input variable definitions +variable "project_id" { + type = string + description = "The ID of the Google Cloud project where the Kafka cluster will be deployed" +} + +variable "kafka_config" { + type = object({ + cluster_id = string + region = string + vcpu_count = optional(number, 4) # Default to 4 vCPUs + memory_bytes = optional(number, 21474836480) # Default to 20 GB (in bytes) + subnetworks = list(string) + rebalance_mode = optional(string, "NO_REBALANCE") + }) + description = "Configuration for the Kafka cluster" + + validation { + condition = contains(["MODE_UNSPECIFIED", "NO_REBALANCE", "AUTO_REBALANCE_ON_SCALE_UP"], var.kafka_config.rebalance_mode) + error_message = "The rebalance_mode must be one of: MODE_UNSPECIFIED, NO_REBALANCE, AUTO_REBALANCE_ON_SCALE_UP." + } +} + +variable "labels" { + type = map(string) + default = {} + description = "Additional labels for the Kafka cluster" +} + +variable "network_project_ids" { + type = list(string) + description = "List of project IDs where the subnets are located" +} + +variable "topics" { + type = list(object({ + topic_id = string + partition_count = optional(number, 3) + replication_factor = optional(number, 1) + configs = optional(map(string), {}) + })) + description = "The list of topics to create in the Kafka cluster." + default = [] +} \ No newline at end of file diff --git a/fast/project-templates/managed-kafka/versions.tf b/fast/project-templates/managed-kafka/versions.tf new file mode 100644 index 000000000..5f1699eb7 --- /dev/null +++ b/fast/project-templates/managed-kafka/versions.tf @@ -0,0 +1,35 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fabric release: v38.1.0 + +terraform { + required_version = ">= 1.10.2" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.28.0, < 7.0.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.28.0, < 7.0.0" # tftest + } + } + provider_meta "google" { + module_name = "google-pso-tool/cloud-foundation-fabric/modules/gcs:v38.1.0-tf" + } + provider_meta "google-beta" { + module_name = "google-pso-tool/cloud-foundation-fabric/modules/gcs:v38.1.0-tf" + } +}