hunfabric/modules/managed-kafka

Managed Kafka Module

This module allows simplified creation and management of Google Cloud Managed Kafka clusters, including topics, Kafka Connect clusters, and connectors.

TOC

• TOC
• Simple Cluster Example
• Cluster with Topics
• Cluster with Kafka Connect
• Variables
• Outputs

Simple Cluster Example

This example creates a basic Managed Kafka cluster.

module "kafka-cluster" {
  source     = "./fabric/modules/managed-kafka"
  project_id = var.project_id
  location   = var.regions.primary
  cluster_id = "my-kafka-cluster"

  capacity_config = {
    vcpu_count   = 3
    memory_bytes = 3221225472 # 3 GiB
  }

  subnets = [
    var.subnets.primary.id
  ]

  labels = {
    environment = "development"
  }
}
# tftest modules=1 resources=1 inventory=simple.yaml

Cluster with Topics

This example creates a Managed Kafka cluster along with predefined topics.

module "kafka-cluster-with-topics" {
  source     = "./fabric/modules/managed-kafka"
  project_id = var.project_id
  location   = "europe-west1"
  cluster_id = "my-kafka-cluster-topics"

  capacity_config = {
    vcpu_count   = 6
    memory_bytes = 6442450944 # 6 GiB
  }

  subnets = [var.subnets.primary.id]

  topics = {
    topic-a = {
      partition_count    = 3
      replication_factor = 3
      configs = {
        "cleanup.policy" = "delete"
      }
    }
    topic-b = {
      partition_count    = 6
      replication_factor = 3
    }
  }
}
# tftest modules=1 resources=3 inventory=topics.yaml

Cluster with Kafka Connect

This example demonstrates creating a Kafka cluster, a Kafka Connect cluster, and a connector. Note that Connect resources require the google-beta provider.

module "gcs" {
  source     = "./fabric/modules/gcs"
  project_id = var.project_id
  location   = var.region
  name       = "gmk-sink"
  prefix     = var.prefix
}

module "vpc" {
  source     = "./fabric/modules/net-vpc"
  project_id = var.project_id
  name       = "vpc"
  subnets = [
    {
      ip_cidr_range = "10.0.0.0/20"
      name          = "subnet1"
      region        = var.region
    },
    {
      ip_cidr_range = "10.0.16.0/20"
      name          = "subnet2"
      region        = var.region
    },
    {
      ip_cidr_range = "10.0.32.0/20"
      name          = "subnet3"
      region        = var.region
    },
  ]
}

module "kafka-cluster-with-connect" {
  source     = "./fabric/modules/managed-kafka"
  project_id = var.project_id
  location   = var.region
  cluster_id = "my-kafka-cluster-connect"

  capacity_config = {
    vcpu_count   = 3
    memory_bytes = 3221225472 # 3 GiB
  }

  subnets = [
    module.vpc.subnet_ids["${var.region}/subnet1"]
  ]

  connect_clusters = {
    my-connect-cluster = {
      vcpu_count     = 3
      memory_bytes   = 3221225472 # 3 GiB
      primary_subnet = module.vpc.subnet_ids["${var.region}/subnet1"]
      additional_subnets = [
        module.vpc.subnet_ids["${var.region}/subnet2"],
        module.vpc.subnet_ids["${var.region}/subnet3"]
      ]
    }
  }

  connect_connectors = {
    my-gcs-connector = {
      connect_cluster = "my-connect-cluster"
      configs = {
        "connector.class"                = "io.aiven.kafka.connect.gcs.GcsSinkConnector"
        "file.name.prefix"               = ""
        "format.output.type"             = "json"
        "gcs.bucket.name"                = module.gcs.name
        "gcs.credentials.default"        = "true"
        "key.converter"                  = "org.apache.kafka.connect.storage.StringConverter"
        "tasks.max"                      = "3"
        "topics"                         = "topic1"
        "value.converter"                = "org.apache.kafka.connect.json.JsonConverter"
        "value.converter.schemas.enable" = "false"
      }
      task_restart_policy = {
        minimum_backoff = "60s"
        maximum_backoff = "300s"
      }
    }
  }
}
# tftest modules=3 resources=11 inventory=connect.yaml

Variables

name	description	type	required	default
capacity_config	Capacity configuration for the Kafka cluster.	object({…})	✓
cluster_id	The ID of the Kafka cluster.	string	✓
location	The GCP region for the Kafka cluster.	string	✓
project_id	The ID of the project where the Kafka cluster will be created.	string	✓
subnets	List of VPC subnets for the Kafka cluster network configuration.	list(string)	✓
connect_clusters	Map of Kafka Connect cluster configurations to create.	map(object({…}))		{}
connect_connectors	Map of Kafka Connect Connectors to create.	map(object({…}))		{}
kms_key	Customer-managed encryption key (CMEK) used for the Kafka cluster.	string		null
labels	Labels to apply to the Kafka cluster.	map(string)		null
rebalance_mode	Rebalancing mode for the Kafka cluster.	string		null
topics	Map of Kafka topics to create within the cluster.	map(object({…}))		{}

Outputs

name	description	sensitive
connect_cluster_ids	Map of Kafka Connect cluster IDs.
connect_connectors	Map of Kafka Connect Connector IDs.
id	The ID of the Managed Kafka cluster.
topic_ids	Map of Kafka topic IDs.