Add cluster replicas to Bigtable module.

This adds the possiblity to define several clusters in a Bigtable instance,
which enables replication in Bigtable.

Some of the configurations options are moved inside a `map` that defines the
properties of each cluster.

These changes alter the interface of the module, so any previous code using this
module will have to adapt to the new options interface.

modules/bigtable-instance/README.md

@@ -5,7 +5,6 @@ This module allows managing a single BigTable instance, including access configu
 ## TODO
 
 - [ ] support bigtable_app_profile
-- [ ] support cluster replicas
 - [ ] support IAM for tables
 
 ## Examples
@@ -18,8 +17,11 @@ module "bigtable-instance" {
   source     = "./fabric/modules/bigtable-instance"
   project_id = "my-project"
   name       = "instance"
-  cluster_id = "instance"
-  zone       = "europe-west1-b"
+  clusters = {
+    my-cluster = {
+      zone = "europe-west1-b"
+    }
+  }
   tables = {
     test1 = {},
     test2 = {
@@ -41,8 +43,11 @@ module "bigtable-instance" {
   source     = "./fabric/modules/bigtable-instance"
   project_id = "my-project"
   name       = "instance"
-  cluster_id = "instance"
-  zone       = "europe-west1-b"
+  clusters = {
+    my-cluster = {
+      zone = "europe-west1-b"
+    }
+  }
   tables = {
     test1 = {},
     test2 = {
@@ -63,6 +68,29 @@ module "bigtable-instance" {
 # tftest modules=1 resources=4
 ```
 
+### Instance with replication enabled
+
+```hcl
+
+module "bigtable-instance" {
+  source     = "./fabric/modules/bigtable-instance"
+  project_id = "my-project"
+  name       = "instance"
+  clusters = {
+    first-cluster = {
+      zone = "europe-west1-b"
+    }
+    second-cluster = {
+      zone = "europe-southwest1-a"
+    }
+    third-cluster = {
+      zone = "us-central1-b"
+    }
+  }
+}
+# tftest modules=1 resources=1
+```
+
 ### Instance with garbage collection policy
 
 ```hcl
@@ -71,8 +99,11 @@ module "bigtable-instance" {
   source     = "./fabric/modules/bigtable-instance"
   project_id = "my-project"
   name       = "instance"
-  cluster_id = "instance"
-  zone       = "europe-west1-b"
+  clusters = {
+    my-cluster = {
+      zone = "europe-west1-b"
+    }
+  }
   tables = {
     test1 = {
       column_families = {
@@ -102,8 +133,11 @@ module "bigtable-instance" {
   source     = "./fabric/modules/bigtable-instance"
   project_id = "my-project"
   name       = "instance"
-  cluster_id = "instance"
-  zone       = "europe-west1-b"
+  clusters = {
+    my-cluster = {
+      zone = "europe-west1-b"
+    }
+  }
   default_gc_policy = {
     deletion_policy = "ABANDON"
     max_age         = "18h"
@@ -131,9 +165,12 @@ module "bigtable-instance" {
   source     = "./fabric/modules/bigtable-instance"
   project_id = "my-project"
   name       = "instance"
-  cluster_id = "instance"
-  zone       = "europe-west1-b"
-  num_nodes  = 5
+  clusters = {
+    my-cluster = {
+      zone      = "europe-west1-b"
+      num_nodes = 5
+    }
+  }
 }
 # tftest modules=1 resources=1
 ```
@@ -148,13 +185,18 @@ module "bigtable-instance" {
   source     = "./fabric/modules/bigtable-instance"
   project_id = "my-project"
   name       = "instance"
-  cluster_id = "instance"
-  zone       = "europe-southwest1-b"
-  autoscaling_config = {
-    min_nodes  = 3
-    max_nodes  = 7
-    cpu_target = 70
+  clusters = {
+    my-cluster = {
+      zone = "europe-southwest1-b"
+      autoscaling = {
+        min_nodes  = 3
+        max_nodes  = 7
+        cpu_target = 70
+      }
+    }
   }
+
+
 }
 # tftest modules=1 resources=1
 ```
@@ -164,17 +206,20 @@ module "bigtable-instance" {
 ```hcl
 
 module "bigtable-instance" {
-  source       = "./fabric/modules/bigtable-instance"
-  project_id   = "my-project"
-  name         = "instance"
-  cluster_id   = "instance"
-  zone         = "europe-southwest1-a"
-  storage_type = "SSD"
-  autoscaling_config = {
-    min_nodes      = 3
-    max_nodes      = 7
-    cpu_target     = 70
-    storage_target = 4096
+  source     = "./fabric/modules/bigtable-instance"
+  project_id = "my-project"
+  name       = "instance"
+  clusters = {
+    my-cluster = {
+      zone         = "europe-southwest1-a"
+      storage_type = "SSD"
+      autoscaling = {
+        min_nodes      = 3
+        max_nodes      = 7
+        cpu_target     = 70
+        storage_target = 4096
+      }
+    }
   }
 }
 # tftest modules=1 resources=1
@@ -185,19 +230,16 @@ module "bigtable-instance" {
 
 | name | description | type | required | default |
 |---|---|:---:|:---:|:---:|
-| [name](variables.tf#L68) | The name of the Cloud Bigtable instance. | <code>string</code> | ✓ |  |
-| [project_id](variables.tf#L79) | Id of the project where datasets will be created. | <code>string</code> | ✓ |  |
-| [zone](variables.tf#L109) | The zone to create the Cloud Bigtable cluster in. | <code>string</code> | ✓ |  |
-| [autoscaling_config](variables.tf#L17) | Settings for autoscaling of the instance. If you set this variable, the variable num_nodes is ignored. | <code title="object&#40;&#123;&#10;  min_nodes      &#61; number&#10;  max_nodes      &#61; number&#10;  cpu_target     &#61; number&#10;  storage_target &#61; optional&#40;number, null&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
-| [cluster_id](variables.tf#L28) | The ID of the Cloud Bigtable cluster. | <code>string</code> |  | <code>&#34;europe-west1&#34;</code> |
-| [default_gc_policy](variables.tf#L34) | Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families. | <code title="object&#40;&#123;&#10;  deletion_policy &#61; optional&#40;string&#41;&#10;  gc_rules        &#61; optional&#40;string&#41;&#10;  mode            &#61; optional&#40;string&#41;&#10;  max_age         &#61; optional&#40;string&#41;&#10;  max_version     &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
-| [deletion_protection](variables.tf#L46) | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | <code></code> |  | <code>true</code> |
-| [display_name](variables.tf#L51) | The human-readable display name of the Bigtable instance. | <code></code> |  | <code>null</code> |
-| [iam](variables.tf#L56) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
-| [instance_type](variables.tf#L62) | (deprecated) The instance type to create. One of 'DEVELOPMENT' or 'PRODUCTION'. | <code>string</code> |  | <code>null</code> |
-| [num_nodes](variables.tf#L73) | The number of nodes in your Cloud Bigtable cluster. This value is ignored if you are using autoscaling. | <code>number</code> |  | <code>1</code> |
-| [storage_type](variables.tf#L84) | The storage type to use. | <code>string</code> |  | <code>&#34;SSD&#34;</code> |
-| [tables](variables.tf#L90) | Tables to be created in the BigTable instance. | <code title="map&#40;object&#40;&#123;&#10;  split_keys &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;  column_families &#61; optional&#40;map&#40;object&#40;&#10;    &#123;&#10;      gc_policy &#61; optional&#40;object&#40;&#123;&#10;        deletion_policy &#61; optional&#40;string&#41;&#10;        gc_rules        &#61; optional&#40;string&#41;&#10;        mode            &#61; optional&#40;string&#41;&#10;        max_age         &#61; optional&#40;string&#41;&#10;        max_version     &#61; optional&#40;string&#41;&#10;      &#125;&#41;, null&#41;&#10;  &#125;&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [clusters](variables.tf#L17) | Clusters to be created in the BigTable instance. Set more than one cluster to enable replication. If you set autoscaling, num_nodes will be ignored. | <code title="map&#40;object&#40;&#123;&#10;  zone         &#61; optional&#40;string&#41;&#10;  storage_type &#61; optional&#40;string&#41;&#10;  num_nodes    &#61; optional&#40;number&#41;&#10;  autoscaling &#61; optional&#40;object&#40;&#123;&#10;    min_nodes      &#61; number&#10;    max_nodes      &#61; number&#10;    cpu_target     &#61; number&#10;    storage_target &#61; optional&#40;number&#41;&#10;  &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | ✓ |  |
+| [name](variables.tf#L78) | The name of the Cloud Bigtable instance. | <code>string</code> | ✓ |  |
+| [project_id](variables.tf#L83) | Id of the project where datasets will be created. | <code>string</code> | ✓ |  |
+| [default_autoscaling](variables.tf#L33) | Default settings for autoscaling of clusters. This will be the default autoscaling for any cluster not specifying any autoscaling details. | <code title="object&#40;&#123;&#10;  min_nodes      &#61; number&#10;  max_nodes      &#61; number&#10;  cpu_target     &#61; number&#10;  storage_target &#61; optional&#40;number&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [default_gc_policy](variables.tf#L44) | Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families. | <code title="object&#40;&#123;&#10;  deletion_policy &#61; optional&#40;string&#41;&#10;  gc_rules        &#61; optional&#40;string&#41;&#10;  mode            &#61; optional&#40;string&#41;&#10;  max_age         &#61; optional&#40;string&#41;&#10;  max_version     &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [deletion_protection](variables.tf#L56) | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | <code></code> |  | <code>true</code> |
+| [display_name](variables.tf#L61) | The human-readable display name of the Bigtable instance. | <code></code> |  | <code>null</code> |
+| [iam](variables.tf#L66) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [instance_type](variables.tf#L72) | (deprecated) The instance type to create. One of 'DEVELOPMENT' or 'PRODUCTION'. | <code>string</code> |  | <code>null</code> |
+| [tables](variables.tf#L88) | Tables to be created in the BigTable instance. | <code title="map&#40;object&#40;&#123;&#10;  split_keys &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;  column_families &#61; optional&#40;map&#40;object&#40;&#10;    &#123;&#10;      gc_policy &#61; optional&#40;object&#40;&#123;&#10;        deletion_policy &#61; optional&#40;string&#41;&#10;        gc_rules        &#61; optional&#40;string&#41;&#10;        mode            &#61; optional&#40;string&#41;&#10;        max_age         &#61; optional&#40;string&#41;&#10;        max_version     &#61; optional&#40;string&#41;&#10;      &#125;&#41;, null&#41;&#10;  &#125;&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 
 ## Outputs
 

modules/bigtable-instance/main.tf

@@ -15,7 +15,6 @@
  */
 
 locals {
-  num_nodes = var.autoscaling_config == null ? var.num_nodes : null
   gc_pairs = flatten([
     for table, table_obj in var.tables : [
       for cf, cf_obj in table_obj.column_families : {
@@ -25,30 +24,45 @@ locals {
       }
     ]
   ])
+
+  clusters_autoscaling = flatten([
+    for cluster, obj in var.clusters : [{
+      cluster_id   = cluster
+      zone         = obj.zone
+      storage_type = obj.storage_type
+      num_nodes    = obj.autoscaling == null && var.default_autoscaling == null ? obj.num_nodes : null
+      autoscaling  = obj.autoscaling == null ? var.default_autoscaling : obj.autoscaling
+    }]
+  ])
 }
 
 resource "google_bigtable_instance" "default" {
   project = var.project_id
   name    = var.name
-  cluster {
-    cluster_id   = var.cluster_id
-    zone         = var.zone
-    storage_type = var.storage_type
-    num_nodes    = local.num_nodes
-    dynamic "autoscaling_config" {
-      for_each = var.autoscaling_config == null ? [] : [""]
-      content {
-        min_nodes      = var.autoscaling_config.min_nodes
-        max_nodes      = var.autoscaling_config.max_nodes
-        cpu_target     = var.autoscaling_config.cpu_target
-        storage_target = var.autoscaling_config.storage_target
+
+  instance_type = var.instance_type
+  display_name        = var.display_name == null ? var.display_name : var.name
+  deletion_protection = var.deletion_protection
+
+  dynamic "cluster" {
+    for_each = { for k, v in local.clusters_autoscaling : k => v }
+    content {
+      cluster_id   = cluster.value.cluster_id
+      zone         = cluster.value.zone
+      storage_type = cluster.value.storage_type
+      num_nodes    = cluster.value.num_nodes
+
+      dynamic "autoscaling_config" {
+        for_each = cluster.value.autoscaling == null ? [] : [""]
+        content {
+          min_nodes      = cluster.value.autoscaling.min_nodes
+          max_nodes      = cluster.value.autoscaling.max_nodes
+          cpu_target     = cluster.value.autoscaling.cpu_target
+          storage_target = cluster.value.autoscaling.storage_target
+        }
       }
     }
   }
-  instance_type = var.instance_type
-
-  display_name        = var.display_name == null ? var.display_name : var.name
-  deletion_protection = var.deletion_protection
 }
 
 resource "google_bigtable_instance_iam_binding" "default" {

modules/bigtable-instance/variables.tf

@@ -14,23 +14,33 @@
  * limitations under the License.
  */
 
-variable "autoscaling_config" {
-  description = "Settings for autoscaling of the instance. If you set this variable, the variable num_nodes is ignored."
+variable "clusters" {
+  description = "Clusters to be created in the BigTable instance. Set more than one cluster to enable replication. If you set autoscaling, num_nodes will be ignored."
+  nullable    = false
+  type = map(object({
+    zone         = optional(string)
+    storage_type = optional(string)
+    num_nodes    = optional(number)
+    autoscaling = optional(object({
+      min_nodes      = number
+      max_nodes      = number
+      cpu_target     = number
+      storage_target = optional(number)
+    }))
+  }))
+}
+
+variable "default_autoscaling" {
+  description = "Default settings for autoscaling of clusters. This will be the default autoscaling for any cluster not specifying any autoscaling details."
   type = object({
     min_nodes      = number
     max_nodes      = number
     cpu_target     = number
-    storage_target = optional(number, null)
+    storage_target = optional(number)
   })
   default = null
 }
 
-variable "cluster_id" {
-  description = "The ID of the Cloud Bigtable cluster."
-  type        = string
-  default     = "europe-west1"
-}
-
 variable "default_gc_policy" {
   description = "Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families."
   type = object({
@@ -70,23 +80,11 @@ variable "name" {
   type        = string
 }
 
-variable "num_nodes" {
-  description = "The number of nodes in your Cloud Bigtable cluster. This value is ignored if you are using autoscaling."
-  type        = number
-  default     = 1
-}
-
 variable "project_id" {
   description = "Id of the project where datasets will be created."
   type        = string
 }
 
-variable "storage_type" {
-  description = "The storage type to use."
-  type        = string
-  default     = "SSD"
-}
-
 variable "tables" {
   description = "Tables to be created in the BigTable instance."
   nullable    = false
@@ -105,8 +103,3 @@ variable "tables" {
   }))
   default = {}
 }
-
-variable "zone" {
-  description = "The zone to create the Cloud Bigtable cluster in."
-  type        = string
-}

tests/modules/bigtable_instance/fixture/main.tf

@@ -28,5 +28,9 @@ module "test" {
     }
 
   }
-  zone = var.zone
+  clusters = {
+    test = {
+      zone = var.zone
+    }
+  }
 }