New module for Cloud SQL instances

modules/cloudsql-instance/README.md

@@ -0,0 +1,131 @@
+# Minimalistic VPC module
+
+This module manages the creation of Cloud SQL a instances with potential read replicas in other regions. This module can also create an initial set of users and databases though the `users` and `databases` parameters.
+
+Note that this module assumes that some options are the same for both the primary instance and all the replicas (e.g. tier, disks, labels, flags, etc).
+
+## Simple example
+
+This example shows how to setup a project, VPC and a standalone Cloud SQL instance.
+
+```hcl
+module "project" {
+  source          = "./modules/project"
+  billing_account = var.billing_account_id
+  parent          = var.organization_id
+  name            = "my-db-project"
+  services = [
+    "servicenetworking.googleapis.com"
+  ]
+}
+
+module "vpc" {
+  source     = "./modules/net-vpc"
+  project_id = module.project.project_id
+  name       = "my-network"
+
+  private_service_networking_range = "10.60.0.0/16"
+}
+
+module "db" {
+  source           = "./modules/cloudsql-instance"
+  project_id       = module.project.project_id
+  network          = module.vpc.self_link
+  name             = "db"
+  region           = "europe-west1"
+  database_version = "POSTGRES_13"
+  tier             = "db-g1-small"
+}
+# tftest:modules=3:resources=6
+```
+
+## Cross-regional read replica
+
+```hcl
+module "db" {
+  source           = "./modules/cloudsql-instance"
+  project_id       = var.project_id
+  network          = var.vpc.self_link
+  name             = "db"
+  region           = "europe-west1"
+  database_version = "POSTGRES_13"
+  tier             = "db-g1-small"
+
+  replicas = {
+    replica1 = "europe-west3"
+    replica2 = "us-central1"
+  }
+}
+# tftest:modules=1:resources=3
+```
+
+## Custom flags, databases and users
+
+```hcl
+module "db" {
+  source           = "./modules/cloudsql-instance"
+  project_id       = var.project_id
+  network          = var.vpc.self_link
+  name             = "db"
+  region           = "europe-west1"
+  database_version = "MYSQL_8_0"
+  tier             = "db-g1-small"
+
+  flags = {
+    disconnect_on_expired_password = "on"
+  }
+
+  databases = [
+    "people",
+    "departments"
+  ]
+
+  users = {
+    # generatea password for user1
+    user1 = null
+    # assign a password to user2
+    user2  = "mypassword"
+  }
+}
+# tftest:modules=1:resources=6
+```
+
+<!-- BEGIN TFDOC -->
+## Variables
+
+| name | description | type | required | default |
+|---|---|:---: |:---:|:---:|
+| database_version | Database type and version to create. | <code title="">string</code> | ✓ |  |
+| name | Name of primary replica. | <code title="">string</code> | ✓ |  |
+| network | VPC self link where the instances will be deployed. Private Service Networking must be enabled and configured in this VPC. | <code title="">string</code> | ✓ |  |
+| project_id | The ID of the project where this instances will be created. | <code title="">string</code> | ✓ |  |
+| region | Region of the primary replica. | <code title="">string</code> | ✓ |  |
+| tier | The machine type to use for the instances. | <code title="">string</code> | ✓ |  |
+| *authorized_networks* | Map of NAME=>CIDR_RANGE to allow to connect to the database(s). | <code title="map&#40;string&#41;">map(string)</code> |  | <code title="">null</code> |
+| *availability_type* | Availability type for the primary replica. Either `ZONAL` or `REGIONAL` | <code title="">string</code> |  | <code title="">ZONAL</code> |
+| *backup_configuration* | Backup settings for primary instance. Will be automatically enabled if using MySQL with one or more replicas | <code title="object&#40;&#123;&#10;enabled            &#61; bool&#10;binary_log_enabled &#61; bool&#10;&#125;&#41;">object({...})</code> |  | <code title="&#123;&#10;enabled            &#61; false&#10;binary_log_enabled &#61; false&#10;&#125;">...</code> |
+| *databases* | Databases to create once the primary instance is created. | <code title="list&#40;string&#41;">list(string)</code> |  | <code title="">null</code> |
+| *deletion_protection* | None | <code title="">bool</code> |  | <code title="">false</code> |
+| *disk_size* | Disk size in GB. Set to null to enable autoresize. | <code title="">number</code> |  | <code title="">null</code> |
+| *disk_type* | The type of data disk: `PD_SSD` or `PD_HDD`. | <code title="">string</code> |  | <code title="">PD_SSD</code> |
+| *flags* | Map FLAG_NAME=>VALUE for database-specific tuning. | <code title="map&#40;string&#41;">map(string)</code> |  | <code title="">null</code> |
+| *labels* | Labels to be attached to all instances. | <code title="map&#40;string&#41;">map(string)</code> |  | <code title="">null</code> |
+| *prefix* | Prefix used to generate instance names. | <code title="">string</code> |  | <code title="">null</code> |
+| *replicas* | Map of NAME=>REGION for additional read replicas. Set to null to disable replica creation. | <code title="map&#40;any&#41;">map(any)</code> |  | <code title="">null</code> |
+| *users* | Map of users to create in the primary instance (and replicated to other replicas) in the format USER=>PASSWORD. For MySQL, anything afterr the first `@` (if persent) will be used as the user's host. Set PASSWORD to null if you want to get an autogenerated password | <code title="map&#40;string&#41;">map(string)</code> |  | <code title="">null</code> |
+
+## Outputs
+
+| name | description | sensitive |
+|---|---|:---:|
+| connection_name | Connection name of the primary instance |  |
+| connection_names | Connection names of all instances |  |
+| id | ID of the primary instance |  |
+| ids | IDs of all instances |  |
+| instances | Cloud SQL instance resources | ✓ |
+| ip | IP address of the primary instance |  |
+| ips | IP addresses of all instances |  |
+| self_link | Self link of the primary instance |  |
+| self_links | Self links of all instances |  |
+| user_passwords | Map of containing the password of all users created through terraform. | ✓ |
+<!-- END TFDOC -->

modules/cloudsql-instance/main.tf

@@ -0,0 +1,159 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  prefix       = var.prefix == null ? "" : "${var.prefix}-"
+  is_mysql     = can(regex("^MYSQL", var.database_version))
+  has_replicas = try(length(var.replicas) > 0, false)
+
+  users = {
+    for user, password in coalesce(var.users, {}) :
+    (user) => (
+      local.is_mysql
+      ? {
+        name     = split("@", user)[0]
+        host     = try(split("@", user)[1], null)
+        password = try(random_password.passwords[user].result, password)
+      }
+      : {
+        name     = user
+        host     = null
+        password = try(random_password.passwords[user].result, password)
+      }
+    )
+  }
+
+}
+
+resource "google_sql_database_instance" "primary" {
+  project          = var.project_id
+  name             = "${local.prefix}${var.name}"
+  region           = var.region
+  database_version = var.database_version
+
+  settings {
+    tier              = var.tier
+    disk_autoresize   = var.disk_size == null
+    disk_size         = var.disk_size
+    disk_type         = var.disk_type
+    availability_type = var.availability_type
+    user_labels       = var.labels
+
+    ip_configuration {
+      ipv4_enabled    = false
+      private_network = var.network
+      dynamic "authorized_networks" {
+        for_each = var.authorized_networks != null ? var.authorized_networks : {}
+        iterator = network
+        content {
+          name  = network.key
+          value = network.value
+        }
+      }
+    }
+
+    backup_configuration {
+      // Enable backup if the user asks for it or if the user is
+      // deploying MySQL with replicas
+      enabled = var.backup_configuration.enabled || (local.is_mysql && local.has_replicas)
+
+      // enable binary log if the user asks for it or we have replicas,
+      // but only form MySQL
+      binary_log_enabled = (
+        local.is_mysql
+        ? var.backup_configuration.binary_log_enabled || local.has_replicas
+        : null
+      )
+    }
+
+    dynamic "database_flags" {
+      for_each = var.flags != null ? var.flags : {}
+      iterator = flag
+      content {
+        name  = flag.key
+        value = flag.value
+      }
+    }
+  }
+  deletion_protection = var.deletion_protection
+}
+
+resource "google_sql_database_instance" "replicas" {
+  for_each             = local.has_replicas ? var.replicas : {}
+  project              = var.project_id
+  name                 = "${local.prefix}${each.key}"
+  region               = each.value
+  database_version     = var.database_version
+  master_instance_name = google_sql_database_instance.primary.name
+
+  settings {
+    tier            = var.tier
+    disk_autoresize = var.disk_size == null
+    disk_size       = var.disk_size
+    disk_type       = var.disk_type
+    # availability_type = var.availability_type
+    user_labels = var.labels
+
+    ip_configuration {
+      ipv4_enabled    = false
+      private_network = var.network
+      dynamic "authorized_networks" {
+        for_each = var.authorized_networks != null ? var.authorized_networks : {}
+        iterator = network
+        content {
+          name  = network.key
+          value = network.value
+        }
+      }
+    }
+
+    dynamic "database_flags" {
+      for_each = var.flags != null ? var.flags : {}
+      iterator = flag
+      content {
+        name  = flag.key
+        value = flag.value
+      }
+    }
+  }
+  deletion_protection = var.deletion_protection
+}
+
+resource "google_sql_database" "databases" {
+  for_each = var.databases != null ? toset(var.databases) : toset([])
+  project  = var.project_id
+  instance = google_sql_database_instance.primary.name
+  name     = each.key
+}
+
+resource "random_password" "passwords" {
+  for_each = toset([
+    for user, password in coalesce(var.users, {}) :
+    user
+    if password == null
+  ])
+  length  = 16
+  special = true
+}
+
+resource "google_sql_user" "users" {
+  for_each = local.users
+  project  = var.project_id
+  instance = google_sql_database_instance.primary.name
+  name     = each.value.name
+  host     = each.value.host
+  password = each.value.password
+}

modules/cloudsql-instance/outputs.tf

@@ -0,0 +1,89 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  _all_intances = merge(
+    { primary = google_sql_database_instance.primary },
+    google_sql_database_instance.replicas
+  )
+}
+
+output "connection_name" {
+  description = "Connection name of the primary instance"
+  value       = google_sql_database_instance.primary.connection_name
+}
+
+output "connection_names" {
+  description = "Connection names of all instances"
+  value = {
+    for id, instance in local._all_intances :
+    id => instance.connection_name
+  }
+}
+
+output "id" {
+  description = "ID of the primary instance"
+  value       = google_sql_database_instance.primary.private_ip_address
+}
+
+output "ids" {
+  description = "IDs of all instances"
+  value = {
+    for id, instance in local._all_intances :
+    id => instance.id
+  }
+}
+
+output "instances" {
+  description = "Cloud SQL instance resources"
+  value       = local._all_intances
+  sensitive   = true
+}
+
+output "ip" {
+  description = "IP address of the primary instance"
+  value       = google_sql_database_instance.primary.private_ip_address
+}
+
+output "ips" {
+  description = "IP addresses of all instances"
+  value = {
+    for id, instance in local._all_intances :
+    id => instance.private_ip_address
+  }
+}
+
+output "self_link" {
+  description = "Self link of the primary instance"
+  value       = google_sql_database_instance.primary.self_link
+}
+
+output "self_links" {
+  description = "Self links of all instances"
+  value = {
+    for id, instance in local._all_intances :
+    id => instance.self_link
+  }
+}
+
+output "user_passwords" {
+  description = "Map of containing the password of all users created through terraform."
+  value = {
+    for name, user in google_sql_user.users :
+    name => user.password
+  }
+  sensitive = true
+}

modules/cloudsql-instance/variables.tf

@@ -0,0 +1,122 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "authorized_networks" {
+  description = "Map of NAME=>CIDR_RANGE to allow to connect to the database(s)."
+  type        = map(string)
+  default     = null
+}
+
+variable "availability_type" {
+  description = "Availability type for the primary replica. Either `ZONAL` or `REGIONAL`"
+  type        = string
+  default     = "ZONAL"
+}
+
+variable "backup_configuration" {
+  description = "Backup settings for primary instance. Will be automatically enabled if using MySQL with one or more replicas"
+  type = object({
+    enabled            = bool
+    binary_log_enabled = bool
+  })
+  default = {
+    enabled            = false
+    binary_log_enabled = false
+  }
+}
+
+variable "database_version" {
+  description = "Database type and version to create."
+  type        = string
+}
+
+variable "databases" {
+  description = "Databases to create once the primary instance is created."
+  type        = list(string)
+  default     = null
+}
+
+variable "disk_size" {
+  description = "Disk size in GB. Set to null to enable autoresize."
+  type        = number
+  default     = null
+}
+
+variable "disk_type" {
+  description = "The type of data disk: `PD_SSD` or `PD_HDD`."
+  type        = string
+  default     = "PD_SSD"
+}
+
+variable "flags" {
+  description = "Map FLAG_NAME=>VALUE for database-specific tuning."
+  type        = map(string)
+  default     = null
+}
+
+variable "labels" {
+  description = "Labels to be attached to all instances."
+  type        = map(string)
+  default     = null
+}
+
+variable "name" {
+  description = "Name of primary replica."
+  type        = string
+}
+
+variable "network" {
+  description = "VPC self link where the instances will be deployed. Private Service Networking must be enabled and configured in this VPC."
+  type        = string
+}
+
+variable "prefix" {
+  description = "Prefix used to generate instance names."
+  type        = string
+  default     = null
+}
+
+variable "project_id" {
+  description = "The ID of the project where this instances will be created."
+  type        = string
+}
+
+variable "region" {
+  description = "Region of the primary replica."
+  type        = string
+}
+
+variable "replicas" {
+  description = "Map of NAME=>REGION for additional read replicas. Set to null to disable replica creation."
+  type        = map(any)
+  default     = null
+}
+
+variable "users" {
+  description = "Map of users to create in the primary instance (and replicated to other replicas) in the format USER=>PASSWORD. For MySQL, anything afterr the first `@` (if persent) will be used as the user's host. Set PASSWORD to null if you want to get an autogenerated password"
+  type        = map(string)
+  default     = null
+}
+
+variable "tier" {
+  description = "The machine type to use for the instances."
+  type        = string
+}
+
+variable "deletion_protection" {
+  type    = bool
+  default = false
+}

tests/examples/variables.tf

@@ -60,7 +60,7 @@ variable "subnet" {
 variable "vpc" {
   default = {
     name      = "vpc_name"
-    self_link = "vpc_self_link"
+    self_link = "projects/xxx/global/networks/yyy"
   }
 }
 

tests/modules/cloudsql_instance/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/modules/cloudsql_instance/fixture/main.tf

@@ -0,0 +1,37 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "test" {
+  source               = "../../../../modules/cloudsql-instance"
+  project_id           = "my-project"
+  authorized_networks  = var.authorized_networks
+  availability_type    = var.availability_type
+  backup_configuration = var.backup_configuration
+  database_version     = var.database_version
+  databases            = var.databases
+  disk_size            = var.disk_size
+  disk_type            = var.disk_type
+  flags                = var.flags
+  labels               = var.labels
+  name                 = var.name
+  network              = var.network
+  prefix               = var.prefix
+  region               = var.region
+  replicas             = var.replicas
+  users                = var.users
+  tier                 = var.tier
+  deletion_protection  = var.deletion_protection
+}

tests/modules/cloudsql_instance/fixture/variables.tf

@@ -0,0 +1,106 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "authorized_networks" {
+  type    = map(string)
+  default = null
+}
+
+variable "availability_type" {
+  type    = string
+  default = "ZONAL"
+}
+
+variable "backup_configuration" {
+  type = object({
+    enabled            = bool
+    binary_log_enabled = bool
+  })
+  default = {
+    enabled            = false
+    binary_log_enabled = false
+  }
+}
+
+variable "database_version" {
+  type    = string
+  default = "POSTGRES_13"
+}
+
+variable "databases" {
+  type    = list(string)
+  default = null
+}
+
+variable "disk_size" {
+  type    = number
+  default = null
+}
+
+variable "disk_type" {
+  type    = string
+  default = "PD_SSD"
+}
+
+variable "flags" {
+  type    = map(string)
+  default = null
+}
+
+variable "labels" {
+  type    = map(string)
+  default = null
+}
+
+variable "name" {
+  type    = string
+  default = "db"
+}
+
+variable "network" {
+  type    = string
+  default = "projects/xxx/global/networks/yyy"
+}
+
+variable "prefix" {
+  type    = string
+  default = null
+}
+
+variable "region" {
+  type    = string
+  default = "europe-west1"
+}
+
+variable "replicas" {
+  type    = map(any)
+  default = null
+}
+
+variable "users" {
+  type    = map(string)
+  default = null
+}
+
+variable "tier" {
+  type    = string
+  default = "db-g1-small"
+}
+
+variable "deletion_protection" {
+  type    = bool
+  default = false
+}

tests/modules/cloudsql_instance/test_plan.py

@@ -0,0 +1,125 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import pytest
+
+from collections import Counter
+
+FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture')
+
+from pprint import pprint
+
+
+def test_simple_instance(plan_runner):
+  "Test standalone instance."
+  _, resources = plan_runner(FIXTURES_DIR)
+
+  pprint(resources)
+  assert len(resources) == 1
+  r = resources[0]
+  assert r['values']['project'] == 'my-project'
+  assert r['values']['name'] == 'db'
+  assert r['values']['region'] == 'europe-west1'
+
+
+def test_prefix(plan_runner):
+  "Test instance prefix."
+  _, resources = plan_runner(FIXTURES_DIR, prefix="prefix")
+
+  assert len(resources) == 1
+  r = resources[0]
+  assert r['values']['name'] == 'prefix-db'
+
+  replicas = """{
+    replica1 = "europe-west3"
+    replica2 = "us-central1"
+  }"""
+
+  _, resources = plan_runner(FIXTURES_DIR, prefix="prefix")
+  assert len(resources) == 1
+  r = resources[0]
+  assert r['values']['name'] == 'prefix-db'
+
+
+def test_replicas(plan_runner):
+  "Test replicated instance."
+
+  replicas = """{
+    replica1 = "europe-west3"
+    replica2 = "us-central1"
+  }"""
+  _, resources = plan_runner(FIXTURES_DIR, replicas=replicas, prefix="prefix")
+  assert len(resources) == 3
+
+  primary = [r for r in resources if r['name'] == 'primary'][0]
+  replica1 = [
+      r for r in resources
+      if r['name'] == 'replicas' and r['index'] == 'replica1'
+  ][0]
+  replica2 = [
+      r for r in resources
+      if r['name'] == 'replicas' and r['index'] == 'replica2'
+  ][0]
+
+  assert replica1['values']['name'] == 'prefix-replica1'
+  assert replica2['values']['name'] == 'prefix-replica2'
+
+  assert replica1['values']['master_instance_name'] == 'prefix-db'
+  assert replica2['values']['master_instance_name'] == 'prefix-db'
+
+  assert replica1['values']['region'] == 'europe-west3'
+  assert replica2['values']['region'] == 'us-central1'
+
+
+def test_mysql_replicas_enables_backup(plan_runner):
+  replicas = """{
+    replica1 = "europe-west3"
+  }"""
+  _, resources = plan_runner(FIXTURES_DIR,
+                             replicas=replicas,
+                             database_version="MYSQL_8_0")
+  assert len(resources) == 2
+  primary = [r for r in resources if r['name'] == 'primary'][0]
+  backup_config = primary['values']['settings'][0]['backup_configuration'][0]
+  assert backup_config['enabled']
+  assert backup_config['binary_log_enabled']
+
+
+def test_users(plan_runner):
+  "Test user creation."
+
+  users = """{
+    user1 = "123"
+    user2 = null
+  }"""
+  _, resources = plan_runner(FIXTURES_DIR, users=users)
+  types = Counter(r['type'] for r in resources)
+  assert types == {
+      'google_sql_user': 2,
+      'google_sql_database_instance': 1,
+      'random_password': 1
+  }
+
+
+def test_databases(plan_runner):
+  "Test database creation."
+
+  databases = '["db1", "db2"]'
+  _, resources = plan_runner(FIXTURES_DIR, databases=databases)
+
+  resources = [r for r in resources if r['type'] == 'google_sql_database']
+  assert len(resources) == 2
+  assert all(r['values']['instance'] == "db" for r in resources)
+  assert sorted(r['values']['name'] for r in resources) == ["db1", "db2"]