Make deletion protection consistent across all modules (#1735)

* Expose deletion_protection in GKE modules * Make deletion protection consistent across all modules * Add deletion_protection option to blueprints * Fix blueprints tests * Fix types * Update READMEs * Fix dp readme * Fix cmek blueprint default deletion_protection * Fix blueprints tests
2023-10-05 17:31:07 +02:00
parent 6faf34f54e
commit dfc5023e0b
74 changed files with 468 additions and 330 deletions
--- a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf
+++ b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -91,7 +91,7 @@ module "drop-cs-0" {
  location       = var.location
  storage_class  = "MULTI_REGIONAL"
  encryption_key = try(local.service_encryption_keys.storage, null)
-  force_destroy  = var.data_force_destroy
+  force_destroy  = !var.deletion_protection
  # retention_policy = {
  #   retention_period = 7776000 # 90 * 24 * 60 * 60
  #   is_locked        = false
--- a/blueprints/data-solutions/data-platform-foundations/02-load.tf
+++ b/blueprints/data-solutions/data-platform-foundations/02-load.tf
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -98,6 +98,7 @@ module "load-cs-df-0" {
  location       = var.location
  storage_class  = "MULTI_REGIONAL"
  encryption_key = try(local.service_encryption_keys.storage, null)
+  force_destroy  = !var.deletion_protection
 }

 module "load-vpc" {
--- a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
+++ b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -109,6 +109,7 @@ module "orch-cs-0" {
  location       = var.location
  storage_class  = "MULTI_REGIONAL"
  encryption_key = try(local.service_encryption_keys.storage, null)
+  force_destroy  = !var.deletion_protection
 }

 module "orch-vpc" {
@@ -164,6 +165,7 @@ module "orch-cs-df-template" {
  location       = var.region
  storage_class  = "REGIONAL"
  encryption_key = try(local.service_encryption_keys.storage, null)
+  force_destroy  = !var.deletion_protection
 }

 module "orch-cs-build-staging" {
@@ -174,6 +176,7 @@ module "orch-cs-build-staging" {
  location       = var.region
  storage_class  = "REGIONAL"
  encryption_key = try(local.service_encryption_keys.storage, null)
+  force_destroy  = !var.deletion_protection
 }

 module "orch-sa-df-build" {
--- a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf
+++ b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -97,6 +97,7 @@ module "transf-cs-df-0" {
  location       = var.location
  storage_class  = "MULTI_REGIONAL"
  encryption_key = try(local.service_encryption_keys.storage, null)
+  force_destroy  = !var.deletion_protection
 }

 module "transf-sa-bq-0" {
--- a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf
+++ b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -156,7 +156,7 @@ module "dwh-lnd-cs-0" {
  location       = var.location
  storage_class  = "MULTI_REGIONAL"
  encryption_key = try(local.service_encryption_keys.storage, null)
-  force_destroy  = var.data_force_destroy
+  force_destroy  = !var.deletion_protection
 }

 module "dwh-cur-cs-0" {
@@ -167,7 +167,7 @@ module "dwh-cur-cs-0" {
  location       = var.location
  storage_class  = "MULTI_REGIONAL"
  encryption_key = try(local.service_encryption_keys.storage, null)
-  force_destroy  = var.data_force_destroy
+  force_destroy  = !var.deletion_protection
 }

 module "dwh-conf-cs-0" {
@@ -178,5 +178,5 @@ module "dwh-conf-cs-0" {
  location       = var.location
  storage_class  = "MULTI_REGIONAL"
  encryption_key = try(local.service_encryption_keys.storage, null)
-  force_destroy  = var.data_force_destroy
+  force_destroy  = !var.deletion_protection
 }
--- a/blueprints/data-solutions/data-platform-foundations/README.md
+++ b/blueprints/data-solutions/data-platform-foundations/README.md
@@ -279,19 +279,19 @@ terraform destroy

 | name | description | type | required | default |
 |---|---|:---:|:---:|:---:|
-| [organization_domain](variables.tf#L165) | Organization domain. | <code>string</code> | ✓ |  |
-| [prefix](variables.tf#L170) | Prefix used for resource names. | <code>string</code> | ✓ |  |
-| [project_config](variables.tf#L179) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object&#40;&#123;&#10;  billing_account_id &#61; optional&#40;string, null&#41;&#10;  parent             &#61; string&#10;  project_ids &#61; optional&#40;object&#40;&#123;&#10;    drop     &#61; string&#10;    load     &#61; string&#10;    orc      &#61; string&#10;    trf      &#61; string&#10;    dwh-lnd  &#61; string&#10;    dwh-cur  &#61; string&#10;    dwh-conf &#61; string&#10;    common   &#61; string&#10;    exp      &#61; string&#10;    &#125;&#41;, &#123;&#10;    drop     &#61; &#34;drp&#34;&#10;    load     &#61; &#34;lod&#34;&#10;    orc      &#61; &#34;orc&#34;&#10;    trf      &#61; &#34;trf&#34;&#10;    dwh-lnd  &#61; &#34;dwh-lnd&#34;&#10;    dwh-cur  &#61; &#34;dwh-cur&#34;&#10;    dwh-conf &#61; &#34;dwh-conf&#34;&#10;    common   &#61; &#34;cmn&#34;&#10;    exp      &#61; &#34;exp&#34;&#10;    &#125;&#10;  &#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |
+| [organization_domain](variables.tf#L166) | Organization domain. | <code>string</code> | ✓ |  |
+| [prefix](variables.tf#L171) | Prefix used for resource names. | <code>string</code> | ✓ |  |
+| [project_config](variables.tf#L180) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object&#40;&#123;&#10;  billing_account_id &#61; optional&#40;string, null&#41;&#10;  parent             &#61; string&#10;  project_ids &#61; optional&#40;object&#40;&#123;&#10;    drop     &#61; string&#10;    load     &#61; string&#10;    orc      &#61; string&#10;    trf      &#61; string&#10;    dwh-lnd  &#61; string&#10;    dwh-cur  &#61; string&#10;    dwh-conf &#61; string&#10;    common   &#61; string&#10;    exp      &#61; string&#10;    &#125;&#41;, &#123;&#10;    drop     &#61; &#34;drp&#34;&#10;    load     &#61; &#34;lod&#34;&#10;    orc      &#61; &#34;orc&#34;&#10;    trf      &#61; &#34;trf&#34;&#10;    dwh-lnd  &#61; &#34;dwh-lnd&#34;&#10;    dwh-cur  &#61; &#34;dwh-cur&#34;&#10;    dwh-conf &#61; &#34;dwh-conf&#34;&#10;    common   &#61; &#34;cmn&#34;&#10;    exp      &#61; &#34;exp&#34;&#10;    &#125;&#10;  &#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |
 | [composer_config](variables.tf#L17) | Cloud Composer config. | <code title="object&#40;&#123;&#10;  disable_deployment &#61; optional&#40;bool&#41;&#10;  environment_size   &#61; optional&#40;string, &#34;ENVIRONMENT_SIZE_SMALL&#34;&#41;&#10;  software_config &#61; optional&#40;&#10;    object&#40;&#123;&#10;      airflow_config_overrides       &#61; optional&#40;any&#41;&#10;      pypi_packages                  &#61; optional&#40;any&#41;&#10;      env_variables                  &#61; optional&#40;map&#40;string&#41;&#41;&#10;      image_version                  &#61; string&#10;      cloud_data_lineage_integration &#61; optional&#40;bool, true&#41;&#10;    &#125;&#41;,&#10;    &#123; image_version &#61; &#34;composer-2-airflow-2&#34; &#125;&#10;  &#41;&#10;  workloads_config &#61; optional&#40;&#10;    object&#40;&#123;&#10;      scheduler &#61; optional&#40;&#10;        object&#40;&#123;&#10;          cpu        &#61; number&#10;          memory_gb  &#61; number&#10;          storage_gb &#61; number&#10;          count      &#61; number&#10;        &#125;&#41;,&#10;        &#123;&#10;          cpu        &#61; 0.5&#10;          memory_gb  &#61; 1.875&#10;          storage_gb &#61; 1&#10;          count      &#61; 1&#10;        &#125;&#10;      &#41;&#10;      web_server &#61; optional&#40;&#10;        object&#40;&#123;&#10;          cpu        &#61; number&#10;          memory_gb  &#61; number&#10;          storage_gb &#61; number&#10;        &#125;&#41;,&#10;        &#123;&#10;          cpu        &#61; 0.5&#10;          memory_gb  &#61; 1.875&#10;          storage_gb &#61; 1&#10;        &#125;&#10;      &#41;&#10;      worker &#61; optional&#40;&#10;        object&#40;&#123;&#10;          cpu        &#61; number&#10;          memory_gb  &#61; number&#10;          storage_gb &#61; number&#10;          min_count  &#61; number&#10;          max_count  &#61; number&#10;        &#125;&#41;,&#10;        &#123;&#10;          cpu        &#61; 0.5&#10;          memory_gb  &#61; 1.875&#10;          storage_gb &#61; 1&#10;          min_count  &#61; 1&#10;          max_count  &#61; 3&#10;        &#125;&#10;      &#41;&#10;  &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  environment_size &#61; &#34;ENVIRONMENT_SIZE_SMALL&#34;&#10;  software_config &#61; &#123;&#10;    image_version &#61; &#34;composer-2-airflow-2&#34;&#10;  &#125;&#10;  workloads_config &#61; &#123;&#10;    scheduler &#61; &#123;&#10;      cpu        &#61; 0.5&#10;      memory_gb  &#61; 1.875&#10;      storage_gb &#61; 1&#10;      count      &#61; 1&#10;    &#125;&#10;    web_server &#61; &#123;&#10;      cpu        &#61; 0.5&#10;      memory_gb  &#61; 1.875&#10;      storage_gb &#61; 1&#10;    &#125;&#10;    worker &#61; &#123;&#10;      cpu        &#61; 0.5&#10;      memory_gb  &#61; 1.875&#10;      storage_gb &#61; 1&#10;      min_count  &#61; 1&#10;      max_count  &#61; 3&#10;    &#125;&#10;  &#125;&#10;&#125;">&#123;&#8230;&#125;</code> |
 | [data_catalog_tags](variables.tf#L106) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code title="map&#40;object&#40;&#123;&#10;  description &#61; optional&#40;string&#41;&#10;  iam         &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code title="&#123;&#10;  &#34;3_Confidential&#34; &#61; &#123;&#125;&#10;  &#34;2_Private&#34;      &#61; &#123;&#125;&#10;  &#34;1_Sensitive&#34;    &#61; &#123;&#125;&#10;&#125;">&#123;&#8230;&#125;</code> |
-| [data_force_destroy](variables.tf#L120) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | <code>bool</code> |  | <code>false</code> |
-| [groups](variables.tf#L126) | User groups. | <code>map&#40;string&#41;</code> |  | <code title="&#123;&#10;  data-analysts  &#61; &#34;gcp-data-analysts&#34;&#10;  data-engineers &#61; &#34;gcp-data-engineers&#34;&#10;  data-security  &#61; &#34;gcp-data-security&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |
-| [location](variables.tf#L136) | Location used for multi-regional resources. | <code>string</code> |  | <code>&#34;eu&#34;</code> |
-| [network_config](variables.tf#L142) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object&#40;&#123;&#10;  host_project      &#61; string&#10;  network_self_link &#61; string&#10;  subnet_self_links &#61; object&#40;&#123;&#10;    load           &#61; string&#10;    transformation &#61; string&#10;    orchestration  &#61; string&#10;  &#125;&#41;&#10;  composer_ip_ranges &#61; object&#40;&#123;&#10;    cloudsql   &#61; string&#10;    gke_master &#61; string&#10;  &#125;&#41;&#10;  composer_secondary_ranges &#61; object&#40;&#123;&#10;    pods     &#61; string&#10;    services &#61; string&#10;  &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
-| [project_services](variables.tf#L213) | List of core services enabled on all projects. | <code>list&#40;string&#41;</code> |  | <code title="&#91;&#10;  &#34;cloudresourcemanager.googleapis.com&#34;,&#10;  &#34;iam.googleapis.com&#34;,&#10;  &#34;serviceusage.googleapis.com&#34;,&#10;  &#34;stackdriver.googleapis.com&#34;&#10;&#93;">&#91;&#8230;&#93;</code> |
-| [project_suffix](variables.tf#L224) | Suffix used only for project ids. | <code>string</code> |  | <code>null</code> |
-| [region](variables.tf#L230) | Region used for regional resources. | <code>string</code> |  | <code>&#34;europe-west1&#34;</code> |
-| [service_encryption_keys](variables.tf#L236) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object&#40;&#123;&#10;  bq       &#61; string&#10;  composer &#61; string&#10;  dataflow &#61; string&#10;  storage  &#61; string&#10;  pubsub   &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [deletion_protection](variables.tf#L120) | Prevent Terraform from destroying data storage resources (storage buckets, GKE clusters, CloudSQL instances) in this blueprint. When this field is set in Terraform state, a terraform destroy or terraform apply that would delete data storage resources will fail. | <code>bool</code> |  | <code>false</code> |
+| [groups](variables.tf#L127) | User groups. | <code>map&#40;string&#41;</code> |  | <code title="&#123;&#10;  data-analysts  &#61; &#34;gcp-data-analysts&#34;&#10;  data-engineers &#61; &#34;gcp-data-engineers&#34;&#10;  data-security  &#61; &#34;gcp-data-security&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |
+| [location](variables.tf#L137) | Location used for multi-regional resources. | <code>string</code> |  | <code>&#34;eu&#34;</code> |
+| [network_config](variables.tf#L143) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object&#40;&#123;&#10;  host_project      &#61; string&#10;  network_self_link &#61; string&#10;  subnet_self_links &#61; object&#40;&#123;&#10;    load           &#61; string&#10;    transformation &#61; string&#10;    orchestration  &#61; string&#10;  &#125;&#41;&#10;  composer_ip_ranges &#61; object&#40;&#123;&#10;    cloudsql   &#61; string&#10;    gke_master &#61; string&#10;  &#125;&#41;&#10;  composer_secondary_ranges &#61; object&#40;&#123;&#10;    pods     &#61; string&#10;    services &#61; string&#10;  &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [project_services](variables.tf#L214) | List of core services enabled on all projects. | <code>list&#40;string&#41;</code> |  | <code title="&#91;&#10;  &#34;cloudresourcemanager.googleapis.com&#34;,&#10;  &#34;iam.googleapis.com&#34;,&#10;  &#34;serviceusage.googleapis.com&#34;,&#10;  &#34;stackdriver.googleapis.com&#34;&#10;&#93;">&#91;&#8230;&#93;</code> |
+| [project_suffix](variables.tf#L225) | Suffix used only for project ids. | <code>string</code> |  | <code>null</code> |
+| [region](variables.tf#L231) | Region used for regional resources. | <code>string</code> |  | <code>&#34;europe-west1&#34;</code> |
+| [service_encryption_keys](variables.tf#L237) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object&#40;&#123;&#10;  bq       &#61; string&#10;  composer &#61; string&#10;  dataflow &#61; string&#10;  storage  &#61; string&#10;  pubsub   &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |

 ## Outputs

--- a/blueprints/data-solutions/data-platform-foundations/variables.tf
+++ b/blueprints/data-solutions/data-platform-foundations/variables.tf
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -117,10 +117,11 @@ variable "data_catalog_tags" {
  }
 }

-variable "data_force_destroy" {
-  description = "Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage."
+variable "deletion_protection" {
+  description = "Prevent Terraform from destroying data storage resources (storage buckets, GKE clusters, CloudSQL instances) in this blueprint. When this field is set in Terraform state, a terraform destroy or terraform apply that would delete data storage resources will fail."
  type        = bool
  default     = false
+  nullable    = false
 }

 variable "groups" {