Remove plg project, composer 2

fast/stages/03-data-platform/dev/README.md

@@ -31,13 +31,13 @@ The Data Platform manages:
 As per our GCP best practices the Data Platform relies on user groups to assign roles to human identities. These are the specific groups used by the Data Platform and their access patterns, from the [module documentation](../../../../blueprints/data-solutions/data-platform-foundations/#groups):
 
 - *Data Engineers* They handle and run the Data Hub, with read access to all resources in order to troubleshoot possible issues with pipelines. This team can also impersonate any service account.
-- *Data Analysts*. They perform analysis on datasets, with read access to the data warehouse Curated or Confidential projects depending on their privileges, and BigQuery READ/WRITE access to the playground project.
+- *Data Analysts*. They perform analysis on datasets, with read access to the data warehouse Curated or Confidential projects depending on their privileges.
 - *Data Security*:. They handle security configurations related to the Data Hub. This team has admin access to the common project to configure Cloud DLP templates or Data Catalog policy tags.
 
-|Group|Landing|Load|Transformation|Data Warehouse Landing|Data Warehouse Curated|Data Warehouse Confidential|Data Warehouse Playground|Orchestration|Common|
-|-|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|
-|Data Engineers|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|
-|Data Analysts|-|-|-|-|-|`READ`|`READ`/`WRITE`|-|-|
+|Group|Landing|Load|Transformation|Data Warehouse Landing|Data Warehouse Curated|Data Warehouse Confidential|Orchestration|Common|
+|-|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|
+|Data Engineers|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|`ADMIN`|
+|Data Analysts|-|-|-|-|-|`READ`|-|-|
 |Data Security|-|-|-|-|-|-|-|-|`ADMIN`|
 
 ### Network

fast/stages/03-data-platform/dev/main.tf

@@ -37,7 +37,6 @@ module "data-platform" {
     composer_ip_ranges = {
       cloudsql   = var.network_config_composer.cloudsql_range
       gke_master = var.network_config_composer.gke_master_range
-      web_server = var.network_config_composer.web_server_range
     }
     composer_secondary_ranges = {
       pods     = var.network_config_composer.gke_pods_name

fast/stages/03-data-platform/dev/variables.tf

@@ -34,14 +34,47 @@ variable "billing_account" {
 variable "composer_config" {
   description = "Cloud Composer configuration options."
   type = object({
-    node_count      = number
-    airflow_version = string
-    env_variables   = map(string)
+    disable_deployment = optional(bool)
+    environment_size   = string
+    software_config = object({
+      airflow_config_overrides = optional(any)
+      pypi_packages            = optional(any)
+      env_variables            = optional(map(string))
+      image_version            = string
+    })
+    workloads_config = object({
+      scheduler = object(
+        {
+          cpu        = number
+          memory_gb  = number
+          storage_gb = number
+          count      = number
+        }
+      )
+      web_server = object(
+        {
+          cpu        = number
+          memory_gb  = number
+          storage_gb = number
+        }
+      )
+      worker = object(
+        {
+          cpu        = number
+          memory_gb  = number
+          storage_gb = number
+          min_count  = number
+          max_count  = number
+        }
+      )
+    })
   })
   default = {
-    node_count      = 3
-    airflow_version = "composer-1.17.5-airflow-2.1.4"
-    env_variables   = {}
+    environment_size = "ENVIRONMENT_SIZE_SMALL"
+    software_config = {
+      image_version = "composer-2-airflow-2"
+    }
+    workloads_config = null
   }
 }
 
@@ -101,14 +134,12 @@ variable "network_config_composer" {
     gke_master_range  = string
     gke_pods_name     = string
     gke_services_name = string
-    web_server_range  = string
   })
   default = {
     cloudsql_range    = "192.168.254.0/24"
     gke_master_range  = "192.168.255.0/28"
     gke_pods_name     = "pods"
     gke_services_name = "services"
-    web_server_range  = "192.168.255.16/28"
   }
 }