Implement additional GCS attributes in project factory (#3583)

* implement additional bucket attributes in project factory

* update FAST schemas

* fmt/tfdoc

modules/dataplex-aspect-types/schemas/aspect-type.schema.md

@@ -18,18 +18,18 @@
 
 - **iam**<a name="refs-iam"></a>: *object*
   <br>*additional properties: false*
-  - **`^roles/`**: *array*
+  - **`^(?:roles/|\$custom_roles:)`**: *array*
     - items: *string*
-      <br>*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|[a-z])*
+      <br>*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:||\$iam_principals:[a-z0-9_-]+)*
 - **iam_bindings**<a name="refs-iam_bindings"></a>: *object*
   <br>*additional properties: false*
   - **`^[a-z0-9_-]+$`**: *object*
     <br>*additional properties: false*
     - **members**: *array*
       - items: *string*
-        <br>*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|[a-z])*
+        <br>*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\$iam_principals:[a-z0-9_-]+)*
     - **role**: *string*
-      <br>*pattern: ^roles/*
+      <br>*pattern: ^(?:roles/|\$custom_roles:)*
     - **condition**: *object*
       <br>*additional properties: false*
       - ⁺**expression**: *string*
@@ -40,9 +40,9 @@
   - **`^[a-z0-9_-]+$`**: *object*
     <br>*additional properties: false*
     - **member**: *string*
-      <br>*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|[a-z])*
+      <br>*pattern: ^(?:domain:|group:|serviceAccount:|user:|principal:|principalSet:|\$iam_principals:[a-z0-9_-]+)*
     - **role**: *string*
-      <br>*pattern: ^roles/*
+      <br>*pattern: ^(?:roles/|\$custom_roles:)*
     - **condition**: *object*
       <br>*additional properties: false*
       - ⁺**expression**: *string*

modules/project-factory/automation.tf

@@ -39,10 +39,13 @@ locals {
   )
   _automation_buckets = {
     for k, v in local._automation : k => merge(v.bucket, {
-      automation_project = v.project
-      parent_name        = v.parent_name
-      name               = lookup(v.bucket, "name", "tf-state")
-      create             = lookup(v.bucket, "create", true)
+      automation_project    = v.project
+      parent_name           = v.parent_name
+      name                  = lookup(v.bucket, "name", "tf-state")
+      create                = lookup(v.bucket, "create", true)
+      lifecycle_rules       = lookup(v.bucket, "lifecycle_rules", {})
+      retention_policy      = lookup(v.bucket, "retention_policy", null)
+      soft_delete_retention = lookup(v.bucket, "soft_delete_retention", null)
       prefix = try(coalesce(
         local.data_defaults.overrides.prefix,
         v.prefix,
@@ -99,6 +102,7 @@ module "automation-bucket" {
   iam_bindings          = lookup(each.value, "iam_bindings", {})
   iam_bindings_additive = lookup(each.value, "iam_bindings_additive", {})
   labels                = lookup(each.value, "labels", {})
+  lifecycle_rules       = each.value.lifecycle_rules
   managed_folders       = lookup(each.value, "managed_folders", {})
   location = each.value.create == false ? null : coalesce(
     local.data_defaults.overrides.locations.storage,
@@ -114,6 +118,8 @@ module "automation-bucket" {
   versioning = lookup(
     each.value, "versioning", false
   )
+  retention_policy      = each.value.retention_policy
+  soft_delete_retention = each.value.soft_delete_retention
 }
 
 module "automation-service-accounts" {

modules/project-factory/projects-buckets.tf

@@ -51,6 +51,8 @@ locals {
           opts, "versioning", false
         )
         retention_policy        = lookup(opts, "retention_policy", null)
+        soft_delete_retention   = lookup(opts, "soft_delete_retention", null)
+        lifecycle_rules         = lookup(opts, "lifecycle_rules", {})
         enable_object_retention = lookup(opts, "enable_object_retention", null)
       }
     ]
@@ -84,6 +86,7 @@ module "buckets" {
   iam_bindings_additive = each.value.iam_bindings_additive
   iam_by_principals     = each.value.iam_by_principals
   labels                = each.value.labels
+  lifecycle_rules       = each.value.lifecycle_rules
   location = coalesce(
     local.data_defaults.overrides.locations.storage,
     lookup(each.value, "location", null),
@@ -94,5 +97,6 @@ module "buckets" {
   uniform_bucket_level_access = each.value.uniform_bucket_level_access
   versioning                  = each.value.versioning
   retention_policy            = each.value.retention_policy
+  soft_delete_retention       = each.value.soft_delete_retention
   enable_object_retention     = each.value.enable_object_retention
 }

modules/project-factory/schemas/folder.schema.md

@@ -29,6 +29,8 @@
   <br>*additional properties: false*
   - **project**: *string*
     <br>*pattern: ^(projects/|\$project_ids:|\$project_numbers:)*
+- **billing_budgets**: *array*
+  - items: *string*
 - **contacts**: *object*
   <br>*additional properties: false*
   - **`^(\S+@\S+\.\S+|\$email_addresses:\S+)$`**: *array*

modules/project-factory/schemas/project.schema.json

@@ -919,6 +919,104 @@
             "type": "string"
           }
         },
+        "lifecycle_rules": {
+          "type": "object",
+          "additionalProperties": false,
+          "patternProperties": {
+            "^[a-zA-Z0-9_-]+$": {
+              "type": "object",
+              "additionalProperties": false,
+              "required": [
+                "action",
+                "condition"
+              ],
+              "properties": {
+                "action": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "required": [
+                    "type"
+                  ],
+                  "properties": {
+                    "type": {
+                      "type": "string",
+                      "enum": [
+                        "Delete",
+                        "SetStorageClass",
+                        "AbortIncompleteMultipartUpload"
+                      ]
+                    },
+                    "storage_class": {
+                      "type": "string"
+                    }
+                  }
+                },
+                "condition": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "properties": {
+                    "age": {
+                      "type": "number"
+                    },
+                    "created_before": {
+                      "type": "string"
+                    },
+                    "custom_time_before": {
+                      "type": "string"
+                    },
+                    "days_since_custom_time": {
+                      "type": "number"
+                    },
+                    "days_since_noncurrent_time": {
+                      "type": "number"
+                    },
+                    "matches_prefix": {
+                      "type": "array",
+                      "items": {
+                        "type": "string"
+                      }
+                    },
+                    "matches_storage_class": {
+                      "type": "array",
+                      "items": {
+                        "type": "string",
+                        "enum": [
+                          "STANDARD",
+                          "MULTI_REGIONAL",
+                          "REGIONAL",
+                          "NEARLINE",
+                          "COLDLINE",
+                          "ARCHIVE",
+                          "DURABLE_REDUCED_AVAILABILITY"
+                        ]
+                      }
+                    },
+                    "matches_suffix": {
+                      "type": "array",
+                      "items": {
+                        "type": "string"
+                      }
+                    },
+                    "noncurrent_time_before": {
+                      "type": "string"
+                    },
+                    "num_newer_versions": {
+                      "type": "number"
+                    },
+                    "with_state": {
+                      "type": "string",
+                      "enum": [
+                        "LIVE",
+                        "ARCHIVED",
+                        "ANY"
+                      ]
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
         "location": {
           "type": "string"
         },
@@ -963,13 +1061,16 @@
           "additionalProperties": false,
           "properties": {
             "retention_period": {
-              "type": "number"
+              "type": "string"
             },
             "is_locked": {
               "type": "boolean"
             }
           }
         },
+        "soft_delete_retention": {
+          "type": "number"
+        },
         "enable_object_retention": {
           "type": "boolean"
         }

modules/project-factory/schemas/project.schema.md

@@ -251,6 +251,33 @@
   - **force_destroy**: *boolean*
   - **labels**: *object*
     <br>*additional properties: string*
+  - **lifecycle_rules**: *object*
+    <br>*additional properties: false*
+    - **`^[a-zA-Z0-9_-]+$`**: *object*
+      <br>*additional properties: false*
+      - ⁺**action**: *object*
+        <br>*additional properties: false*
+        - ⁺**type**: *string*
+          <br>*enum: ['Delete', 'SetStorageClass', 'AbortIncompleteMultipartUpload']*
+        - **storage_class**: *string*
+      - ⁺**condition**: *object*
+        <br>*additional properties: false*
+        - **age**: *number*
+        - **created_before**: *string*
+        - **custom_time_before**: *string*
+        - **days_since_custom_time**: *number*
+        - **days_since_noncurrent_time**: *number*
+        - **matches_prefix**: *array*
+          - items: *string*
+        - **matches_storage_class**: *array*
+          - items: *string*
+            <br>*enum: ['STANDARD', 'MULTI_REGIONAL', 'REGIONAL', 'NEARLINE', 'COLDLINE', 'ARCHIVE', 'DURABLE_REDUCED_AVAILABILITY']*
+        - **matches_suffix**: *array*
+          - items: *string*
+        - **noncurrent_time_before**: *string*
+        - **num_newer_versions**: *number*
+        - **with_state**: *string*
+          <br>*enum: ['LIVE', 'ARCHIVED', 'ANY']*
   - **location**: *string*
   - **managed_folders**: *object*
     <br>*additional properties: false*
@@ -266,8 +293,9 @@
   - **versioning**: *boolean*
   - **retention_policy**: *object*
     <br>*additional properties: false*
-    - **retention_period**: *number*
+    - **retention_period**: *string*
     - **is_locked**: *boolean*
+  - **soft_delete_retention**: *number*
   - **enable_object_retention**: *boolean*
 - **buckets**<a name="refs-buckets"></a>: *object*
   <br>*additional properties: false*

modules/project-factory/variables-projects.tf

@@ -69,6 +69,30 @@ variable "projects" {
             }))
           })), {})
         })), {})
+        lifecycle_rules = optional(map(object({
+          action = object({
+            type          = string
+            storage_class = optional(string)
+          })
+          condition = object({
+            age                        = optional(number)
+            created_before             = optional(string)
+            custom_time_before         = optional(string)
+            days_since_custom_time     = optional(number)
+            days_since_noncurrent_time = optional(number)
+            matches_prefix             = optional(list(string))
+            matches_storage_class      = optional(list(string))
+            matches_suffix             = optional(list(string))
+            noncurrent_time_before     = optional(string)
+            num_newer_versions         = optional(number)
+            with_state                 = optional(string)
+          })
+        })), {})
+        retention_policy = optional(object({
+          retention_period = string
+          is_locked        = optional(bool)
+        }))
+        soft_delete_retention = optional(number)
       }))
       service_accounts = optional(map(object({
         description = optional(string)
@@ -151,6 +175,30 @@ variable "projects" {
           }))
         })), {})
       })), {})
+      lifecycle_rules = optional(map(object({
+        action = object({
+          type          = string
+          storage_class = optional(string)
+        })
+        condition = object({
+          age                        = optional(number)
+          created_before             = optional(string)
+          custom_time_before         = optional(string)
+          days_since_custom_time     = optional(number)
+          days_since_noncurrent_time = optional(number)
+          matches_prefix             = optional(list(string))
+          matches_storage_class      = optional(list(string))
+          matches_suffix             = optional(list(string))
+          noncurrent_time_before     = optional(string)
+          num_newer_versions         = optional(number)
+          with_state                 = optional(string)
+        })
+      })), {})
+      retention_policy = optional(object({
+        retention_period = string
+        is_locked        = optional(bool)
+      }))
+      soft_delete_retention = optional(number)
     })), {})
     contacts = optional(map(list(string)), {})
     datasets = optional(map(object({