Project factory additions, project module reuse implementation (#2899)

* add support for buckets

* add project-level interpolation for own SAs

* docs

* project reuse changes

* fix example

* tfdoc

* update check documentation tool

* fast tests

* blueprints

* typo

tests/fast/addons/a1_resman_tenants/simple.yaml

@@ -25,7 +25,7 @@ counts:
   google_logging_project_bucket_config: 4
   google_org_policy_policy: 6
   google_organization_iam_member: 6
-  google_project: 6
+  google_project: 4
   google_project_iam_audit_config: 2
   google_project_iam_binding: 32
   google_project_iam_member: 34
@@ -43,4 +43,4 @@ counts:
   google_tags_tag_key: 1
   google_tags_tag_value: 4
   modules: 50
-  resources: 291
+  resources: 289

tests/modules/project/examples/data.yaml

@@ -26,11 +26,16 @@ values:
     auto_create_network: false
     billing_account: 123456-123456-123456
     deletion_policy: DELETE
+    effective_labels:
+      goog-terraform-provisioned: 'true'
     folder_id: '1122334455'
     labels: null
     name: test-project
     org_id: null
     project_id: test-project
+    tags: null
+    terraform_labels:
+      goog-terraform-provisioned: 'true'
     timeouts: null
   module.dataset.google_bigquery_dataset.default:
     dataset_id: bq_sink
@@ -39,6 +44,8 @@ values:
     default_table_expiration_ms: null
     delete_contents_on_destroy: true
     description: Terraform managed.
+    effective_labels:
+      goog-terraform-provisioned: 'true'
     external_dataset_reference: []
     friendly_name: null
     labels: null
@@ -46,15 +53,20 @@ values:
     max_time_travel_hours: '168'
     project: project-id
     resource_tags: null
+    terraform_labels:
+      goog-terraform-provisioned: 'true'
     timeouts: null
   module.gcs.google_storage_bucket.bucket[0]:
     autoclass: []
     cors: []
     custom_placement_config: []
     default_event_based_hold: null
+    effective_labels:
+      goog-terraform-provisioned: 'true'
     enable_object_retention: null
     encryption: []
     force_destroy: true
+    hierarchical_namespace: []
     labels: null
     lifecycle_rule: []
     location: EU
@@ -64,6 +76,8 @@ values:
     requester_pays: null
     retention_policy: []
     storage_class: STANDARD
+    terraform_labels:
+      goog-terraform-provisioned: 'true'
     timeouts: null
     uniform_bucket_level_access: true
   module.host-project.google_compute_shared_vpc_host_project.shared_vpc_host[0]:
@@ -73,18 +87,27 @@ values:
     auto_create_network: false
     billing_account: 123456-123456-123456
     deletion_policy: DELETE
+    effective_labels:
+      goog-terraform-provisioned: 'true'
     folder_id: '1122334455'
     labels: null
     name: test-host
     org_id: null
     project_id: test-host
+    tags: null
+    terraform_labels:
+      goog-terraform-provisioned: 'true'
     timeouts: null
   module.kms.google_kms_crypto_key.default["key-global"]:
+    effective_labels:
+      goog-terraform-provisioned: 'true'
     labels: null
     name: key-global
     purpose: ENCRYPT_DECRYPT
     rotation_period: null
     skip_initial_version_creation: false
+    terraform_labels:
+      goog-terraform-provisioned: 'true'
     timeouts: null
   module.kms.google_kms_key_ring.default[0]:
     location: global
@@ -95,11 +118,9 @@ values:
     condition: []
     role: roles/cloudkms.cryptoKeyEncrypterDecrypter
   module.project.data.google_bigquery_default_service_account.bq_sa[0]:
-    project: test-project
-  module.project.data.google_project.project[0]:
-    project_id: test-project
+    project: test-test-project
   module.project.data.google_storage_project_service_account.gcs_sa[0]:
-    project: test-project
+    project: test-test-project
     user_project: null
   module.project.google_bigquery_dataset_iam_member.bq-sinks-binding["info"]:
     condition: []
@@ -107,7 +128,7 @@ values:
   module.project.google_compute_shared_vpc_service_project.shared_vpc_service[0]:
     deletion_policy: null
     host_project: test-host
-    service_project: test-project
+    service_project: test-test-project
     timeouts: null
   module.project.google_kms_crypto_key_iam_member.service_agent_cmek["key-0.compute-system"]:
     condition: []
@@ -120,7 +141,7 @@ values:
     disabled: null
     filter: resource.type=gce_instance
     name: no-gce-instances
-    project: test-project
+    project: test-test-project
   module.project.google_logging_project_sink.sink["debug"]:
     custom_writer_identity: null
     description: debug (Terraform-managed).
@@ -132,7 +153,7 @@ values:
       name: no-compute
     filter: severity=DEBUG
     name: debug
-    project: test-project
+    project: test-test-project
     unique_writer_identity: true
   module.project.google_logging_project_sink.sink["info"]:
     bigquery_options:
@@ -143,7 +164,7 @@ values:
     exclusions: []
     filter: severity=INFO
     name: info
-    project: test-project
+    project: test-test-project
     unique_writer_identity: true
   module.project.google_logging_project_sink.sink["notice"]:
     custom_writer_identity: null
@@ -153,7 +174,7 @@ values:
     exclusions: []
     filter: severity=NOTICE
     name: notice
-    project: test-project
+    project: test-test-project
     unique_writer_identity: true
   module.project.google_logging_project_sink.sink["warnings"]:
     custom_writer_identity: null
@@ -163,12 +184,12 @@ values:
     exclusions: []
     filter: severity=WARNING
     name: warnings
-    project: test-project
+    project: test-test-project
     unique_writer_identity: true
   module.project.google_org_policy_policy.default["compute.disableGuestAttributesAccess"]:
     dry_run_spec: []
-    name: projects/test-project/policies/compute.disableGuestAttributesAccess
-    parent: projects/test-project
+    name: projects/test-test-project/policies/compute.disableGuestAttributesAccess
+    parent: projects/test-test-project
     spec:
     - inherit_from_parent: null
       reset: null
@@ -177,12 +198,13 @@ values:
         condition: []
         deny_all: null
         enforce: 'TRUE'
+        parameters: null
         values: []
     timeouts: null
   module.project.google_org_policy_policy.default["compute.skipDefaultNetworkCreation"]:
     dry_run_spec: []
-    name: projects/test-project/policies/compute.skipDefaultNetworkCreation
-    parent: projects/test-project
+    name: projects/test-test-project/policies/compute.skipDefaultNetworkCreation
+    parent: projects/test-test-project
     spec:
     - inherit_from_parent: null
       reset: null
@@ -191,12 +213,13 @@ values:
         condition: []
         deny_all: null
         enforce: 'TRUE'
+        parameters: null
         values: []
     timeouts: null
   module.project.google_org_policy_policy.default["compute.trustedImageProjects"]:
     dry_run_spec: []
-    name: projects/test-project/policies/compute.trustedImageProjects
-    parent: projects/test-project
+    name: projects/test-test-project/policies/compute.trustedImageProjects
+    parent: projects/test-test-project
     spec:
     - inherit_from_parent: null
       reset: null
@@ -205,6 +228,7 @@ values:
         condition: []
         deny_all: null
         enforce: null
+        parameters: null
         values:
         - allowed_values:
           - projects/my-project
@@ -212,8 +236,8 @@ values:
     timeouts: null
   module.project.google_org_policy_policy.default["compute.vmExternalIpAccess"]:
     dry_run_spec: []
-    name: projects/test-project/policies/compute.vmExternalIpAccess
-    parent: projects/test-project
+    name: projects/test-test-project/policies/compute.vmExternalIpAccess
+    parent: projects/test-test-project
     spec:
     - inherit_from_parent: null
       reset: null
@@ -222,12 +246,13 @@ values:
         condition: []
         deny_all: 'TRUE'
         enforce: null
+        parameters: null
         values: []
     timeouts: null
   module.project.google_org_policy_policy.default["iam.allowedPolicyMemberDomains"]:
     dry_run_spec: []
-    name: projects/test-project/policies/iam.allowedPolicyMemberDomains
-    parent: projects/test-project
+    name: projects/test-test-project/policies/iam.allowedPolicyMemberDomains
+    parent: projects/test-test-project
     spec:
     - inherit_from_parent: null
       reset: null
@@ -236,6 +261,7 @@ values:
         condition: []
         deny_all: null
         enforce: null
+        parameters: null
         values:
         - allowed_values:
           - C0xxxxxxx
@@ -244,8 +270,8 @@ values:
     timeouts: null
   module.project.google_org_policy_policy.default["iam.disableServiceAccountKeyCreation"]:
     dry_run_spec: []
-    name: projects/test-project/policies/iam.disableServiceAccountKeyCreation
-    parent: projects/test-project
+    name: projects/test-test-project/policies/iam.disableServiceAccountKeyCreation
+    parent: projects/test-test-project
     spec:
     - inherit_from_parent: null
       reset: null
@@ -254,12 +280,13 @@ values:
         condition: []
         deny_all: null
         enforce: 'TRUE'
+        parameters: null
         values: []
     timeouts: null
   module.project.google_org_policy_policy.default["iam.disableServiceAccountKeyUpload"]:
     dry_run_spec: []
-    name: projects/test-project/policies/iam.disableServiceAccountKeyUpload
-    parent: projects/test-project
+    name: projects/test-test-project/policies/iam.disableServiceAccountKeyUpload
+    parent: projects/test-test-project
     spec:
     - inherit_from_parent: null
       reset: null
@@ -272,11 +299,13 @@ values:
           title: condition
         deny_all: null
         enforce: 'TRUE'
+        parameters: null
         values: []
       - allow_all: null
         condition: []
         deny_all: null
         enforce: 'FALSE'
+        parameters: null
         values: []
     timeouts: null
   module.project.google_project_iam_audit_config.default["allServices"]:
@@ -284,7 +313,7 @@ values:
     - exempted_members:
       - group:organization-admins@example.org
       log_type: ADMIN_READ
-    project: test-project
+    project: test-test-project
     service: allServices
   module.project.google_project_iam_audit_config.default["storage.googleapis.com"]:
     audit_log_config:
@@ -292,39 +321,39 @@ values:
       log_type: DATA_READ
     - exempted_members: []
       log_type: DATA_WRITE
-    project: test-project
+    project: test-test-project
     service: storage.googleapis.com
   module.project.google_project_iam_binding.authoritative["roles/apigee.serviceAgent"]:
     condition: []
-    project: test-project
+    project: test-test-project
     role: roles/apigee.serviceAgent
   module.project.google_project_iam_binding.authoritative["roles/cloudasset.owner"]:
     condition: []
     members:
     - group:organization-admins@example.org
-    project: test-project
+    project: test-test-project
     role: roles/cloudasset.owner
   module.project.google_project_iam_binding.authoritative["roles/cloudsupport.techSupportEditor"]:
     condition: []
     members:
     - group:organization-admins@example.org
-    project: test-project
+    project: test-test-project
     role: roles/cloudsupport.techSupportEditor
   module.project.google_project_iam_binding.authoritative["roles/editor"]:
     condition: []
-    project: test-project
+    project: test-test-project
     role: roles/editor
   module.project.google_project_iam_binding.authoritative["roles/iam.securityReviewer"]:
     condition: []
     members:
     - group:organization-admins@example.org
-    project: test-project
+    project: test-test-project
     role: roles/iam.securityReviewer
   module.project.google_project_iam_binding.authoritative["roles/logging.admin"]:
     condition: []
     members:
     - group:organization-admins@example.org
-    project: test-project
+    project: test-test-project
     role: roles/logging.admin
   module.project.google_project_iam_binding.bindings["iam_admin_conditional"]:
     condition:
@@ -334,12 +363,12 @@ values:
       title: delegated_network_user_one
     members:
     - group:organization-admins@example.org
-    project: test-project
+    project: test-test-project
     role: roles/resourcemanager.projectIamAdmin
   module.project.google_project_iam_member.bindings["group-owner"]:
     condition: []
     member: group:organization-admins@example.org
-    project: test-project
+    project: test-test-project
     role: roles/owner
   module.project.google_project_iam_member.bucket-sinks-binding["debug"]:
     condition:
@@ -347,23 +376,23 @@ values:
     role: roles/logging.bucketWriter
   module.project.google_project_iam_member.service_agents["apigee"]:
     condition: []
-    project: test-project
+    project: test-test-project
     role: roles/apigee.serviceAgent
   module.project.google_project_iam_member.service_agents["compute-system"]:
     condition: []
-    project: test-project
+    project: test-test-project
     role: roles/compute.serviceAgent
   module.project.google_project_iam_member.service_agents["container-engine-robot"]:
     condition: []
-    project: test-project
+    project: test-test-project
     role: roles/container.serviceAgent
   module.project.google_project_iam_member.service_agents["gkenode"]:
     condition: []
-    project: test-project
+    project: test-test-project
     role: roles/container.defaultNodeServiceAgent
   module.project.google_project_iam_member.service_agents["serverless-robot-prod"]:
     condition: []
-    project: test-project
+    project: test-test-project
     role: roles/run.serviceAgent
   module.project.google_project_iam_member.shared_vpc_host_robots["roles/cloudasset.owner:cloudservices"]:
     condition: []
@@ -396,55 +425,55 @@ values:
   module.project.google_project_service.project_services["apigee.googleapis.com"]:
     disable_dependent_services: false
     disable_on_destroy: false
-    project: test-project
+    project: test-test-project
     service: apigee.googleapis.com
     timeouts: null
   module.project.google_project_service.project_services["bigquery.googleapis.com"]:
     disable_dependent_services: false
     disable_on_destroy: false
-    project: test-project
+    project: test-test-project
     service: bigquery.googleapis.com
     timeouts: null
   module.project.google_project_service.project_services["compute.googleapis.com"]:
     disable_dependent_services: false
     disable_on_destroy: false
-    project: test-project
+    project: test-test-project
     service: compute.googleapis.com
     timeouts: null
   module.project.google_project_service.project_services["container.googleapis.com"]:
     disable_dependent_services: false
     disable_on_destroy: false
-    project: test-project
+    project: test-test-project
     service: container.googleapis.com
     timeouts: null
   module.project.google_project_service.project_services["logging.googleapis.com"]:
     disable_dependent_services: false
     disable_on_destroy: false
-    project: test-project
+    project: test-test-project
     service: logging.googleapis.com
     timeouts: null
   module.project.google_project_service.project_services["run.googleapis.com"]:
     disable_dependent_services: false
     disable_on_destroy: false
-    project: test-project
+    project: test-test-project
     service: run.googleapis.com
     timeouts: null
   module.project.google_project_service.project_services["storage.googleapis.com"]:
     disable_dependent_services: false
     disable_on_destroy: false
-    project: test-project
+    project: test-test-project
     service: storage.googleapis.com
     timeouts: null
   module.project.google_project_service_identity.default["apigee.googleapis.com"]:
-    project: test-project
+    project: test-test-project
     service: apigee.googleapis.com
     timeouts: null
   module.project.google_project_service_identity.default["container.googleapis.com"]:
-    project: test-project
+    project: test-test-project
     service: container.googleapis.com
     timeouts: null
   module.project.google_project_service_identity.default["run.googleapis.com"]:
-    project: test-project
+    project: test-test-project
     service: run.googleapis.com
     timeouts: null
   module.project.google_pubsub_topic_iam_member.pubsub-sinks-binding["notice"]:
@@ -457,12 +486,17 @@ values:
     condition: []
     role: roles/storage.objectCreator
   module.pubsub.google_pubsub_topic.default:
+    effective_labels:
+      goog-terraform-provisioned: 'true'
     ingestion_data_source_settings: []
     kms_key_name: null
     labels: null
     message_retention_duration: null
     name: pubsub_sink
     project: project-id
+    schema_settings: []
+    terraform_labels:
+      goog-terraform-provisioned: 'true'
     timeouts: null
 
 counts:
@@ -479,7 +513,7 @@ counts:
   google_logging_project_exclusion: 1
   google_logging_project_sink: 4
   google_org_policy_policy: 7
-  google_project: 3
+  google_project: 2
   google_project_iam_audit_config: 2
   google_project_iam_binding: 7
   google_project_iam_member: 14
@@ -491,7 +525,7 @@ counts:
   google_storage_bucket_iam_member: 1
   google_storage_project_service_account: 1
   modules: 8
-  resources: 64
+  resources: 63
 
 
 outputs: {}

tests/modules/project_factory/examples/example.yaml

@@ -12,15 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+values:
 values:
   module.project-factory.module.automation-buckets["dev-tb-app0-0/state"].google_storage_bucket.bucket[0]:
     autoclass: []
     cors: []
     custom_placement_config: []
     default_event_based_hold: null
+    effective_labels:
+      goog-terraform-provisioned: 'true'
     enable_object_retention: null
     encryption: []
     force_destroy: false
+    hierarchical_namespace: []
     labels: null
     lifecycle_rule: []
     location: EU
@@ -30,8 +34,12 @@ values:
     requester_pays: null
     retention_policy: []
     storage_class: STANDARD
+    terraform_labels:
+      goog-terraform-provisioned: 'true'
     timeouts: null
     uniform_bucket_level_access: true
+    versioning:
+    - enabled: false
   ? module.project-factory.module.automation-buckets["dev-tb-app0-0/state"].google_storage_bucket_iam_binding.authoritative["roles/storage.objectCreator"]
   : bucket: test-pf-dev-tb-app0-0-state
     condition: []
@@ -53,6 +61,8 @@ values:
     description: Team B app 0 read-only automation sa.
     disabled: false
     display_name: Service account ro for dev-tb-app0-0.
+    email: test-pf-dev-tb-app0-0-ro@test-pf-teams-iac-0.iam.gserviceaccount.com
+    member: serviceAccount:test-pf-dev-tb-app0-0-ro@test-pf-teams-iac-0.iam.gserviceaccount.com
     project: test-pf-teams-iac-0
     timeouts: null
   module.project-factory.module.automation-service-accounts["dev-tb-app0-0/rw"].google_service_account.service_account[0]:
@@ -61,6 +71,8 @@ values:
     description: Team B app 0 read/write automation sa.
     disabled: false
     display_name: Service account rw for dev-tb-app0-0.
+    email: test-pf-dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com
+    member: serviceAccount:test-pf-dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com
     project: test-pf-teams-iac-0
     timeouts: null
   module.project-factory.module.billing-account[0].google_billing_budget.default["test-100"]:
@@ -102,8 +114,10 @@ values:
     type: email
     user_labels: null
   module.project-factory.module.hierarchy-folder-lvl-1["team-a"].google_folder.folder[0]:
+    deletion_protection: false
     display_name: Team A
     parent: folders/5678901234
+    tags: null
     timeouts: null
   module.project-factory.module.hierarchy-folder-lvl-1["team-a"].google_folder_iam_binding.authoritative["roles/viewer"]:
     condition: []
@@ -112,22 +126,42 @@ values:
     - group:team-a-admins@example.org
     role: roles/viewer
   module.project-factory.module.hierarchy-folder-lvl-1["team-b"].google_folder.folder[0]:
+    deletion_protection: false
     display_name: Team B
     parent: folders/5678901234
+    tags: null
     timeouts: null
   module.project-factory.module.hierarchy-folder-lvl-1["team-c"].google_folder.folder[0]:
+    deletion_protection: false
     display_name: Team C
     parent: folders/5678901234
+    tags: null
     timeouts: null
   module.project-factory.module.hierarchy-folder-lvl-2["team-a/app-0"].google_folder.folder[0]:
+    deletion_protection: false
     display_name: App 0
+    tags: null
     timeouts: null
   module.project-factory.module.hierarchy-folder-lvl-2["team-b/app-0"].google_folder.folder[0]:
+    deletion_protection: false
     display_name: App 0
+    tags: null
     timeouts: null
   module.project-factory.module.hierarchy-folder-lvl-2["team-b/app-0"].google_tags_tag_binding.binding["drs-allow-all"]:
     tag_value: tagValues/123456
     timeouts: null
+  module.project-factory.module.projects-iam["dev-tb-app0-0"].google_project_iam_binding.authoritative["roles/owner"]:
+    condition: []
+    members:
+    - serviceAccount:test-pf-dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com
+    project: test-pf-dev-tb-app0-0
+    role: roles/owner
+  module.project-factory.module.projects-iam["dev-tb-app0-0"].google_project_iam_binding.authoritative["roles/viewer"]:
+    condition: []
+    members:
+    - serviceAccount:test-pf-dev-tb-app0-0-ro@test-pf-teams-iac-0.iam.gserviceaccount.com
+    project: test-pf-dev-tb-app0-0
+    role: roles/viewer
   module.project-factory.module.projects["dev-ta-app0-be"].data.google_storage_project_service_account.gcs_sa[0]:
     project: test-pf-dev-ta-app0-be
     user_project: null
@@ -154,6 +188,7 @@ values:
     effective_labels:
       app: app-0
       environment: test
+      goog-terraform-provisioned: 'true'
       team: team-a
     labels:
       app: app-0
@@ -161,9 +196,11 @@ values:
       team: team-a
     name: test-pf-dev-ta-app0-be
     project_id: test-pf-dev-ta-app0-be
+    tags: null
     terraform_labels:
       app: app-0
       environment: test
+      goog-terraform-provisioned: 'true'
       team: team-a
     timeouts: null
   ? module.project-factory.module.projects["dev-ta-app0-be"].google_project_iam_member.service_agents["container-engine-robot"]
@@ -228,25 +265,16 @@ values:
     deletion_policy: DELETE
     effective_labels:
       environment: test
+      goog-terraform-provisioned: 'true'
     labels:
       environment: test
     name: test-pf-dev-tb-app0-0
     project_id: test-pf-dev-tb-app0-0
+    tags: null
     terraform_labels:
       environment: test
+      goog-terraform-provisioned: 'true'
     timeouts: null
-  module.project-factory.module.projects["dev-tb-app0-0"].google_project_iam_binding.authoritative["roles/owner"]:
-    condition: []
-    members:
-    - serviceAccount:test-pf-dev-tb-app0-0-rw@test-pf-teams-iac-0.iam.gserviceaccount.com
-    project: test-pf-dev-tb-app0-0
-    role: roles/owner
-  module.project-factory.module.projects["dev-tb-app0-0"].google_project_iam_binding.authoritative["roles/viewer"]:
-    condition: []
-    members:
-    - serviceAccount:test-pf-dev-tb-app0-0-ro@test-pf-teams-iac-0.iam.gserviceaccount.com
-    project: test-pf-dev-tb-app0-0
-    role: roles/viewer
   module.project-factory.module.projects["dev-tb-app0-0"].google_project_iam_member.service_agents["serverless-robot-prod"]:
     condition: []
     project: test-pf-dev-tb-app0-0
@@ -289,14 +317,17 @@ values:
     deletion_policy: DELETE
     effective_labels:
       environment: test
+      goog-terraform-provisioned: 'true'
     folder_id: '5678901234'
     labels:
       environment: test
     name: test-pf-teams-iac-0
     org_id: null
     project_id: test-pf-teams-iac-0
+    tags: null
     terraform_labels:
       environment: test
+      goog-terraform-provisioned: 'true'
     timeouts: null
   module.project-factory.module.projects["teams-iac-0"].google_project_iam_member.service_agents["container-engine-robot"]:
     condition: []
@@ -346,6 +377,8 @@ values:
     description: null
     disabled: false
     display_name: Backend instances.
+    email: app-0-be@test-pf-dev-ta-app0-be.iam.gserviceaccount.com
+    member: serviceAccount:app-0-be@test-pf-dev-ta-app0-be.iam.gserviceaccount.com
     project: test-pf-dev-ta-app0-be
     timeouts: null
   ? module.project-factory.module.service-accounts["dev-ta-app0-be/app-0-fe"].google_project_iam_member.project-roles["test-pf-dev-net-spoke-0-roles/compute.networkUser"]
@@ -366,6 +399,8 @@ values:
     description: null
     disabled: false
     display_name: Frontend instances.
+    email: app-0-fe@test-pf-dev-ta-app0-be.iam.gserviceaccount.com
+    member: serviceAccount:app-0-fe@test-pf-dev-ta-app0-be.iam.gserviceaccount.com
     project: test-pf-dev-ta-app0-be
     timeouts: null
 
@@ -388,5 +423,5 @@ counts:
   google_storage_bucket_iam_binding: 2
   google_storage_project_service_account: 3
   google_tags_tag_binding: 1
-  modules: 15
+  modules: 16
   resources: 56