Skip IAM grants for service agents that are not created on API activation (#3448)

* Skip IAM grants for service agents that are not created on API activation

* Fix tests

modules/project/service-agents.tf

@@ -116,16 +116,7 @@ locals {
       } if alltrue([
         var.service_agents_config.grant_default_roles,
         agent.role != null,
-        # TODO: improve the detection below
+        !agent.skip_iam
-        # this skips IAM role grants to the non-primary agents listed below
-        # as it's failing, possibly because the agents don't exist
-        # after API activation
-        !contains([
-          "apigateway", "apigateway-mgmt", "bigqueryspark", "bigquerytardis",
-          "firebase", "krmapihosting", "krmapihosting-dataplane", "logging",
-          "networkactions", "prod-bigqueryomni", "scc-notification",
-          "securitycenter"
-        ], agent.name)
     ])
   }
   services = [

tests/fast/stages/s0_org_setup/not-simple.yaml

@@ -2776,7 +2776,7 @@ counts:
   google_organization_iam_custom_role: 7
   google_project: 3
   google_project_iam_binding: 16
-  google_project_iam_member: 17
+  google_project_iam_member: 15
   google_project_service: 33
   google_project_service_identity: 9
   google_service_account: 16
@@ -2793,5 +2793,5 @@ counts:
   google_tags_tag_value_iam_binding: 4
   local_file: 9
   modules: 46
-  resources: 310
+  resources: 308
   terraform_data: 2

tests/fast/stages/s3_data_platform_dev/simple.yaml

@@ -24,7 +24,7 @@ counts:
   google_folder_iam_binding: 3
   google_project: 3
   google_project_iam_binding: 23
-  google_project_iam_member: 15
+  google_project_iam_member: 12
   google_project_service: 18
   google_project_service_identity: 6
   google_service_account: 6
@@ -37,4 +37,4 @@ counts:
   google_tags_tag_key: 1
   google_tags_tag_value: 1
   modules: 19
-  resources: 112
+  resources: 109

tests/modules/cloud_run_v2/examples/service-vpc-access-connector-create-sharedvpc.yaml

@@ -53,6 +53,6 @@ counts:
   google_cloud_run_v2_service: 1
   google_vpc_access_connector: 1
   modules: 4
-  resources: 60
+  resources: 59
 
 outputs: {}

tests/modules/project/examples/data.yaml

@@ -516,7 +516,7 @@ counts:
   google_project: 2
   google_project_iam_audit_config: 2
   google_project_iam_binding: 7
-  google_project_iam_member: 15
+  google_project_iam_member: 14
   google_project_service: 7
   google_project_service_identity: 3
   google_pubsub_topic: 1
@@ -525,7 +525,6 @@ counts:
   google_storage_bucket_iam_member: 1
   google_storage_project_service_account: 1
   modules: 8
-  resources: 64
+  resources: 63
-
 
 outputs: {}

tools/build_service_agents.py

@@ -53,9 +53,26 @@ ALIASES = {
     'serverless-robot-prod': ['cloudrun', 'run'],
 }
 
-IGNORED_AGENTS = [
+IGNORED_AGENTS = []
-    # gcp-sa-ns-authz agent gets created on first create op
+
-    'service-PROJECT_NUMBER@gcp-sa-ns-authz.iam.gserviceaccount.com'
+SKIP_IAM_AGENTS = [
+    # skips IAM role grants to the non-primary agents listed below as
+    # it's failing, possibly because the agents don't exist after API
+    # activation
+    'service-PROJECT_NUMBER@gcp-sa-apigateway-mgmt.iam.gserviceaccount.com',
+    'service-PROJECT_NUMBER@gcp-sa-apigateway.iam.gserviceaccount.com',
+    'service-PROJECT_NUMBER@gcp-sa-bigqueryspark.iam.gserviceaccount.com',
+    'service-PROJECT_NUMBER@gcp-sa-bigquerytardis.iam.gserviceaccount.com',
+    'service-PROJECT_NUMBER@gcp-sa-connectedsheets.iam.gserviceaccount.com',
+    'service-PROJECT_NUMBER@gcp-sa-firebase.iam.gserviceaccount.com',
+    'service-PROJECT_NUMBER@gcp-sa-krmapihosting-dataplane.iam.gserviceaccount.com',
+    'service-PROJECT_NUMBER@gcp-sa-krmapihosting.iam.gserviceaccount.com',
+    'service-PROJECT_NUMBER@gcp-sa-logging.iam.gserviceaccount.com',
+    'service-PROJECT_NUMBER@gcp-sa-networkactions.iam.gserviceaccount.com',
+    'service-PROJECT_NUMBER@gcp-sa-prod-bigqueryomni.iam.gserviceaccount.com',
+    'service-PROJECT_NUMBER@gcp-sa-scc-notification.iam.gserviceaccount.com',
+    'service-PROJECT_NUMBER@gcp-sa-securitycenter.iam.gserviceaccount.com',
+    'service-PROJECT_NUMBER@gcp-sa-ns-authz.iam.gserviceaccount.com',
 ]
 
 AGENT_NAME_OVERRIDE = {
@@ -115,6 +132,7 @@ class Agent:
   role: str
   is_primary: bool
   aliases: list[str]
+  skip_iam: bool
 
 
 @click.command()
@@ -173,6 +191,8 @@ def main(mode, e2e=False):
       name = identity.split('@')[1].split('.')[0]
       name = name.removeprefix('gcp-sa-')
 
+    skip_iam = identity in SKIP_IAM_AGENTS
+
     # Replace identifiers based on mode
     if mode == 'project':
       identity = identity.replace('PROJECT_NUMBER', '${project_number}')
@@ -200,6 +220,7 @@ def main(mode, e2e=False):
         role=col2.code.get_text() if 'roles/' in agent_text else None,
         is_primary=PRIMARY_OVERRIDE.get(name, is_primary),
         aliases=ALIASES.get(name, []),
+        skip_iam=skip_iam,
     )
 
     if mode == 'project' and agent.name == 'cloudservices':