Refactor service agent management (#2423)

* Service agents script

* Service agents update

* WIP

* Update script and terraform

* Fix tests

* Fix linter

* Update docs

* Bring back pf example inventory

* Fix tests

* Fix more tests

* Fix tests

* Use dataclasses for build_service_agents.py

* Remove unneeded field() from build_service_agents

* Re-enable CMEK depends_on in project outputs

* Update tools/requirements.txt

* Enable storage in GCS example projects

* Fix tests

* Add CMEK Service Agents dependencies for services

* Fix typos and data platform cmek

* More typos
This commit is contained in:
Julio Castillo
2024-07-23 22:05:38 +02:00
committed by GitHub
parent e49b2d686f
commit c0bf32e797
147 changed files with 3676 additions and 1301 deletions

View File

@@ -205,7 +205,7 @@ module "project-factory" {
projects_data_path = "data/projects"
}
}
# tftest modules=16 resources=56 files=prj-app-1,prj-app-2,prj-app-3,budget-test-100,h-0-0,h-1-0,h-0-1,h-1-1,h-1-1-p0 inventory=example.yaml
# tftest modules=16 resources=70 files=prj-app-1,prj-app-2,prj-app-3,budget-test-100,h-0-0,h-1-0,h-0-1,h-1-1,h-1-1-p0 inventory=example.yaml
```
A simple hierarchy of folders:
@@ -254,7 +254,7 @@ labels:
team: foo
parent: folders/12345678
service_encryption_key_ids:
compute:
storage.googleapis.com:
- projects/kms-central-prj/locations/europe-west3/keyRings/my-keyring/cryptoKeys/europe3-gce
services:
- container.googleapis.com
@@ -298,12 +298,12 @@ services:
- storage.googleapis.com
shared_vpc_service_config:
host_project: foo-host
service_identity_iam:
service_agent_iam:
"roles/vpcaccess.user":
- cloudrun
"roles/container.hostServiceAgentUser":
- container-engine
service_identity_subnet_iam:
service_agent_subnet_iam:
europe-west1/prod-default-ew1:
- cloudservices
- container-engine
@@ -393,7 +393,7 @@ update_rules:
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
| [factories_config](variables.tf#L96) | Path to folder with YAML resource description data files. | <code title="object&#40;&#123;&#10; hierarchy &#61; optional&#40;object&#40;&#123;&#10; folders_data_path &#61; string&#10; parent_ids &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10; &#125;&#41;&#41;&#10; projects_data_path &#61; optional&#40;string&#41;&#10; budgets &#61; optional&#40;object&#40;&#123;&#10; billing_account &#61; string&#10; budgets_data_path &#61; string&#10; notification_channels &#61; optional&#40;map&#40;any&#41;, &#123;&#125;&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | |
| [data_defaults](variables.tf#L17) | Optional default values used when corresponding project data from files are missing. | <code title="object&#40;&#123;&#10; billing_account &#61; optional&#40;string&#41;&#10; contacts &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; labels &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10; metric_scopes &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; parent &#61; optional&#40;string&#41;&#10; prefix &#61; optional&#40;string&#41;&#10; service_encryption_key_ids &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; services &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; shared_vpc_service_config &#61; optional&#40;object&#40;&#123;&#10; host_project &#61; string&#10; network_users &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; service_identity_iam &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; service_identity_subnet_iam &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; service_iam_grants &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; network_subnet_users &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; &#125;&#41;, &#123; host_project &#61; null &#125;&#41;&#10; tag_bindings &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10; service_accounts &#61; optional&#40;map&#40;object&#40;&#123;&#10; display_name &#61; optional&#40;string, &#34;Terraform-managed.&#34;&#41;&#10; iam_self_roles &#61; optional&#40;list&#40;string&#41;&#41;&#10; &#125;&#41;&#41;, &#123;&#125;&#41;&#10; vpc_sc &#61; optional&#40;object&#40;&#123;&#10; perimeter_name &#61; string&#10; perimeter_bridges &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; is_dry_run &#61; optional&#40;bool, false&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>&#123;&#125;</code> |
| [data_defaults](variables.tf#L17) | Optional default values used when corresponding project data from files are missing. | <code title="object&#40;&#123;&#10; billing_account &#61; optional&#40;string&#41;&#10; contacts &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; labels &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10; metric_scopes &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; parent &#61; optional&#40;string&#41;&#10; prefix &#61; optional&#40;string&#41;&#10; service_encryption_key_ids &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; services &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; shared_vpc_service_config &#61; optional&#40;object&#40;&#123;&#10; host_project &#61; string&#10; network_users &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; service_agent_iam &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; service_agent_subnet_iam &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; service_iam_grants &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; network_subnet_users &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; &#125;&#41;, &#123; host_project &#61; null &#125;&#41;&#10; tag_bindings &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10; service_accounts &#61; optional&#40;map&#40;object&#40;&#123;&#10; display_name &#61; optional&#40;string, &#34;Terraform-managed.&#34;&#41;&#10; iam_self_roles &#61; optional&#40;list&#40;string&#41;&#41;&#10; &#125;&#41;&#41;, &#123;&#125;&#41;&#10; vpc_sc &#61; optional&#40;object&#40;&#123;&#10; perimeter_name &#61; string&#10; perimeter_bridges &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; is_dry_run &#61; optional&#40;bool, false&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>&#123;&#125;</code> |
| [data_merges](variables.tf#L52) | Optional values that will be merged with corresponding data from files. Combines with `data_defaults`, file data, and `data_overrides`. | <code title="object&#40;&#123;&#10; contacts &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; labels &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10; metric_scopes &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; service_encryption_key_ids &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; services &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; tag_bindings &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10; service_accounts &#61; optional&#40;map&#40;object&#40;&#123;&#10; display_name &#61; optional&#40;string, &#34;Terraform-managed.&#34;&#41;&#10; iam_self_roles &#61; optional&#40;list&#40;string&#41;&#41;&#10; &#125;&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>&#123;&#125;</code> |
| [data_overrides](variables.tf#L71) | Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`. | <code title="object&#40;&#123;&#10; billing_account &#61; optional&#40;string&#41;&#10; contacts &#61; optional&#40;map&#40;list&#40;string&#41;&#41;&#41;&#10; parent &#61; optional&#40;string&#41;&#10; prefix &#61; optional&#40;string&#41;&#10; service_encryption_key_ids &#61; optional&#40;map&#40;list&#40;string&#41;&#41;&#41;&#10; tag_bindings &#61; optional&#40;map&#40;string&#41;&#41;&#10; services &#61; optional&#40;list&#40;string&#41;&#41;&#10; service_accounts &#61; optional&#40;map&#40;object&#40;&#123;&#10; display_name &#61; optional&#40;string, &#34;Terraform-managed.&#34;&#41;&#10; iam_self_roles &#61; optional&#40;list&#40;string&#41;&#41;&#10; &#125;&#41;&#41;&#41;&#10; vpc_sc &#61; optional&#40;object&#40;&#123;&#10; perimeter_name &#61; string&#10; perimeter_bridges &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; is_dry_run &#61; optional&#40;bool, false&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>&#123;&#125;</code> |
@@ -430,7 +430,7 @@ module "project-factory" {
projects_data_path = "data/projects"
}
}
# tftest modules=4 resources=14 files=test-0,test-1,test-2
# tftest modules=4 resources=22 files=test-0,test-1,test-2
```
```yaml