kovagoadi/hunfabric
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New FAST data platform (#3066)

Browse Source 
* copy from broken dp dev branch

* remove extra excalidraw file

* fix networking yaml

* tfdoc

* tfdoc

* nuke old data platform

* fix tests

* tests

* tflint

* high level diagram

* make location optional in composer schema

* add composer outputs

* docs

* remove schema docs

* tfdoc

* update service agent encryption composer def for composer 3

* encryption keys

* typo

* typo

* fix security IAM

* inventory

* tflint

* Fix roles and diagram.

* Fix tflint

* Fix test DP.

* Fix test

* Diagrams excalidraw gz

---------

Co-authored-by: lcaggio <lorenzo.caggioni@gmail.com>
This commit is contained in:
Ludovico Magnocavallo
2025-05-03 23:21:38 +02:00
committed by GitHub
parent 4c7ff3381e
commit cb7bed50e3
113 changed files with 3307 additions and 4433 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/fast/stages/s0_bootstrap/cicd.yaml
View File

@@ -343,7 +343,7 @@ counts:
google_project_iam_audit_config: 1
google_project_iam_binding: 19
google_project_iam_member: 23
google_project_service: 32
google_project_service: 33
google_project_service_identity: 8
google_service_account: 12
google_service_account_iam_binding: 12
@@ -356,4 +356,4 @@ counts:
google_tags_tag_value: 2
local_file: 13
modules: 26
resources: 287
resources: 288

4
tests/fast/stages/s0_bootstrap/simple.yaml
View File

@@ -28,7 +28,7 @@ counts:
google_project_iam_audit_config: 1
google_project_iam_binding: 19
google_project_iam_member: 17
google_project_service: 32
google_project_service: 33
google_project_service_identity: 8
google_service_account: 6
google_service_account_iam_binding: 6
@@ -41,7 +41,7 @@ counts:
google_tags_tag_value: 2
local_file: 8
modules: 20
resources: 250
resources: 251
outputs:
automation: __missing__

28
tests/fast/stages/s1_resman/simple.yaml
View File

@@ -13,23 +13,23 @@
# limitations under the License.
counts:
google_folder: 14
google_folder_iam_binding: 67
google_folder: 16
google_folder_iam_binding: 74
google_org_policy_policy: 2
google_organization_iam_member: 20
google_project_iam_member: 17
google_service_account: 17
google_service_account_iam_binding: 17
google_storage_bucket: 8
google_storage_bucket_iam_binding: 16
google_storage_bucket_iam_member: 17
google_storage_bucket_object: 19
google_tags_tag_binding: 14
google_organization_iam_member: 21
google_project_iam_member: 19
google_service_account: 19
google_service_account_iam_binding: 19
google_storage_bucket: 9
google_storage_bucket_iam_binding: 18
google_storage_bucket_iam_member: 19
google_storage_bucket_object: 21
google_tags_tag_binding: 16
google_tags_tag_key: 2
google_tags_tag_value: 13
google_tags_tag_value_iam_binding: 4
modules: 40
resources: 247
modules: 45
resources: 272
outputs:
cicd_repositories:
@@ -40,6 +40,8 @@ outputs:
name: cloud-foundation-fabric/1-resman
type: github
service_accounts:
data-platform-dev-ro: fast2-dev-resman-dp-0r@fast2-prod-automation.iam.gserviceaccount.com
data-platform-dev-rw: fast2-dev-resman-dp-0@fast2-prod-automation.iam.gserviceaccount.com
gcve-dev-ro: fast2-dev-resman-gcve-0r@fast2-prod-automation.iam.gserviceaccount.com
gcve-dev-rw: fast2-dev-resman-gcve-0@fast2-prod-automation.iam.gserviceaccount.com
gke-dev-ro: fast2-dev-resman-gke-0r@fast2-prod-automation.iam.gserviceaccount.com

4
tests/fast/stages/s2_security/simple.tfvars
View File

@@ -24,10 +24,6 @@ certificate_authorities = {
location = "europe-west8"
}
}
custom_roles = {
project_iam_viewer = "organizations/123456789012/roles/bar"
service_project_network_admin = "organizations/123456789012/roles/foo"
}
environments = {
dev = {
is_default = false

13
tests/fast/stages/s3_data_platform_dev/__init__.py Normal file
View File

@@ -0,0 +1,13 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

44
tests/fast/stages/s3_data_platform_dev/simple.tfvars Normal file
View File

@@ -0,0 +1,44 @@
automation = {
outputs_bucket = "fast2-prod-iac-core-outputs"
}
billing_account = {
id = "000000-111111-222222"
}
environments = {
dev = {
is_default = false
name = "Development"
short_name = "dev"
tag_name = "development"
}
}
factories_config = {
context = {
iam_principals = {
data-consumer-bi = "group:gcp-consumer-bi@example.com"
dp-product-a-0 = "group:gcp-data-product-a-0@example.com"
}
}
}
folder_ids = {
data-platform-dev = "folders/00000000000000"
}
host_project_ids = {
dev-spoke-0 = "fast2-dev-net-spoke-0"
}
organization = {
domain = "fast.example.com"
id = 123456789012
customer_id = "C00000000"
}
prefix = "fast2"
subnet_self_links = {
dev-spoke-0 = {
"europe-west8/dev-dataplatform" = "projects/fast2-dev-net-spoke-0/regions/europe-west8/subnetworks/dev-dataplatform"
}
}
vpc_self_links = {
dev-spoke-0 = "projects/fast2-dev-net-spoke-0/global/networks/dev-spoke-0"
}

41
tests/fast/stages/s3_data_platform_dev/simple.yaml Normal file
View File

@@ -0,0 +1,41 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
counts:
google_bigquery_dataset: 1
google_bigquery_dataset_iam_binding: 1
google_bigquery_default_service_account: 2
google_composer_environment: 1
google_compute_shared_vpc_service_project: 1
google_data_catalog_policy_tag: 3
google_data_catalog_taxonomy: 1
google_dataplex_aspect_type: 1
google_folder: 2
google_folder_iam_binding: 5
google_project: 3
google_project_iam_binding: 21
google_project_iam_member: 13
google_project_service: 17
google_project_service_identity: 6
google_service_account: 6
google_service_account_iam_binding: 4
google_storage_bucket: 3
google_storage_bucket_iam_binding: 5
google_storage_bucket_object: 5
google_storage_project_service_account: 2
google_tags_location_tag_binding: 2
google_tags_tag_key: 1
google_tags_tag_value: 1
modules: 19
resources: 107

18
tests/fast/stages/s3_data_platform_dev/tftest.yaml Normal file
View File

@@ -0,0 +1,18 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
module: fast/stages/3-data-platform-dev
tests:
simple:

2
tests/fixtures.py
View File

@@ -191,7 +191,7 @@ def plan_validator(module_path, inventory_paths, basedir, tf_var_files=None,
# - put the values coming from user's inventory the right
# side of any comparison operators.
# - include a descriptive error message to the assert
print(yaml.dump({'values': summary.values}))
# print(yaml.dump({'values': summary.values}))
# print("", yaml.dump({'counts': summary.counts}))
if 'values' in inventory: