diff --git a/fast/stages/3-data-platform-dev/README.md b/fast/stages/3-data-platform-dev/README.md index 7188028de..99a4865e7 100644 --- a/fast/stages/3-data-platform-dev/README.md +++ b/fast/stages/3-data-platform-dev/README.md @@ -233,18 +233,18 @@ The following table lists the available substitutions. | [environments](variables-fast.tf#L34) | Environment names. | object({…}) | ✓ | | 1-resman | | [prefix](variables-fast.tf#L69) | Prefix used for resources that need unique names. Use a maximum of 9 chars for organizations, and 11 chars for tenants. | string | ✓ | | 0-bootstrap | | [aspect_types](variables.tf#L17) | Aspect templates. Merged with those defined via the factory. | map(object({…})) | | {} | | -| [central_project_config](variables.tf#L48) | Configuration for the top-level central project. | object({…}) | | {} | | -| [encryption_keys](variables.tf#L84) | Default encryption keys for services, in service => { region => key id } format. Overridable on a per-object basis. | object({…}) | | {} | | -| [exposure_config](variables.tf#L95) | Data exposure configuration. | object({…}) | | {} | | -| [factories_config](variables.tf#L113) | Configuration for the resource factories. | object({…}) | | {} | | +| [central_project_config](variables.tf#L48) | Configuration for the top-level central project. | object({…}) | | {} | | +| [encryption_keys](variables.tf#L85) | Default encryption keys for services, in service => { region => key id } format. Overridable on a per-object basis. | object({…}) | | {} | | +| [exposure_config](variables.tf#L96) | Data exposure configuration. | object({…}) | | {} | | +| [factories_config](variables.tf#L114) | Configuration for the resource factories. | object({…}) | | {} | | | [folder_ids](variables-fast.tf#L45) | Folder name => id mappings. | map(string) | | {} | 1-resman | | [host_project_ids](variables-fast.tf#L53) | Shared VPC host project name => id mappings. | map(string) | | {} | 2-networking | | [kms_keys](variables-fast.tf#L61) | KMS key ids. | map(string) | | {} | 2-security | -| [location](variables.tf#L128) | Default location used when no location is specified. | string | | "europe-west1" | | -| [outputs_location](variables.tf#L135) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | string | | null | | +| [location](variables.tf#L129) | Default location used when no location is specified. | string | | "europe-west1" | | +| [outputs_location](variables.tf#L136) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | string | | null | | | [regions](variables-fast.tf#L79) | Region mappings. | map(string) | | {} | 2-networking | -| [secure_tags](variables.tf#L141) | Resource manager tags created in the central project. | map(object({…})) | | {} | | -| [stage_config](variables.tf#L162) | Stage configuration used to find environment and resource ids, and to generate names. | object({…}) | | {…} | | +| [secure_tags](variables.tf#L142) | Resource manager tags created in the central project. | map(object({…})) | | {} | | +| [stage_config](variables.tf#L163) | Stage configuration used to find environment and resource ids, and to generate names. | object({…}) | | {…} | | | [subnet_self_links](variables-fast.tf#L87) | Subnet VPC name => { name => self link } mappings. | map(map(string)) | | {} | 2-networking | | [tag_values](variables-fast.tf#L95) | FAST-managed resource manager tag values. | map(string) | | {} | 1-resman | | [vpc_self_links](variables-fast.tf#L103) | Shared VPC name => self link mappings. | map(string) | | {} | 2-networking | diff --git a/fast/stages/3-data-platform-dev/data-domains-composer.tf b/fast/stages/3-data-platform-dev/data-domains-composer.tf index 5d4d2df40..5403ba906 100644 --- a/fast/stages/3-data-platform-dev/data-domains-composer.tf +++ b/fast/stages/3-data-platform-dev/data-domains-composer.tf @@ -20,10 +20,6 @@ locals { { region = var.location, short_name = v.short_name }, try(v.deploy_config.composer, {}) ) - if( - try(v.deploy_config.composer.node_config.network, null) != null && - try(v.deploy_config.composer.node_config.subnetwork, null) != null - ) } dd_composer_keys = { for k, v in local.dd_composer : k => try( @@ -74,12 +70,12 @@ resource "google_composer_environment" "default" { network = try( var.vpc_self_links[each.value.node_config.network], each.value.node_config.network, - "-" + null ) subnetwork = try( var.subnet_self_links[each.value.node_config.network][each.value.node_config.subnetwork], each.value.node_config.subnetwork, - "-" + null ) } software_config { diff --git a/fast/stages/3-data-platform-dev/data-domains.tf b/fast/stages/3-data-platform-dev/data-domains.tf index 64837b061..70fae18df 100644 --- a/fast/stages/3-data-platform-dev/data-domains.tf +++ b/fast/stages/3-data-platform-dev/data-domains.tf @@ -180,7 +180,7 @@ module "dd-projects-iam" { ) iam_by_principals = { for k, v in each.value.project_config.iam_by_principals : - lookup(var.factories_config.context.iam_by_principals, k, k) => v + lookup(var.factories_config.context.iam_principals, k, k) => v } shared_vpc_service_config = ( each.value.project_config.shared_vpc_service_config == null diff --git a/fast/stages/3-data-platform-dev/data/data-domains/domain-0/_config.yaml b/fast/stages/3-data-platform-dev/data/data-domains/domain-0/_config.yaml index 1dad385b7..67b688a62 100644 --- a/fast/stages/3-data-platform-dev/data/data-domains/domain-0/_config.yaml +++ b/fast/stages/3-data-platform-dev/data/data-domains/domain-0/_config.yaml @@ -23,11 +23,12 @@ automation: deploy_config: composer: + {} + # Uncomment for VPC Network Connectivity # region defaults to var.location - # encryption_key: composer-dev-europe-west8 - node_config: - network: dev-spoke-0 - subnetwork: europe-west8/dev-dataplatform + # node_config: + # network: dev-spoke-0 + # subnetwork: europe-west8/dev-dataplatform project_config: iam: @@ -37,8 +38,16 @@ project_config: - ro roles/composer.environmentAndStorageObjectAdmin: - dp-product-a-0 - roles/monitoring.viewer: - - dp-product-a-0 + iam_by_principals: + dp-product-a-0: + - roles/composer.environmentAndStorageObjectAdmin + - roles/monitoring.viewer + - roles/logging.viewer + dp-domain-a: + - roles/composer.environmentAndStorageObjectAdmin + - roles/monitoring.viewer + - roles/logging.viewer + services: - composer.googleapis.com - datacatalog.googleapis.com @@ -55,14 +64,16 @@ folder_config: bigquery_metadata_viewer: members: - data-consumer-bi - role: roles/dataplex.catalogViewer #roles/bigquery.metadataViewer + role: roles/bigquery.metadataViewer + condition: + title: exposure + description: Expose via secure tag. + expression: resource.matchTag('exposure', 'allow') + dataplex_catalog_viewer: + members: + - data-consumer-bi + role: roles/dataplex.catalogViewer condition: title: exposure description: Expose via secure tag. expression: resource.matchTag('exposure', 'allow') - iam_by_principals: - data-consumer-bi: - - roles/datalineage.viewer - dp-product-a-0: - - "roles/logging.viewer" - - "roles/monitoring.viewer" diff --git a/fast/stages/3-data-platform-dev/data/data-domains/domain-0/product-0.yaml b/fast/stages/3-data-platform-dev/data/data-domains/domain-0/product-0.yaml index efbcfef45..7c268bbc4 100644 --- a/fast/stages/3-data-platform-dev/data/data-domains/domain-0/product-0.yaml +++ b/fast/stages/3-data-platform-dev/data/data-domains/domain-0/product-0.yaml @@ -15,6 +15,7 @@ # yaml-language-server: $schema=../../../schemas/data-product.schema.json short_name: p0 + services: - bigquery.googleapis.com - cloudaicompanion.googleapis.com @@ -24,9 +25,11 @@ services: - dataplex.googleapis.com - datalineage.googleapis.com - storage.googleapis.com + automation: impersonation_principals: - dp-product-a-0 + exposure_layer: bigquery: datasets: @@ -40,6 +43,9 @@ exposure_layer: iam: "roles/storage.objectViewer": - data-consumer-bi + "roles/storage.bucketViewer": + - data-consumer-bi + iam_by_principals: rw: - roles/editor @@ -65,14 +71,7 @@ iam_by_principals: - "roles/iam.serviceAccountUser" - "roles/storage.bucketViewer" - "roles/storage.objectAdmin" -# iam_bindings_additive: -# test-tag: -# member: rw -# role: roles/storage.objectViewer -# condition: -# title: Storage viewer on exposed resources. -# expression: | -# resource.matchTag('${tag_values["exposure/allow"]}') + service_accounts: processing: description: Processing service account. diff --git a/fast/stages/3-data-platform-dev/schemas/data-domain.schema.json b/fast/stages/3-data-platform-dev/schemas/data-domain.schema.json index 690c91524..c1d5632a5 100644 --- a/fast/stages/3-data-platform-dev/schemas/data-domain.schema.json +++ b/fast/stages/3-data-platform-dev/schemas/data-domain.schema.json @@ -36,9 +36,7 @@ "composer": { "type": "object", "additionalProperties": false, - "required": [ - "node_config" - ], + "required": [], "properties": { "encryption_key": { "type": "string" diff --git a/fast/stages/3-data-platform-dev/terraform.tfvars.sample b/fast/stages/3-data-platform-dev/terraform.tfvars.sample new file mode 100644 index 000000000..aeb71a5ba --- /dev/null +++ b/fast/stages/3-data-platform-dev/terraform.tfvars.sample @@ -0,0 +1,42 @@ +central_project_config = { + iam_by_principals = { + "group:dp-platform-0@example.com" = [ + "roles/bigquery.dataOwner", + "roles/bigquerydatapolicy.admin", + "roles/datacatalog.tagTemplateOwner", + "roles/datacatalog.categoryAdmin", + "roles/datacatalog.entryGroupOwner", + "roles/datacatalog.glossaryOwner", + "roles/dataplex.admin", + "roles/dataplex.aspectTypeOwner" + ] + "group:dp-domain-a@example.com" = [ + "roles/datacatalog.tagTemplateUser", + "roles/datacatalog.tagTemplateViewer", + "roles/datacatalog.entryViewer", + "roles/datacatalog.glossaryUser", + "roles/dataplex.aspectTypeUser" + ] + "group:dp-product-a-0@example.com" = [ + "roles/datacatalog.tagTemplateUser", + "roles/datacatalog.tagTemplateViewer", + "roles/datacatalog.categoryAdmin", + "roles/datacatalog.entryViewer", + "roles/datacatalog.glossaryUser", + "roles/dataplex.aspectTypeUser" + ] + } +} + +factories_config = { + context = { + iam_principals = { + data-consumer-bi = "group:data-consumer-bi@example.com" + dp-product-a-0 = "group:dp-product-a-0@example.com" + dp-domain-a = "group:dp-domain-a@example.com" + dp-platform = "group:dp-platform-0@example.com" + } + } + aspect_types = "data/aspect-types" + data_domains = "data/data-domains" +} diff --git a/fast/stages/3-data-platform-dev/variables.tf b/fast/stages/3-data-platform-dev/variables.tf index 1c23e1edf..14cb88006 100644 --- a/fast/stages/3-data-platform-dev/variables.tf +++ b/fast/stages/3-data-platform-dev/variables.tf @@ -73,7 +73,8 @@ variable "central_project_config" { "bigquery.googleapis.com", "datacatalog.googleapis.com", "logging.googleapis.com", - "monitoring.googleapis.com" + "monitoring.googleapis.com", + "storage.googleapis.com", ]) short_name = optional(string, "central-0") }) diff --git a/tests/fast/stages/s3_data_platform_dev/simple.tfvars b/tests/fast/stages/s3_data_platform_dev/simple.tfvars index e5dd22f76..adeece159 100644 --- a/tests/fast/stages/s3_data_platform_dev/simple.tfvars +++ b/tests/fast/stages/s3_data_platform_dev/simple.tfvars @@ -17,6 +17,7 @@ factories_config = { iam_principals = { data-consumer-bi = "group:gcp-consumer-bi@example.com" dp-product-a-0 = "group:gcp-data-product-a-0@example.com" + dp-domain-a = "group:gcp-data-domain-a@example.com" } } } diff --git a/tests/fast/stages/s3_data_platform_dev/simple.yaml b/tests/fast/stages/s3_data_platform_dev/simple.yaml index e4319eb6f..867b6e1b2 100644 --- a/tests/fast/stages/s3_data_platform_dev/simple.yaml +++ b/tests/fast/stages/s3_data_platform_dev/simple.yaml @@ -22,20 +22,20 @@ counts: google_data_catalog_taxonomy: 1 google_dataplex_aspect_type: 1 google_folder: 2 - google_folder_iam_binding: 5 + google_folder_iam_binding: 3 google_project: 3 - google_project_iam_binding: 21 + google_project_iam_binding: 22 google_project_iam_member: 13 - google_project_service: 17 + google_project_service: 18 google_project_service_identity: 6 google_service_account: 6 google_service_account_iam_binding: 4 google_storage_bucket: 3 - google_storage_bucket_iam_binding: 5 + google_storage_bucket_iam_binding: 6 google_storage_bucket_object: 5 - google_storage_project_service_account: 2 + google_storage_project_service_account: 3 google_tags_location_tag_binding: 2 google_tags_tag_key: 1 google_tags_tag_value: 1 modules: 19 - resources: 107 + resources: 109