kovagoadi/hunfabric
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix Tests, rely on iam additive.

Browse Source
This commit is contained in:
lcaggio
2023-02-22 01:36:01 +01:00
parent 2564c9b06a
commit 2108b4650d
11 changed files with 120 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
blueprints/data-solutions/data-platform-foundations/01-dropoff.tf
View File

@@ -15,24 +15,22 @@
# tfdoc:file:description drop off project and resources.
locals {
group_iam_drp = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor",
"roles/pubsub.editor",
"roles/storage.admin",
]
}
iam_drp = {
"roles/bigquery.dataEditor" = [module.drop-sa-bq-0.iam_email]
"roles/bigquery.user" = [module.load-sa-df-0.iam_email]
"roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email]
"roles/bigquery.dataEditor" = [
module.drop-sa-bq-0.iam_email, local.groups_iam.data-engineers
]
"roles/bigquery.user" = [
module.load-sa-df-0.iam_email, local.groups_iam.data-engineers
]
"roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email]
"roles/pubsub.subscriber" = [
module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
]
"roles/storage.objectAdmin" = [module.load-sa-df-0.iam_email]
"roles/storage.objectCreator" = [module.drop-sa-cs-0.iam_email]
"roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email]
"roles/storage.admin" = [module.load-sa-df-0.iam_email]
"roles/storage.objectAdmin" = [
module.load-sa-df-0.iam_email, module.load-sa-df-0.iam_email
]
}
}
@@ -43,9 +41,8 @@ module "drop-project" {
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}"
# group_iam = local.group_iam_drp
iam = var.project_config.billing_account_id != null ? local.iam_drp : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_drp : null
iam = var.project_config.billing_account_id != null ? local.iam_drp : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_drp : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",

29
blueprints/data-solutions/data-platform-foundations/02-load.tf
View File

@@ -15,18 +15,15 @@
# tfdoc:file:description Load project and VPC.
locals {
group_iam_load = {
(local.groups.data-engineers) = [
"roles/compute.viewer",
"roles/dataflow.admin",
"roles/dataflow.developer",
"roles/viewer",
]
}
iam_load = {
"roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
module.orch-sa-cmp-0.iam_email,
module.load-sa-df-0.iam_email,
local.groups_iam.data-engineers
]
"roles/dataflow.developer" = [
local.groups_iam.data-engineers
]
"roles/dataflow.worker" = [module.load-sa-df-0.iam_email]
"roles/storage.objectAdmin" = local.load_service_accounts
@@ -56,9 +53,8 @@ module "load-project" {
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.load : "${var.project_config.project_ids.load}${local.project_suffix}"
# group_iam = local.group_iam_load
iam = var.project_config.billing_account_id != null ? local.iam_load : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_load : null
iam = var.project_config.billing_account_id != null ? local.iam_load : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_load : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
@@ -90,8 +86,13 @@ module "load-sa-df-0" {
name = "load-df-0"
display_name = "Data platform Dataflow load service account"
iam = {
"roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers]
"roles/iam.serviceAccountUser" = [module.orch-sa-cmp-0.iam_email]
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers,
module.orch-sa-cmp-0.iam_email
],
"roles/iam.serviceAccountUser" = [
module.orch-sa-cmp-0.iam_email
]
}
}

41
blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
View File

@@ -15,29 +15,22 @@
# tfdoc:file:description Orchestration project and VPC.
locals {
group_iam_orch = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor",
"roles/bigquery.jobUser",
"roles/cloudbuild.builds.editor",
"roles/composer.admin",
"roles/composer.environmentAndStorageObjectAdmin",
"roles/iap.httpsResourceAccessor",
"roles/iam.serviceAccountUser",
"roles/storage.objectAdmin",
"roles/storage.admin",
"roles/artifactregistry.admin",
"roles/serviceusage.serviceUsageConsumer",
]
}
iam_orch = {
"roles/artifactregistry.admin" = [local.groups_iam.data-engineers]
"roles/artifactregistry.reader" = [module.load-sa-df-0.iam_email]
"roles/bigquery.dataEditor" = [
module.load-sa-df-0.iam_email,
module.transf-sa-df-0.iam_email,
local.groups_iam.data-engineers
]
"roles/bigquery.jobUser" = [
module.orch-sa-cmp-0.iam_email,
local.groups_iam.data-engineers
]
"roles/cloudbuild.builds.editor" = [local.groups_iam.data-engineers]
"roles/cloudbuild.serviceAgent" = [module.orch-sa-df-build.iam_email]
"roles/composer.admin" = [local.groups_iam.data-engineers]
"roles/composer.environmentAndStorageObjectAdmin" = [local.groups_iam.data-engineers]
"roles/composer.ServiceAgentV2Ext" = [
"serviceAccount:${module.orch-project.service_accounts.robots.composer}"
]
@@ -45,19 +38,16 @@ locals {
module.orch-sa-cmp-0.iam_email
]
"roles/iam.serviceAccountUser" = [
module.orch-sa-cmp-0.iam_email
module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers
]
"roles/iap.httpsResourceAccessor" = [local.groups_iam.data-engineers]
"roles/serviceusage.serviceUsageConsumer" = [local.groups_iam.data-engineers]
"roles/storage.objectAdmin" = [
module.orch-sa-cmp-0.iam_email,
module.orch-sa-df-build.iam_email,
"serviceAccount:${module.orch-project.service_accounts.robots.composer}",
"serviceAccount:${module.orch-project.service_accounts.robots.cloudbuild}",
]
"roles/artifactregistry.reader" = [
module.load-sa-df-0.iam_email,
]
"roles/cloudbuild.serviceAgent" = [
module.orch-sa-df-build.iam_email,
local.groups_iam.data-engineers
]
"roles/storage.objectViewer" = [module.load-sa-df-0.iam_email]
}
@@ -85,10 +75,9 @@ module "orch-project" {
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}"
# group_iam = local.group_iam_orch
iam = var.project_config.billing_account_id != null ? local.iam_orch : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null
oslogin = false
iam = var.project_config.billing_account_id != null ? local.iam_orch : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null
oslogin = false
services = concat(var.project_services, [
"artifactregistry.googleapis.com",
"bigquery.googleapis.com",

19
blueprints/data-solutions/data-platform-foundations/04-transformation.tf
View File

@@ -15,22 +15,14 @@
# tfdoc:file:description Trasformation project and VPC.
locals {
group_iam_trf = {
(local.groups.data-engineers) = [
"roles/bigquery.jobUser",
"roles/dataflow.admin",
]
}
iam_trf = {
"roles/bigquery.jobUser" = [
module.transf-sa-bq-0.iam_email,
module.transf-sa-bq-0.iam_email, local.groups_iam.data-engineers
]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email,
]
"roles/dataflow.worker" = [
module.transf-sa-df-0.iam_email
module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers
]
"roles/dataflow.worker" = [module.transf-sa-df-0.iam_email]
"roles/storage.objectAdmin" = [
module.transf-sa-df-0.iam_email,
"serviceAccount:${module.transf-project.service_accounts.robots.dataflow}"
@@ -55,9 +47,8 @@ module "transf-project" {
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}"
# group_iam = local.group_iam_trf
iam = var.project_config.billing_account_id != null ? local.iam_orch : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null
iam = var.project_config.billing_account_id != null ? local.iam_trf : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_trf : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",

69
blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf
View File

@@ -15,54 +15,48 @@
# tfdoc:file:description Data Warehouse projects.
locals {
dwh_group_iam = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor",
"roles/storage.admin",
],
(local.groups.data-analysts) = [
"roles/bigquery.dataViewer",
"roles/bigquery.jobUser",
"roles/bigquery.metadataViewer",
"roles/bigquery.user",
"roles/datacatalog.viewer",
"roles/datacatalog.tagTemplateViewer",
"roles/storage.objectViewer",
]
}
dwh_lnd_iam = {
"roles/bigquery.dataOwner" = [
module.load-sa-df-0.iam_email,
]
"roles/bigquery.dataViewer" = [
module.transf-sa-df-0.iam_email,
module.transf-sa-bq-0.iam_email,
local.groups_iam.data-engineers
]
"roles/bigquery.jobUser" = [
module.load-sa-df-0.iam_email,
]
"roles/datacatalog.categoryAdmin" = [
module.transf-sa-bq-0.iam_email
]
"roles/storage.objectCreator" = [
module.load-sa-df-0.iam_email,
module.load-sa-df-0.iam_email, local.groups_iam.data-engineers
]
"roles/datacatalog.categoryAdmin" = [module.transf-sa-bq-0.iam_email]
"roles/datacatalog.tagTemplateViewer" = [local.groups_iam.data-engineers]
"roles/datacatalog.viewer" = [local.groups_iam.data-engineers]
"roles/storage.objectCreator" = [module.load-sa-df-0.iam_email]
"roles/storage.objectViewer" = [local.groups_iam.data-engineers]
}
dwh_iam = {
"roles/bigquery.dataOwner" = [
module.transf-sa-df-0.iam_email,
module.transf-sa-bq-0.iam_email,
]
"roles/bigquery.dataViewer" = [
local.groups_iam.data-analysts,
local.groups_iam.data-engineers
]
"roles/bigquery.jobUser" = [
module.transf-sa-bq-0.iam_email,
local.groups_iam.data-analysts,
local.groups_iam.data-engineers
]
"roles/datacatalog.categoryAdmin" = [
module.load-sa-df-0.iam_email
"roles/datacatalog.tagTemplateViewer" = [
local.groups_iam.data-analysts, local.groups_iam.data-engineers
]
"roles/storage.objectCreator" = [
module.transf-sa-df-0.iam_email,
"roles/datacatalog.viewer" = [
local.groups_iam.data-analysts, local.groups_iam.data-engineers
]
"roles/storage.objectViewer" = [
module.transf-sa-df-0.iam_email,
local.groups_iam.data-analysts, local.groups_iam.data-engineers
]
"roles/storage.objectAdmin" = [module.transf-sa-df-0.iam_email]
}
dwh_services = concat(var.project_services, [
"bigquery.googleapis.com",
@@ -87,10 +81,9 @@ module "dwh-lnd-project" {
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-lnd : "${var.project_config.project_ids.dwh-lnd}${local.project_suffix}"
# group_iam = local.dwh_group_iam
iam = var.project_config.billing_account_id != null ? local.dwh_lnd_iam : {}
iam_additive = var.project_config.billing_account_id == null ? local.dwh_lnd_iam : {}
services = local.dwh_services
iam = var.project_config.billing_account_id != null ? local.dwh_lnd_iam : {}
iam_additive = var.project_config.billing_account_id == null ? local.dwh_lnd_iam : {}
services = local.dwh_services
service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)]
@@ -104,10 +97,9 @@ module "dwh-cur-project" {
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-cur : "${var.project_config.project_ids.dwh-cur}${local.project_suffix}"
# group_iam = local.dwh_group_iam
iam = var.project_config.billing_account_id != null ? local.dwh_iam : {}
iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : {}
services = local.dwh_services
iam = var.project_config.billing_account_id != null ? local.dwh_iam : {}
iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : {}
services = local.dwh_services
service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)]
@@ -121,10 +113,9 @@ module "dwh-conf-project" {
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-conf : "${var.project_config.project_ids.dwh-conf}${local.project_suffix}"
# group_iam = local.dwh_group_iam
iam = var.project_config.billing_account_id != null ? local.dwh_iam : null
iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : null
services = local.dwh_services
iam = var.project_config.billing_account_id != null ? local.dwh_iam : null
iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : null
services = local.dwh_services
service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)]

29
blueprints/data-solutions/data-platform-foundations/06-common.tf
View File

@@ -15,29 +15,21 @@
# tfdoc:file:description common project.
locals {
group_iam_common = {
(local.groups.data-analysts) = [
"roles/datacatalog.viewer",
]
(local.groups.data-engineers) = [
"roles/dlp.reader",
"roles/dlp.user",
"roles/dlp.estimatesAdmin",
]
(local.groups.data-security) = [
"roles/dlp.admin",
"roles/datacatalog.admin"
]
}
iam_common = {
"roles/dlp.admin" = [local.groups_iam.data-security]
"roles/dlp.estimatesAdmin" = [local.groups_iam.data-engineers]
"roles/dlp.reader" = [local.groups_iam.data-engineers]
"roles/dlp.user" = [
module.load-sa-df-0.iam_email,
module.transf-sa-df-0.iam_email
module.transf-sa-df-0.iam_email,
local.groups_iam.data-engineers
]
"roles/datacatalog.admin" = [local.groups_iam.data-security]
"roles/datacatalog.viewer" = [
module.load-sa-df-0.iam_email,
module.transf-sa-df-0.iam_email,
module.transf-sa-bq-0.iam_email
module.transf-sa-bq-0.iam_email,
local.groups_iam.data-analysts
]
"roles/datacatalog.categoryFineGrainedReader" = [
module.transf-sa-df-0.iam_email,
@@ -54,9 +46,8 @@ module "common-project" {
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.common : "${var.project_config.project_ids.common}${local.project_suffix}"
# group_iam = local.group_iam_common
iam = var.project_config.billing_account_id != null ? local.iam_common : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_common : null
iam = var.project_config.billing_account_id != null ? local.iam_common : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_common : null
services = concat(var.project_services, [
"datacatalog.googleapis.com",
"dlp.googleapis.com",

10
blueprints/data-solutions/data-platform-foundations/README.md
View File

@@ -215,13 +215,13 @@ module "data-platform" {
source = "./fabric/blueprints/data-solutions/data-platform-foundations"
organization_domain = "example.com"
project_config = {
billing_account_id = "123456-123456-123456"
parent = "folders/12345678"
}
prefix = "myprefix"
billing_account_id = "123456-123456-123456"
parent = "folders/12345678"
}
prefix = "myprefix"
}
# tftest modules=43 resources=265
# tftest modules=43 resources=278
```
## Customizations

8
blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py
View File

@@ -123,7 +123,7 @@ with models.DAG(
task_id="upsert_table_customers",
project_id=DWH_LAND_PRJ,
dataset_id=DWH_LAND_BQ_DATASET,
impersonation_chain=[TRF_SA_DF],
impersonation_chain=[LOD_SA_DF],
table_resource={
"tableReference": {"tableId": "customers"},
},
@@ -133,7 +133,7 @@ with models.DAG(
task_id="upsert_table_purchases",
project_id=DWH_LAND_PRJ,
dataset_id=DWH_LAND_BQ_DATASET,
impersonation_chain=[TRF_SA_BQ],
impersonation_chain=[LOD_SA_DF],
table_resource={
"tableReference": {"tableId": "purchases"}
},
@@ -167,7 +167,7 @@ with models.DAG(
project_id=DWH_LAND_PRJ,
dataset_id=DWH_LAND_BQ_DATASET,
table_id="customers",
impersonation_chain=[TRF_SA_BQ],
impersonation_chain=[LOD_SA_DF],
include_policy_tags=True,
schema_fields_updates=[
{ "mode": "REQUIRED", "name": "id", "type": "INTEGER", "description": "ID" },
@@ -182,7 +182,7 @@ with models.DAG(
project_id=DWH_LAND_PRJ,
dataset_id=DWH_LAND_BQ_DATASET,
table_id="purchases",
impersonation_chain=[TRF_SA_BQ],
impersonation_chain=[LOD_SA_DF],
include_policy_tags=True,
schema_fields_updates=[
{ "mode": "REQUIRED", "name": "id", "type": "INTEGER", "description": "ID" },

4
blueprints/data-solutions/data-platform-foundations/demo/delete_table.py
View File

@@ -122,13 +122,13 @@ with models.DAG(
delete_table_customers = BigQueryDeleteTableOperator(
task_id="delete_table_customers",
deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".customers",
impersonation_chain=[TRF_SA_DF]
impersonation_chain=[LOD_SA_DF]
)
delete_table_purchases = BigQueryDeleteTableOperator(
task_id="delete_table_purchases",
deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".purchases",
impersonation_chain=[TRF_SA_DF]
impersonation_chain=[LOD_SA_DF]
)
delete_table_customer_purchase_curated = BigQueryDeleteTableOperator(