kovagoadi/hunfabric
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor variables

Browse Source
This commit is contained in:
Lorenzo Caggioni
2022-02-07 21:28:54 +01:00
parent 94d94876a0
commit d8bad57790
7 changed files with 100 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
examples/data-solutions/data-platform-foundations/02-load.tf
View File

@@ -83,13 +83,13 @@ module "lod-prj" {
}
module "lod-vpc" {
count = var.network_config.network != null ? 0 : 1
count = var.network_config.network_self_link != null ? 0 : 1
source = "../../../modules/net-vpc"
project_id = module.lod-prj.project_id
name = "${local.prefix_lod}-vpc"
subnets = [
{
ip_cidr_range = var.network_config.vpc_subnet.load.range
ip_cidr_range = "10.10.0.0/24"
name = "${local.prefix_lod}-subnet"
region = var.location_config.region
secondary_ip_range = {}
@@ -98,18 +98,18 @@ module "lod-vpc" {
}
module "lod-vpc-firewall" {
count = var.network_config.network != null ? 0 : 1
count = var.network_config.network_self_link != null ? 0 : 1
source = "../../../modules/net-vpc-firewall"
project_id = module.lod-prj.project_id
network = local._networks.load.network_name
admin_ranges = [local._networks.load.subnet_range]
admin_ranges = ["10.10.0.0/24"]
}
module "lod-nat" {
count = var.network_config.enable_cloud_nat ? 1 : 0
count = var.network_config.network_self_link != null ? 0 : 1
source = "../../../modules/net-cloudnat"
project_id = module.lod-prj.project_id
region = var.location_config.region
name = "${local.prefix_lod}-default"
router_network = local._networks.load.network_name
router_network = module.lod-vpc[0].name
}

84
examples/data-solutions/data-platform-foundations/03-composer.tf
View File

@@ -31,63 +31,65 @@ module "orc-sa-cmp-0" {
resource "google_composer_environment" "orc-cmp-0" {
name = "${local.prefix_orc}-cmp-0"
region = var.composer_config.region
region = var.location_config.region
provider = google-beta
project = module.orc-prj.project_id
config {
node_count = var.composer_config.node_count
node_config {
zone = "${var.composer_config.region}-b"
zone = "${var.location_config.region}-b"
service_account = module.orc-sa-cmp-0.email
network = local._networks.orchestration.network
subnetwork = local._networks.orchestration.subnet
tags = ["composer-worker", "http-server", "https-server"]
ip_allocation_policy {
use_ip_aliases = var.composer_config.ip_allocation_policy.use_ip_aliases
cluster_secondary_range_name = var.composer_config.ip_allocation_policy.cluster_secondary_range_name
services_secondary_range_name = var.composer_config.ip_allocation_policy.services_secondary_range_name
use_ip_aliases = "true"
cluster_secondary_range_name = try(var.network_config.composer_secondary_ranges.pods, "pods")
services_secondary_range_name = try(var.network_config.composer_secondary_ranges.services, "services")
}
}
software_config {
image_version = "composer-1.17.5-airflow-2.1.4"
env_variables = {
DTL_L0_PRJ = module.dtl-0-prj.project_id
DTL_L0_BQ_DATASET = module.dtl-0-bq-0.dataset_id
DTL_L0_GCS = module.dtl-0-cs-0.url
DTL_L1_PRJ = module.dtl-1-prj.project_id
DTL_L1_BQ_DATASET = module.dtl-1-bq-0.dataset_id
DTL_L1_GCS = module.dtl-1-cs-0.url
DTL_L2_PRJ = module.dtl-2-prj.project_id
DTL_L2_BQ_DATASET = module.dtl-2-bq-0.dataset_id
DTL_L2_GCS = module.dtl-2-cs-0.url
DTL_PLG_PRJ = module.dtl-plg-prj.project_id
DTL_PLG_BQ_DATASET = module.dtl-plg-bq-0.dataset_id
DTL_PLG_GCS = module.dtl-plg-cs-0.url
GCP_REGION = var.composer_config.region
LND_PRJ = module.lnd-prj.project_id
LND_BQ = module.lnd-bq-0.dataset_id
LND_GCS = module.lnd-cs-0.url
LND_PS = module.lnd-ps-0.id
LOD_PRJ = module.lod-prj.project_id
LOD_GCS_STAGING = module.lod-cs-df-0.url
LOD_NET_VPC = local._networks.load.network
LOD_NET_SUBNET = local._networks.load.subnet
LOD_SA_DF = module.lod-sa-df-0.email
ORC_PRJ = module.orc-prj.project_id
ORC_GCS = module.orc-cs-0.url
TRF_PRJ = module.trf-prj.project_id
TRF_GCS_STAGING = module.trf-cs-df-0.url
TRF_NET_VPC = local._networks.transformation.network
TRF_NET_SUBNET = local._networks.transformation.subnet
TRF_SA_DF = module.trf-sa-df-0.email
TRF_SA_BQ = module.trf-sa-bq-0.email
}
image_version = var.composer_config.airflow_version
env_variables = merge(
var.composer_config.env_variables, {
DTL_L0_PRJ = module.dtl-0-prj.project_id
DTL_L0_BQ_DATASET = module.dtl-0-bq-0.dataset_id
DTL_L0_GCS = module.dtl-0-cs-0.url
DTL_L1_PRJ = module.dtl-1-prj.project_id
DTL_L1_BQ_DATASET = module.dtl-1-bq-0.dataset_id
DTL_L1_GCS = module.dtl-1-cs-0.url
DTL_L2_PRJ = module.dtl-2-prj.project_id
DTL_L2_BQ_DATASET = module.dtl-2-bq-0.dataset_id
DTL_L2_GCS = module.dtl-2-cs-0.url
DTL_PLG_PRJ = module.dtl-plg-prj.project_id
DTL_PLG_BQ_DATASET = module.dtl-plg-bq-0.dataset_id
DTL_PLG_GCS = module.dtl-plg-cs-0.url
GCP_REGION = var.location_config.region
LND_PRJ = module.lnd-prj.project_id
LND_BQ = module.lnd-bq-0.dataset_id
LND_GCS = module.lnd-cs-0.url
LND_PS = module.lnd-ps-0.id
LOD_PRJ = module.lod-prj.project_id
LOD_GCS_STAGING = module.lod-cs-df-0.url
LOD_NET_VPC = local._networks.load.network
LOD_NET_SUBNET = local._networks.load.subnet
LOD_SA_DF = module.lod-sa-df-0.email
ORC_PRJ = module.orc-prj.project_id
ORC_GCS = module.orc-cs-0.url
TRF_PRJ = module.trf-prj.project_id
TRF_GCS_STAGING = module.trf-cs-df-0.url
TRF_NET_VPC = local._networks.transformation.network
TRF_NET_SUBNET = local._networks.transformation.subnet
TRF_SA_DF = module.trf-sa-df-0.email
TRF_SA_BQ = module.trf-sa-bq-0.email
}
)
}
private_environment_config {
enable_private_endpoint = "true"
master_ipv4_cidr_block = var.composer_config.ip_range_gke_master
cloud_sql_ipv4_cidr_block = var.composer_config.ip_range_cloudsql
web_server_ipv4_cidr_block = var.composer_config.ip_range_web_server
cloud_sql_ipv4_cidr_block = try(var.network_config.composer_ip_ranges.gke_master, "10.20.10.0/24")
master_ipv4_cidr_block = try(var.network_config.composer_ip_ranges.cloudsql, "10.20.11.0/28")
web_server_ipv4_cidr_block = try(var.network_config.composer_ip_ranges.web_server, "10.20.11.16/28")
}
dynamic "encryption_config" {

44
examples/data-solutions/data-platform-foundations/03-orchestration.tf
View File

@@ -79,11 +79,13 @@ module "orc-prj" {
project_create = var.project_create != null
prefix = var.project_create == null ? null : var.prefix
# additive IAM bindings avoid disrupting bindings in existing project
iam = var.project_create != null ? local.iam_orc : {}
iam_additive = var.project_create == null ? local.iam_orc : {}
group_iam = local.group_iam_orc
oslogin = false
policy_boolean = var.composer_config.project_policy_boolean
iam = var.project_create != null ? local.iam_orc : {}
iam_additive = var.project_create == null ? local.iam_orc : {}
group_iam = local.group_iam_orc
oslogin = false
policy_boolean = {
"constraints/compute.requireOsLogin" = false
}
services = concat(
var.project_services,
[
@@ -111,28 +113,26 @@ module "orc-prj" {
}
module "orc-vpc" {
count = var.network_config.network != null ? 0 : 1
count = var.network_config.network_self_link != null ? 0 : 1
source = "../../../modules/net-vpc"
project_id = module.orc-prj.project_id
name = "${local.prefix_orc}-vpc"
subnets = [
{
ip_cidr_range = var.network_config.vpc_subnet.orchestration.range
name = "${local.prefix_orc}-subnet"
region = var.location_config.region
secondary_ip_range = {}
ip_cidr_range = "10.10.0.0/24"
name = "${local.prefix_orc}-subnet"
region = var.location_config.region
secondary_ip_range = {
pods = var.network_config.vpc_subnet.orchestration.secondary_range.pods
services = var.network_config.vpc_subnet.orchestration.secondary_range.services
pods = "10.10.8.0/22"
services = "10.10.12.0/24"
}
}
]
}
#TODO Check with Simo/Ludo
resource "google_project_iam_binding" "composer_shared_vpc_agent" {
count = var.network_config.network != null ? 1 : 0
project = var.network_config.host_project
count = var.network_config.network_self_link != null ? 1 : 0
project = local._shared_vpc_project
role = "roles/composer.sharedVpcAgent"
members = [
"serviceAccount:${module.orc-prj.service_accounts.robots.composer}"
@@ -140,8 +140,8 @@ resource "google_project_iam_binding" "composer_shared_vpc_agent" {
}
resource "google_project_iam_binding" "gke_host_service_agent_user" {
count = var.network_config.network != null ? 1 : 0
project = var.network_config.host_project
count = var.network_config.network_self_link != null ? 1 : 0
project = local._shared_vpc_project
role = "roles/container.hostServiceAgentUser"
members = [
"serviceAccount:${module.orc-prj.service_accounts.robots.container-engine}"
@@ -149,8 +149,8 @@ resource "google_project_iam_binding" "gke_host_service_agent_user" {
}
resource "google_project_iam_binding" "composer_network_user_agent" {
count = var.network_config.network != null ? 1 : 0
project = var.network_config.host_project
count = var.network_config.network_self_link != null ? 1 : 0
project = local._shared_vpc_project
role = "roles/compute.networkUser"
members = [
module.orc-sa-cmp-0.iam_email,
@@ -164,15 +164,15 @@ resource "google_project_iam_binding" "composer_network_user_agent" {
}
module "orc-vpc-firewall" {
count = var.network_config.network != null ? 0 : 1
count = var.network_config.network_self_link != null ? 0 : 1
source = "../../../modules/net-vpc-firewall"
project_id = module.orc-prj.project_id
network = local._networks.orchestration.network_name
admin_ranges = [local._networks.orchestration.subnet_range]
admin_ranges = ["10.10.0.0/24"]
}
module "orc-nat" {
count = var.network_config.enable_cloud_nat ? 1 : 0
count = var.network_config.network_self_link != null ? 0 : 1
source = "../../../modules/net-cloudnat"
project_id = module.orc-prj.project_id
region = var.location_config.region

12
examples/data-solutions/data-platform-foundations/04-transformation.tf
View File

@@ -77,13 +77,13 @@ module "trf-prj" {
}
module "trf-vpc" {
count = var.network_config.network != null ? 0 : 1
count = var.network_config.network_self_link != null ? 0 : 1
source = "../../../modules/net-vpc"
project_id = module.trf-prj.project_id
name = "${local.prefix_trf}-vpc"
subnets = [
{
ip_cidr_range = var.network_config.vpc_subnet.transformation.range
ip_cidr_range = "10.10.0.0/24"
name = "${local.prefix_trf}-subnet"
region = var.location_config.region
secondary_ip_range = {}
@@ -92,18 +92,18 @@ module "trf-vpc" {
}
module "trf-vpc-firewall" {
count = var.network_config.network != null ? 0 : 1
count = var.network_config.network_self_link != null ? 0 : 1
source = "../../../modules/net-vpc-firewall"
project_id = module.trf-prj.project_id
network = local._networks.transformation.network_name
admin_ranges = [local._networks.transformation.subnet_range]
admin_ranges = ["10.10.0.0/24"]
}
module "trf-nat" {
count = var.network_config.enable_cloud_nat ? 1 : 0
count = var.network_config.network_self_link != null ? 0 : 1
source = "../../../modules/net-cloudnat"
project_id = module.trf-prj.project_id
region = var.location_config.region
name = "${local.prefix_trf}-default"
router_network = local._networks.transformation.network_name
router_network = module.trf-vpc[0].name
}

4
examples/data-solutions/data-platform-foundations/README.md
View File

@@ -8,7 +8,7 @@ The following diagram is a high-level reference of the resources created and man
![Data Platform architecture overview](./images/overview_diagram.png "Data Platform architecture overview")
A demo pipeline is also part of this example: it can be built and run on top of the foundational infrastructure to verify or test the setup quickly.
A demo Airflow pipeline is also part of this example: it can be built and run on top of the foundational infrastructure to verify or test the setup quickly.
## Design overview and choices
@@ -21,7 +21,7 @@ The approach adapts to different high-level requirements:
- least privilege principle
- rely on service account impersonation
The code in this example doesn't address Organization-level configurations (Organization policy, VPC-SC, centralized logs). We expect those to be managed by automation stages external to this script like those in [FAST](../../../fast).
The code in this example doesn't address Organization-level configurations (Organization policy, VPC-SC, centralized logs). We expect those elements to be managed by automation stages external to this script like those in [FAST](../../../fast).
### Project structure

27
examples/data-solutions/data-platform-foundations/main.tf
View File

@@ -17,31 +17,28 @@
locals {
_networks = {
load = {
#TODO Fix Network name logic
network_name = element(split("/", var.network_config.network != null ? var.network_config.network : module.lod-vpc[0].self_link), length(split("/", var.network_config.network != null ? var.network_config.network : module.lod-vpc[0].self_link)) - 1)
network = var.network_config.network != null ? var.network_config.network : module.lod-vpc[0].self_link
subnet = var.network_config.network != null ? var.network_config.vpc_subnet_self_link.load : module.lod-vpc[0].subnet_self_links["${var.composer_config.region}/${local.prefix_lod}-subnet"]
subnet_range = try(var.network_config.vpc_subnet.load.range, null)
network_name = element(split("/", var.network_config.network_self_link != null ? var.network_config.network_self_link : module.lod-vpc[0].self_link), length(split("/", var.network_config.network_self_link != null ? var.network_config.network_self_link : module.lod-vpc[0].self_link)) - 1)
network = var.network_config.network_self_link != null ? var.network_config.network_self_link : module.lod-vpc[0].self_link
subnet = var.network_config.network_self_link != null ? var.network_config.vpc_subnet_self_link.load : module.lod-vpc[0].subnet_self_links["${var.location_config.region}/${local.prefix_lod}-subnet"]
}
orchestration = {
#TODO Fix Network name logic
network_name = element(split("/", var.network_config.network != null ? var.network_config.network : module.orc-vpc[0].self_link), length(split("/", var.network_config.network != null ? var.network_config.network : module.orc-vpc[0].self_link)) - 1)
network = var.network_config.network != null ? var.network_config.network : module.orc-vpc[0].self_link
subnet = var.network_config.network != null ? var.network_config.vpc_subnet_self_link.orchestration : module.orc-vpc[0].subnet_self_links["${var.composer_config.region}/${local.prefix_orc}-subnet"]
subnet_range = try(var.network_config.vpc_subnet.orchestration.range, null)
network_name = element(split("/", var.network_config.network_self_link != null ? var.network_config.network_self_link : module.orc-vpc[0].self_link), length(split("/", var.network_config.network_self_link != null ? var.network_config.network_self_link : module.orc-vpc[0].self_link)) - 1)
network = var.network_config.network_self_link != null ? var.network_config.network_self_link : module.orc-vpc[0].self_link
subnet = var.network_config.network_self_link != null ? var.network_config.vpc_subnet_self_link.orchestration : module.orc-vpc[0].subnet_self_links["${var.location_config.region}/${local.prefix_orc}-subnet"]
}
transformation = {
#TODO Fix Network name logic
network_name = element(split("/", var.network_config.network != null ? var.network_config.network : module.trf-vpc[0].self_link), length(split("/", var.network_config.network != null ? var.network_config.network : module.trf-vpc[0].self_link)) - 1)
network = var.network_config.network != null ? var.network_config.network : module.trf-vpc[0].self_link
subnet = var.network_config.network != null ? var.network_config.vpc_subnet_self_link.transformation : module.trf-vpc[0].subnet_self_links["${var.composer_config.region}/${local.prefix_trf}-subnet"]
subnet_range = try(var.network_config.vpc_subnet.transformation.range, null)
network_name = element(split("/", var.network_config.network_self_link != null ? var.network_config.network_self_link : module.trf-vpc[0].self_link), length(split("/", var.network_config.network_self_link != null ? var.network_config.network_self_link : module.trf-vpc[0].self_link)) - 1)
network = var.network_config.network_self_link != null ? var.network_config.network_self_link : module.trf-vpc[0].self_link
subnet = var.network_config.network_self_link != null ? var.network_config.vpc_subnet_self_link.transformation : module.trf-vpc[0].subnet_self_links["${var.location_config.region}/${local.prefix_trf}-subnet"]
}
}
_shared_vpc_service_config = var.network_config.network != null ? {
_shared_vpc_project = try(regex("projects/([a-z0-9-]{6,30})", var.network_config.network_self_link), null)
_shared_vpc_service_config = var.network_config.network_self_link != null ? {
attach = true
host_project = var.network_config.host_project
host_project = local._shared_vpc_project
} : null
groups = { for k, v in var.groups : k => "${v}@${var.organization.domain}" }

68
examples/data-solutions/data-platform-foundations/variables.tf
View File

@@ -19,43 +19,12 @@ variable "composer_config" {
type = object({
node_count = number
airflow_version = string
environment = map(string)
env_variables = map(string)
})
default = {
node_count = 3
airflow_version = "TODO"
environment = {}
}
}
variable "composer_config" {
type = object({
node_count = number
#TODO Move to network
ip_range_cloudsql = string
ip_range_gke_master = string
ip_range_web_server = string
#TODO hardcoded
project_policy_boolean = map(bool)
region = string
ip_allocation_policy = object({
use_ip_aliases = string
cluster_secondary_range_name = string
services_secondary_range_name = string
})
#TODO Add Env variables, Airflow version
})
default = {
node_count = 3
ip_range_cloudsql = "10.20.10.0/24"
ip_range_gke_master = "10.20.11.0/28"
ip_range_web_server = "10.20.11.16/28"
project_policy_boolean = null
region = "europe-west1"
ip_allocation_policy = {
use_ip_aliases = "true"
cluster_secondary_range_name = "pods"
services_secondary_range_name = "services"
}
airflow_version = "composer-1.17.5-airflow-2.1.4"
env_variables = {}
}
}
@@ -76,11 +45,9 @@ variable "groups" {
}
variable "network_config" {
description = "Network configurations to use. Specify a shared VPC to use, if null networks will be created in projects."
description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values."
type = object({
#TODO hardcoded Cloud NAT
network_self_link = string
#TODO hardcoded VPC ranges
subnet_self_links = object({
load = string
transformation = string
@@ -96,29 +63,12 @@ variable "network_config" {
services = string
})
})
default = {
enable_cloud_nat = false
host_project = null
network = null
vpc_subnet = {
load = {
range = "10.10.0.0/24"
secondary_range = null
}
transformation = {
range = "10.10.0.0/24"
secondary_range = null
}
orchestration = {
range = "10.10.0.0/24"
secondary_range = {
pods = "10.10.8.0/22"
services = "10.10.12.0/24"
}
}
}
vpc_subnet_self_link = null
default = {
network_self_link = null
subnet_self_links = null
composer_ip_ranges = null
composer_secondary_ranges = null
}
}