diff --git a/blueprints/README.md b/blueprints/README.md index e7136d9ce..49b374eaa 100644 --- a/blueprints/README.md +++ b/blueprints/README.md @@ -6,7 +6,7 @@ Currently available blueprints: - **apigee** - [Apigee Hybrid on GKE](./apigee/hybrid-gke/), [Apigee X analytics in BigQuery](./apigee/bigquery-analytics), [Apigee network patterns](./apigee/network-patterns/) - **cloud operations** - [Active Directory Federation Services](./cloud-operations/adfs), [Cloud Asset Inventory feeds for resource change tracking and remediation](./cloud-operations/asset-inventory-feed-remediation), [Fine-grained Cloud DNS IAM via Service Directory](./cloud-operations/dns-fine-grained-iam), [Cloud DNS & Shared VPC design](./cloud-operations/dns-shared-vpc), [Delegated Role Grants](./cloud-operations/iam-delegated-role-grants), [Networking Dashboard](./cloud-operations/network-dashboard), [Managing on-prem service account keys by uploading public keys](./cloud-operations/onprem-sa-key-management), [Compute Image builder with Hashicorp Packer](./cloud-operations/packer-image-builder), [Packer example](./cloud-operations/packer-image-builder/packer), [Compute Engine quota monitoring](./cloud-operations/quota-monitoring), [Scheduled Cloud Asset Inventory Export to Bigquery](./cloud-operations/scheduled-asset-inventory-export-bq), [Configuring workload identity federation with Terraform Cloud/Enterprise workflows](./cloud-operations/terraform-cloud-dynamic-credentials), [TCP healthcheck and restart for unmanaged GCE instances](./cloud-operations/unmanaged-instances-healthcheck), [Migrate for Compute Engine (v5) blueprints](./cloud-operations/vm-migration), [Configuring workload identity federation to access Google Cloud resources from apps running on Azure](./cloud-operations/workload-identity-federation) -- **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder) +- **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder), [BigQuery ML and Vertex AI Pipeline](./data-solutions/dq'ml) - **factories** - [The why and the how of Resource Factories](./factories), [Google Cloud Identity Group Factory](./factories/cloud-identity-group-factory), [Google Cloud BQ Factory](./factories/bigquery-factory), [Google Cloud VPC Firewall Factory](./factories/net-vpc-firewall-yaml), [Minimal Project Factory](./factories/project-factory) - **GKE** - [Binary Authorization Pipeline Blueprint](./gke/binauthz), [Storage API](./gke/binauthz/image), [Multi-cluster mesh on GKE (fleet API)](./gke/multi-cluster-mesh-gke-fleet-api), [GKE Multitenant Blueprint](./gke/multitenant-fleet), [Shared VPC with GKE support](./networking/shared-vpc-gke/) - **networking** - [Calling a private Cloud Function from On-premises](./networking/private-cloud-function-from-onprem), [Decentralized firewall management](./networking/decentralized-firewall), [Decentralized firewall validator](./networking/decentralized-firewall/validator), [Network filtering with Squid](./networking/filtering-proxy), [GLB and multi-regional daisy-chaining through hybrid NEGs](./networking/glb-hybrid-neg-internal), [Hybrid connectivity to on-premise services through PSC](./networking/psc-hybrid), [HTTP Load Balancer with Cloud Armor](./networking/glb-and-armor), [Hub and Spoke via VPN](./networking/hub-and-spoke-vpn), [Hub and Spoke via VPC Peering](./networking/hub-and-spoke-peering), [Internal Load Balancer as Next Hop](./networking/ilb-next-hop), [Network filtering with Squid with isolated VPCs using Private Service Connect](./networking/filtering-proxy-psc), On-prem DNS and Google Private Access, [PSC Producer](./networking/psc-hybrid/psc-producer), [PSC Consumer](./networking/psc-hybrid/psc-consumer), [Shared VPC with optional GKE cluster](./networking/shared-vpc-gke) diff --git a/blueprints/data-solutions/README.md b/blueprints/data-solutions/README.md index 5fc832b3b..9cef8bc26 100644 --- a/blueprints/data-solutions/README.md +++ b/blueprints/data-solutions/README.md @@ -73,6 +73,5 @@ This [blueprint](./shielded-folder/) implements an opinionated folder configurat ### BigQuery ML and Vertex AI Pipeline -This [blueprint](./bq-ml/) implements the infrastructure required to have a fully functional development environment using BigQuery ML and Vertex AI to develop and deploy a machine learning model to be used from Vertex AI endpoint or in BigQuery ML. - +This [blueprint](./bq-ml/) provides the necessary infrastructure to create a complete development environment for building and deploying machine learning models using BigQuery ML and Vertex AI. With this blueprint, you can deploy your models to a Vertex AI endpoint or use them within BigQuery ML.
diff --git a/blueprints/data-solutions/bq-ml/README.md b/blueprints/data-solutions/bq-ml/README.md index ca6a1676a..3efa457e5 100644 --- a/blueprints/data-solutions/bq-ml/README.md +++ b/blueprints/data-solutions/bq-ml/README.md @@ -1,6 +1,6 @@ # BigQuery ML and Vertex AI Pipeline -This blueprint creates the infrastructure needed to deploy and run a Vertex AI environment to develop and deploy a machine learning model to be used from Vertex AI endpoint or in BigQuery. +This blueprint provides the necessary infrastructure to create a complete development environment for building and deploying machine learning models using BigQuery ML and Vertex AI. With this blueprint, you can deploy your models to a Vertex AI endpoint or use them within BigQuery ML. This is the high-level diagram: @@ -30,15 +30,15 @@ This sample creates several distinct groups of resources: ### Virtual Private Cloud (VPC) design -As is often the case in real-world configurations, this blueprint accepts an existing Shared-VPC via the `network_config` variable as input. +As is often the case in real-world configurations, this blueprint accepts an existing Shared-VPC via the `vpc_config` variable as input. -### Customer Managed Encryption Key +### Customer Managed Encryption Keys As is often the case in real-world configurations, this blueprint accepts as input existing Cloud KMS keys to encrypt resources via the `service_encryption_keys` variable. ## Demo -In the repository [`demo`](./demo/) folder, you can find an example of creating a Vertex AI pipeline from a publically available dataset and deploying the model to be used from a Vertex AI managed endpoint or from within Bigquery. +In the [`demo`](./demo/) folder, you can find an example of creating a Vertex AI pipeline from a publicly available dataset and deploying the model to be used from a Vertex AI managed endpoint or from within Bigquery. To run the demo: @@ -47,6 +47,18 @@ To run the demo: - Run the and run [`demo/bmql_pipeline.ipynb`](demo/bmql_pipeline.ipynb) Jupyter Notebook. +## Files + +| name | description | modules | resources | +|---|---|---|---| +| [datastorage.tf](./datastorage.tf) | Datastorage resources. | bigquery-dataset · gcs | | +| [main.tf](./main.tf) | Core resources. | project | | +| [outputs.tf](./outputs.tf) | Output variables. | | | +| [variables.tf](./variables.tf) | Terraform variables. | | | +| [versions.tf](./versions.tf) | Version pins. | | | +| [vertex.tf](./vertex.tf) | Vertex resources. | iam-service-account | google_notebooks_instance · google_vertex_ai_metadata_store | +| [vpc.tf](./vpc.tf) | VPC resources. | net-cloudnat · net-vpc · net-vpc-firewall | google_project_iam_member | + ## Variables | name | description | type | required | default | @@ -54,10 +66,10 @@ To run the demo: | [prefix](variables.tf#L33) | Prefix used for resource names. | string | ✓ | | | [project_id](variables.tf#L51) | Project id references existing project if `project_create` is null. | string | ✓ | | | [location](variables.tf#L17) | The location where resources will be deployed. | string | | "US" | -| [network_config](variables.tf#L23) | Shared VPC network configurations to use. If null networks will be created in projects with pre-configured values. | object({…}) | | null | | [project_create](variables.tf#L42) | Provide values if project creation is needed, use existing project if null. Parent format: folders/folder_id or organizations/org_id. | object({…}) | | null | | [region](variables.tf#L56) | The region where resources will be deployed. | string | | "us-central1" | | [service_encryption_keys](variables.tf#L62) | Cloud KMS to use to encrypt different services. The key location should match the service region. | object({…}) | | null | +| [vpc_config](variables.tf#L23) | Shared VPC network configurations to use. If null networks will be created in projects with pre-configured values. | object({…}) | | null | ## Outputs diff --git a/blueprints/data-solutions/bq-ml/datastorage.tf b/blueprints/data-solutions/bq-ml/datastorage.tf new file mode 100644 index 000000000..ad5910952 --- /dev/null +++ b/blueprints/data-solutions/bq-ml/datastorage.tf @@ -0,0 +1,32 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# tfdoc:file:description Datastorage resources. + +module "bucket" { + source = "../../../modules/gcs" + project_id = module.project.project_id + prefix = var.prefix + location = var.location + name = "data" + encryption_key = try(local.service_encryption_keys.storage, null) # Example assignment of an encryption key +} + +module "dataset" { + source = "../../../modules/bigquery-dataset" + project_id = module.project.project_id + id = "${replace(var.prefix, "-", "_")}_data" + encryption_key = try(local.service_encryption_keys.bq, null) # Example assignment of an encryption key + location = "US" +} diff --git a/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql b/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql index 86309815b..0c8c20635 100644 --- a/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql +++ b/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql @@ -14,12 +14,10 @@ * limitations under the License. */ -SELECT * -FROM ML.EXPLAIN_PREDICT(MODEL `{project-id}.{dataset}.{model-name}`, - (SELECT * EXCEPT (session_id, session_starting_ts, user_id, has_purchased) - FROM `{project-id}.{dataset}.ecommerce_abt` - WHERE extract(ISOYEAR FROM session_starting_ts) = 2023 - ), - STRUCT(5 AS top_k_features, 0.5 AS threshold) -) -LIMIT 100 +SELECT * +FROM ML.EXPLAIN_PREDICT(MODEL `{project-id}.{dataset}.{model-name}`, + (SELECT * EXCEPT (session_id, session_starting_ts, user_id, has_purchased) + FROM `{project-id}.{dataset}.ecommerce_abt` + WHERE extract(ISOYEAR FROM session_starting_ts) = 2023), + STRUCT(5 AS top_k_features, 0.5 AS threshold)) +LIMIT 100 diff --git a/blueprints/data-solutions/bq-ml/demo/sql/features.sql b/blueprints/data-solutions/bq-ml/demo/sql/features.sql index 2b55185f6..a28ba85bc 100644 --- a/blueprints/data-solutions/bq-ml/demo/sql/features.sql +++ b/blueprints/data-solutions/bq-ml/demo/sql/features.sql @@ -14,24 +14,55 @@ * limitations under the License. */ -CREATE view if not exists `{project_id}.{dataset}.ecommerce_abt` as - -with abt as ( - SELECT user_id, session_id, city, postal_code, browser,traffic_source, min(created_at) as session_starting_ts, sum(case when event_type = 'purchase' then 1 else 0 end) has_purchased - FROM `bigquery-public-data.thelook_ecommerce.events` - group by user_id, session_id, city, postal_code, browser, traffic_source -), previous_orders as ( -select user_id, array_agg (struct(created_at as order_creations_ts, o.order_id, o.status, oi.order_cost )) as user_orders - from `bigquery-public-data.thelook_ecommerce.orders` o - join (select order_id, sum(sale_price) order_cost - from `bigquery-public-data.thelook_ecommerce.order_items` group by 1) oi - on o.order_id = oi.order_id - group by 1 +CREATE VIEW if NOT EXISTS `{project_id}.{dataset}.ecommerce_abt` AS +WITH abt AS ( + SELECT user_id, + session_id, + city, + postal_code, + browser, + traffic_source, + min(created_at) AS session_starting_ts, + sum(CASE WHEN event_type = 'purchase' THEN 1 ELSE 0 END) has_purchased + FROM `bigquery-public-data.thelook_ecommerce.events` + GROUP BY user_id, + session_id, + city, + postal_code, + browser, + traffic_source +), previous_orders AS ( + SELECT user_id, + array_agg (struct(created_at AS order_creations_ts, + o.order_id, + o.status, + oi.order_cost)) as user_orders + FROM `bigquery-public-data.thelook_ecommerce.orders` o + JOIN (SELECT order_id, + sum(sale_price) order_cost + FROM `bigquery-public-data.thelook_ecommerce.order_items` + GROUP BY 1) oi + ON o.order_id = oi.order_id + GROUP BY 1 ) -select abt.*, case when extract(DAYOFWEEK from session_starting_ts) in (1,7) then 'WEEKEND' else 'WEEKDAY' end as day_of_week, extract(hour from session_starting_ts) hour_of_day - , (select count(distinct uo.order_id) from unnest(user_orders) uo where uo.order_creations_ts < session_starting_ts and status in ('Shipped', 'Complete', 'Processing') ) as number_of_successful_orders - , IFNULL((select sum(distinct uo.order_cost) from unnest(user_orders) uo where uo.order_creations_ts < session_starting_ts and status in ('Shipped', 'Complete', 'Processing') ), 0) as sum_previous_orders - , (select count(distinct uo.order_id) from unnest(user_orders) uo where uo.order_creations_ts < session_starting_ts and status in ('Cancelled', 'Returned') ) as number_of_unsuccessful_orders -from abt - left join previous_orders pso - on abt.user_id = pso.user_id +SELECT abt.*, + CASE WHEN extract(DAYOFWEEK FROM session_starting_ts) IN (1,7) + THEN 'WEEKEND' + ELSE 'WEEKDAY' + END AS day_of_week, + extract(HOUR FROM session_starting_ts) hour_of_day, + (SELECT count(DISTINCT uo.order_id) + FROM unnest(user_orders) uo + WHERE uo.order_creations_ts < session_starting_ts + AND status IN ('Shipped', 'Complete', 'Processing')) AS number_of_successful_orders, + IFNULL((SELECT sum(DISTINCT uo.order_cost) + FROM unnest(user_orders) uo + WHERE uo.order_creations_ts < session_starting_ts + AND status IN ('Shipped', 'Complete', 'Processing')), 0) AS sum_previous_orders, + (SELECT count(DISTINCT uo.order_id) + FROM unnest(user_orders) uo + WHERE uo.order_creations_ts < session_starting_ts + AND status IN ('Cancelled', 'Returned')) AS number_of_unsuccessful_orders +FROM abt +LEFT JOIN previous_orders pso +ON abt.user_id = pso.user_id diff --git a/blueprints/data-solutions/bq-ml/demo/sql/train.sql b/blueprints/data-solutions/bq-ml/demo/sql/train.sql index 72ce3bb12..2c30f2e67 100644 --- a/blueprints/data-solutions/bq-ml/demo/sql/train.sql +++ b/blueprints/data-solutions/bq-ml/demo/sql/train.sql @@ -22,6 +22,6 @@ OPTIONS(MODEL_TYPE='{model_type}', DATA_SPLIT_METHOD = 'RANDOM', DATA_SPLIT_EVAL_FRACTION = {split_fraction} ) AS -SELECT * EXCEPT (session_id, session_starting_ts, user_id) -FROM `{project_id}.{dataset}.ecommerce_abt_table` -WHERE extract(ISOYEAR FROM session_starting_ts) = 2022 \ No newline at end of file +SELECT * EXCEPT (session_id, session_starting_ts, user_id) +FROM `{project_id}.{dataset}.ecommerce_abt_table` +WHERE extract(ISOYEAR FROM session_starting_ts) = 2022 \ No newline at end of file diff --git a/blueprints/data-solutions/bq-ml/main.tf b/blueprints/data-solutions/bq-ml/main.tf index e91c7424c..6ec7766e7 100644 --- a/blueprints/data-solutions/bq-ml/main.tf +++ b/blueprints/data-solutions/bq-ml/main.tf @@ -14,45 +14,20 @@ # tfdoc:file:description Core resources. -############################################################################### -# Project # -############################################################################### - locals { service_encryption_keys = var.service_encryption_keys - shared_vpc_project = try(var.network_config.host_project, null) - + shared_vpc_project = try(var.vpc_config.host_project, null) subnet = ( local.use_shared_vpc - ? var.network_config.subnet_self_link + ? var.vpc_config.subnet_self_link : values(module.vpc.0.subnet_self_links)[0] ) + use_shared_vpc = var.vpc_config != null vpc = ( local.use_shared_vpc - ? var.network_config.network_self_link + ? var.vpc_config.network_self_link : module.vpc.0.self_link ) - use_shared_vpc = var.network_config != null - - shared_vpc_bindings = { - "roles/compute.networkUser" = [ - "robot-df", "notebooks" - ] - } - - shared_vpc_role_members = { - robot-df = "serviceAccount:${module.project.service_accounts.robots.dataflow}" - notebooks = "serviceAccount:${module.project.service_accounts.robots.notebooks}" - } - - # reassemble in a format suitable for for_each - shared_vpc_bindings_map = { - for binding in flatten([ - for role, members in local.shared_vpc_bindings : [ - for member in members : { role = role, member = member } - ] - ]) : "${binding.role}-${binding.member}" => binding - } } module "project" { @@ -75,12 +50,10 @@ module "project" { "storage.googleapis.com", "storage-component.googleapis.com" ] - shared_vpc_service_config = local.shared_vpc_project == null ? null : { attach = true host_project = local.shared_vpc_project } - service_encryption_key_ids = { compute = [try(local.service_encryption_keys.compute, null)] bq = [try(local.service_encryption_keys.bq, null)] @@ -90,170 +63,3 @@ module "project" { disable_on_destroy = false, disable_dependent_services = false } } - -############################################################################### -# Networking # -############################################################################### - -module "vpc" { - source = "../../../modules/net-vpc" - count = local.use_shared_vpc ? 0 : 1 - project_id = module.project.project_id - name = "${var.prefix}-vpc" - subnets = [ - { - ip_cidr_range = "10.0.0.0/20" - name = "${var.prefix}-subnet" - region = var.region - } - ] -} - -module "vpc-firewall" { - source = "../../../modules/net-vpc-firewall" - count = local.use_shared_vpc ? 0 : 1 - project_id = module.project.project_id - network = module.vpc.0.name - default_rules_config = { - admin_ranges = ["10.0.0.0/20"] - } - ingress_rules = { - #TODO Remove and rely on 'ssh' tag once terraform-provider-google/issues/9273 is fixed - ("${var.prefix}-iap") = { - description = "Enable SSH from IAP on Notebooks." - source_ranges = ["35.235.240.0/20"] - targets = ["notebook-instance"] - rules = [{ protocol = "tcp", ports = [22] }] - } - } -} - -module "cloudnat" { - source = "../../../modules/net-cloudnat" - count = local.use_shared_vpc ? 0 : 1 - project_id = module.project.project_id - name = "${var.prefix}-default" - region = var.region - router_network = module.vpc.0.name -} - -resource "google_project_iam_member" "shared_vpc" { - count = local.use_shared_vpc ? 1 : 0 - project = var.network_config.host_project - role = "roles/compute.networkUser" - member = "serviceAccount:${module.project.service_accounts.robots.notebooks}" -} - - -############################################################################### -# Storage # -############################################################################### - -module "bucket" { - source = "../../../modules/gcs" - project_id = module.project.project_id - prefix = var.prefix - location = var.location - name = "data" - encryption_key = try(local.service_encryption_keys.storage, null) # Example assignment of an encryption key -} - -module "dataset" { - source = "../../../modules/bigquery-dataset" - project_id = module.project.project_id - id = "${replace(var.prefix, "-", "_")}_data" - encryption_key = try(local.service_encryption_keys.bq, null) # Example assignment of an encryption key - location = "US" -} - -############################################################################### -# Vertex AI # -############################################################################### -resource "google_vertex_ai_metadata_store" "store" { - provider = google-beta - project = module.project.project_id - name = "default" #"${var.prefix}-metadata-store" - description = "Vertex Ai Metadata Store" - region = var.region - #TODO Check/Implement P4SA logic for IAM role - # encryption_spec { - # kms_key_name = var.service_encryption_keys.ai_metadata_store - # } -} - -module "service-account-notebook" { - source = "../../../modules/iam-service-account" - project_id = module.project.project_id - name = "notebook-sa" - iam_project_roles = { - (module.project.project_id) = [ - "roles/bigquery.admin", - "roles/bigquery.jobUser", - "roles/bigquery.dataEditor", - "roles/bigquery.user", - "roles/dialogflow.client", - "roles/storage.admin", - "roles/aiplatform.user", - "roles/iam.serviceAccountUser" - ] - } -} - -module "service-account-vertex" { - source = "../../../modules/iam-service-account" - project_id = module.project.project_id - name = "vertex-sa" - iam_project_roles = { - (module.project.project_id) = [ - "roles/bigquery.admin", - "roles/bigquery.jobUser", - "roles/bigquery.dataEditor", - "roles/bigquery.user", - "roles/dialogflow.client", - "roles/storage.admin", - "roles/aiplatform.user" - ] - } -} - -resource "google_notebooks_instance" "playground" { - name = "${var.prefix}-notebook" - location = format("%s-%s", var.region, "b") - machine_type = "e2-medium" - project = module.project.project_id - - container_image { - repository = "gcr.io/deeplearning-platform-release/base-cpu" - tag = "latest" - } - - install_gpu_driver = true - boot_disk_type = "PD_SSD" - boot_disk_size_gb = 110 - disk_encryption = try(local.service_encryption_keys.compute != null, false) ? "CMEK" : null - kms_key = try(local.service_encryption_keys.compute, null) - - no_public_ip = true - no_proxy_access = false - - network = local.vpc - subnet = local.subnet - - service_account = module.service-account-notebook.email - - # Enable Secure Boot - shielded_instance_config { - enable_secure_boot = true - } - - # Remove once terraform-provider-google/issues/9164 is fixed - lifecycle { - ignore_changes = [disk_encryption, kms_key] - } - - #TODO Uncomment once terraform-provider-google/issues/9273 is fixed - # tags = ["ssh"] - depends_on = [ - google_project_iam_member.shared_vpc, - ] -} diff --git a/blueprints/data-solutions/bq-ml/variables.tf b/blueprints/data-solutions/bq-ml/variables.tf index 13552a385..058284ee7 100644 --- a/blueprints/data-solutions/bq-ml/variables.tf +++ b/blueprints/data-solutions/bq-ml/variables.tf @@ -20,7 +20,7 @@ variable "location" { default = "US" } -variable "network_config" { +variable "vpc_config" { description = "Shared VPC network configurations to use. If null networks will be created in projects with pre-configured values." type = object({ host_project = string diff --git a/blueprints/data-solutions/bq-ml/vertex.tf b/blueprints/data-solutions/bq-ml/vertex.tf new file mode 100644 index 000000000..061bf7dac --- /dev/null +++ b/blueprints/data-solutions/bq-ml/vertex.tf @@ -0,0 +1,104 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# tfdoc:file:description Vertex resources. + +resource "google_vertex_ai_metadata_store" "store" { + provider = google-beta + project = module.project.project_id + name = "default" #"${var.prefix}-metadata-store" + description = "Vertex Ai Metadata Store" + region = var.region + #TODO Check/Implement P4SA logic for IAM role + # encryption_spec { + # kms_key_name = var.service_encryption_keys.ai_metadata_store + # } +} + +module "service-account-notebook" { + source = "../../../modules/iam-service-account" + project_id = module.project.project_id + name = "notebook-sa" + iam_project_roles = { + (module.project.project_id) = [ + "roles/bigquery.admin", + "roles/bigquery.jobUser", + "roles/bigquery.dataEditor", + "roles/bigquery.user", + "roles/dialogflow.client", + "roles/storage.admin", + "roles/aiplatform.user", + "roles/iam.serviceAccountUser" + ] + } +} + +module "service-account-vertex" { + source = "../../../modules/iam-service-account" + project_id = module.project.project_id + name = "vertex-sa" + iam_project_roles = { + (module.project.project_id) = [ + "roles/bigquery.admin", + "roles/bigquery.jobUser", + "roles/bigquery.dataEditor", + "roles/bigquery.user", + "roles/dialogflow.client", + "roles/storage.admin", + "roles/aiplatform.user" + ] + } +} + +resource "google_notebooks_instance" "playground" { + name = "${var.prefix}-notebook" + location = format("%s-%s", var.region, "b") + machine_type = "e2-medium" + project = module.project.project_id + + container_image { + repository = "gcr.io/deeplearning-platform-release/base-cpu" + tag = "latest" + } + + install_gpu_driver = true + boot_disk_type = "PD_SSD" + boot_disk_size_gb = 110 + disk_encryption = try(local.service_encryption_keys.compute != null, false) ? "CMEK" : null + kms_key = try(local.service_encryption_keys.compute, null) + + no_public_ip = true + no_proxy_access = false + + network = local.vpc + subnet = local.subnet + + service_account = module.service-account-notebook.email + + # Enable Secure Boot + shielded_instance_config { + enable_secure_boot = true + } + + # Remove once terraform-provider-google/issues/9164 is fixed + lifecycle { + ignore_changes = [disk_encryption, kms_key] + } + + #TODO Uncomment once terraform-provider-google/issues/9273 is fixed + # tags = ["ssh"] + depends_on = [ + google_project_iam_member.shared_vpc, + ] +} diff --git a/blueprints/data-solutions/bq-ml/vpc.tf b/blueprints/data-solutions/bq-ml/vpc.tf new file mode 100644 index 000000000..c581ed5c3 --- /dev/null +++ b/blueprints/data-solutions/bq-ml/vpc.tf @@ -0,0 +1,64 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# tfdoc:file:description VPC resources. + +module "vpc" { + source = "../../../modules/net-vpc" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.project.project_id + name = "${var.prefix}-vpc" + subnets = [ + { + ip_cidr_range = "10.0.0.0/20" + name = "${var.prefix}-subnet" + region = var.region + } + ] +} + +module "vpc-firewall" { + source = "../../../modules/net-vpc-firewall" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.project.project_id + network = module.vpc.0.name + default_rules_config = { + admin_ranges = ["10.0.0.0/20"] + } + ingress_rules = { + #TODO Remove and rely on 'ssh' tag once terraform-provider-google/issues/9273 is fixed + ("${var.prefix}-iap") = { + description = "Enable SSH from IAP on Notebooks." + source_ranges = ["35.235.240.0/20"] + targets = ["notebook-instance"] + rules = [{ protocol = "tcp", ports = [22] }] + } + } +} + +module "cloudnat" { + source = "../../../modules/net-cloudnat" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.project.project_id + name = "${var.prefix}-default" + region = var.region + router_network = module.vpc.0.name +} + +resource "google_project_iam_member" "shared_vpc" { + count = local.use_shared_vpc ? 1 : 0 + project = var.vpc_config.host_project + role = "roles/compute.networkUser" + member = "serviceAccount:${module.project.service_accounts.robots.notebooks}" +}