From 50856e6951763237be2133781acb4a7714bc8c72 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Thu, 23 Feb 2023 18:36:03 +0100 Subject: [PATCH 01/22] First commit --- blueprints/data-solutions/bq-ml/README.md | 6 + blueprints/data-solutions/bq-ml/main.tf | 247 +++++++++++++++++++ blueprints/data-solutions/bq-ml/outputs.tf | 52 ++++ blueprints/data-solutions/bq-ml/variables.tf | 69 ++++++ blueprints/data-solutions/bq-ml/versions.tf | 29 +++ 5 files changed, 403 insertions(+) create mode 100644 blueprints/data-solutions/bq-ml/README.md create mode 100644 blueprints/data-solutions/bq-ml/main.tf create mode 100644 blueprints/data-solutions/bq-ml/outputs.tf create mode 100644 blueprints/data-solutions/bq-ml/variables.tf create mode 100644 blueprints/data-solutions/bq-ml/versions.tf diff --git a/blueprints/data-solutions/bq-ml/README.md b/blueprints/data-solutions/bq-ml/README.md new file mode 100644 index 000000000..42e4832c4 --- /dev/null +++ b/blueprints/data-solutions/bq-ml/README.md @@ -0,0 +1,6 @@ +# BQ ML and Vertex Pipeline + +This blueprint creates #TODO + + + diff --git a/blueprints/data-solutions/bq-ml/main.tf b/blueprints/data-solutions/bq-ml/main.tf new file mode 100644 index 000000000..2d7ab45b8 --- /dev/null +++ b/blueprints/data-solutions/bq-ml/main.tf @@ -0,0 +1,247 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############################################################################### +# Project # +############################################################################### +locals { + service_encryption_keys = var.service_encryption_keys + shared_vpc_project = try(var.network_config.host_project, null) + + subnet = ( + local.use_shared_vpc + ? var.network_config.subnet_self_link + : values(module.vpc.0.subnet_self_links)[0] + ) + vpc = ( + local.use_shared_vpc + ? var.network_config.network_self_link + : module.vpc.0.self_link + ) + use_shared_vpc = var.network_config != null + + shared_vpc_bindings = { + "roles/compute.networkUser" = [ + "robot-df", "notebooks" + ] + } + + shared_vpc_role_members = { + robot-df = "serviceAccount:${module.project.service_accounts.robots.dataflow}" + notebooks = "serviceAccount:${module.project.service_accounts.robots.notebooks}" + } + + # reassemble in a format suitable for for_each + shared_vpc_bindings_map = { + for binding in flatten([ + for role, members in local.shared_vpc_bindings : [ + for member in members : { role = role, member = member } + ] + ]) : "${binding.role}-${binding.member}" => binding + } +} + +module "project" { + source = "../../../modules/project" + name = var.project_id + parent = try(var.project_create.parent, null) + billing_account = try(var.project_create.billing_account_id, null) + project_create = var.project_create != null + prefix = var.project_create == null ? null : var.prefix + services = [ + "aiplatform.googleapis.com", + "bigquery.googleapis.com", + "bigquerystorage.googleapis.com", + "bigqueryreservation.googleapis.com", + "compute.googleapis.com", + "ml.googleapis.com", + "notebooks.googleapis.com", + "servicenetworking.googleapis.com", + "stackdriver.googleapis.com", + "storage.googleapis.com", + "storage-component.googleapis.com" + ] + + shared_vpc_service_config = local.shared_vpc_project == null ? null : { + attach = true + host_project = local.shared_vpc_project + } + + service_encryption_key_ids = { + compute = [try(local.service_encryption_keys.compute, null)] + bq = [try(local.service_encryption_keys.bq, null)] + storage = [try(local.service_encryption_keys.storage, null)] + } + service_config = { + disable_on_destroy = false, disable_dependent_services = false + } +} + +############################################################################### +# Networking # +############################################################################### + +module "vpc" { + source = "../../../modules/net-vpc" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.project.project_id + name = "${var.prefix}-vpc" + subnets = [ + { + ip_cidr_range = "10.0.0.0/20" + name = "${var.prefix}-subnet" + region = var.region + } + ] +} + +module "vpc-firewall" { + source = "../../../modules/net-vpc-firewall" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.project.project_id + network = module.vpc.0.name + default_rules_config = { + admin_ranges = ["10.0.0.0/20"] + } + ingress_rules = { + #TODO Remove and rely on 'ssh' tag once terraform-provider-google/issues/9273 is fixed + ("${var.prefix}-iap") = { + description = "Enable SSH from IAP on Notebooks." + source_ranges = ["35.235.240.0/20"] + targets = ["notebook-instance"] + rules = [{ protocol = "tcp", ports = [22] }] + } + } +} + +module "cloudnat" { + source = "../../../modules/net-cloudnat" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.project.project_id + name = "${var.prefix}-default" + region = var.region + router_network = module.vpc.0.name +} + +resource "google_project_iam_member" "shared_vpc" { + count = local.use_shared_vpc ? 1 : 0 + project = var.network_config.host_project + role = "roles/compute.networkUser" + member = "serviceAccount:${module.project.service_accounts.robots.notebooks}" +} + + +############################################################################### +# Storage # +############################################################################### + +module "bucket" { + source = "../../../modules/gcs" + project_id = module.project.project_id + prefix = var.prefix + location = var.location + name = "data" + encryption_key = try(local.service_encryption_keys.storage, null) # Example assignment of an encryption key +} + +module "dataset" { + source = "../../../modules/bigquery-dataset" + project_id = module.project.project_id + id = "${replace(var.prefix, "-", "_")}_data" + encryption_key = try(local.service_encryption_keys.bq, null) # Example assignment of an encryption key +} + +############################################################################### +# Vertex AI # +############################################################################### +resource "google_vertex_ai_metadata_store" "store" { + provider = google-beta + project = module.project.project_id + name = "${var.prefix}-metadata-store" + description = "Vertex Ai Metadata Store" + region = var.region + #TODO Check/Implement P4SA logic for IAM role + # encryption_spec { + # kms_key_name = var.service_encryption_keys.ai_metadata_store + # } +} + +module "service-account-notebook" { + source = "../../../modules/iam-service-account" + project_id = module.project.project_id + name = "notebook-sa" + iam_project_roles = { + (module.project.project_id) = [ + "roles/bigquery.admin", + "roles/bigquery.jobUser", + "roles/bigquery.dataEditor", + "roles/bigquery.user", + "roles/dialogflow.client", + "roles/storage.admin", + ] + } +} + +module "service-account-vertex" { + source = "../../../modules/iam-service-account" + project_id = module.project.project_id + name = "vertex-sa" + iam_project_roles = { + (module.project.project_id) = [ + "roles/bigquery.admin", + "roles/bigquery.jobUser", + "roles/bigquery.dataEditor", + "roles/bigquery.user", + "roles/dialogflow.client", + "roles/storage.admin", + ] + } +} + +resource "google_notebooks_instance" "playground" { + name = "${var.prefix}-notebook" + location = format("%s-%s", var.region, "b") + machine_type = "e2-medium" + project = module.project.project_id + + container_image { + repository = "gcr.io/deeplearning-platform-release/base-cpu" + tag = "latest" + } + + install_gpu_driver = true + boot_disk_type = "PD_SSD" + boot_disk_size_gb = 110 + disk_encryption = try(local.service_encryption_keys.compute != null, false) ? "CMEK" : null + kms_key = try(local.service_encryption_keys.compute, null) + + no_public_ip = true + no_proxy_access = false + + network = local.vpc + subnet = local.subnet + + service_account = module.service-account-notebook.email + + # Remove once terraform-provider-google/issues/9164 is fixed + lifecycle { + ignore_changes = [disk_encryption, kms_key] + } + + #TODO Uncomment once terraform-provider-google/issues/9273 is fixed + # tags = ["ssh"] + depends_on = [ + google_project_iam_member.shared_vpc, + ] +} diff --git a/blueprints/data-solutions/bq-ml/outputs.tf b/blueprints/data-solutions/bq-ml/outputs.tf new file mode 100644 index 000000000..2b62074b0 --- /dev/null +++ b/blueprints/data-solutions/bq-ml/outputs.tf @@ -0,0 +1,52 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "bucket" { + description = "GCS Bucket URL." + value = module.bucket.url +} + +output "dataset" { + description = "GCS Bucket URL." + value = module.dataset.id +} + +output "notebook" { + description = "Vertex AI notebook details." + value = { + name = resource.google_notebooks_instance.playground.name + id = resource.google_notebooks_instance.playground.id + } +} + +output "project" { + description = "Project id." + value = module.project.project_id +} + +output "vpc" { + description = "VPC Network." + value = local.vpc +} + +output "service-account-vertex" { + description = "Service account to be used for Vertex AI pipelines" + value = module.service-account-vertex.email +} + +output "vertex-ai-metadata-store" { + description = "" + value = google_vertex_ai_metadata_store.store.id + +} diff --git a/blueprints/data-solutions/bq-ml/variables.tf b/blueprints/data-solutions/bq-ml/variables.tf new file mode 100644 index 000000000..3bd0ca65b --- /dev/null +++ b/blueprints/data-solutions/bq-ml/variables.tf @@ -0,0 +1,69 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +variable "location" { + description = "The location where resources will be deployed." + type = string + default = "EU" +} + +variable "network_config" { + description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values." + type = object({ + host_project = string + network_self_link = string + subnet_self_link = string + }) + default = null +} + +variable "prefix" { + description = "Prefix used for resource names." + type = string + validation { + condition = var.prefix != "" + error_message = "Prefix cannot be empty." + } +} + +variable "project_create" { + description = "Provide values if project creation is needed, uses existing project if null. Parent format: folders/folder_id or organizations/org_id." + type = object({ + billing_account_id = string + parent = string + }) + default = null +} + +variable "project_id" { + description = "Project id, references existing project if `project_create` is null." + type = string +} + +variable "region" { + description = "The region where resources will be deployed." + type = string + default = "europe-west1" +} + +variable "service_encryption_keys" { # service encription key + description = "Cloud KMS to use to encrypt different services. Key location should match service region." + type = object({ + bq = string + compute = string + storage = string + }) + default = null +} diff --git a/blueprints/data-solutions/bq-ml/versions.tf b/blueprints/data-solutions/bq-ml/versions.tf new file mode 100644 index 000000000..08492c6f9 --- /dev/null +++ b/blueprints/data-solutions/bq-ml/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3.1" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.50.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.50.0" # tftest + } + } +} + + From a51c68200542780ded5bd37d05375369e0ce85d7 Mon Sep 17 00:00:00 2001 From: Giorgio Conte Date: Fri, 24 Feb 2023 13:27:44 +0000 Subject: [PATCH 02/22] Updated tf file to add the following features: - default location of dataset to US - changed name of vertex metastore to "default" - add ai user and service account us to notebook SA - add ai user to vertex sa --- blueprints/data-solutions/bq-ml/main.tf | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/blueprints/data-solutions/bq-ml/main.tf b/blueprints/data-solutions/bq-ml/main.tf index 2d7ab45b8..77ae55ea5 100644 --- a/blueprints/data-solutions/bq-ml/main.tf +++ b/blueprints/data-solutions/bq-ml/main.tf @@ -160,6 +160,7 @@ module "dataset" { project_id = module.project.project_id id = "${replace(var.prefix, "-", "_")}_data" encryption_key = try(local.service_encryption_keys.bq, null) # Example assignment of an encryption key + location = "US" } ############################################################################### @@ -168,7 +169,7 @@ module "dataset" { resource "google_vertex_ai_metadata_store" "store" { provider = google-beta project = module.project.project_id - name = "${var.prefix}-metadata-store" + name = "default" #"${var.prefix}-metadata-store" description = "Vertex Ai Metadata Store" region = var.region #TODO Check/Implement P4SA logic for IAM role @@ -189,6 +190,8 @@ module "service-account-notebook" { "roles/bigquery.user", "roles/dialogflow.client", "roles/storage.admin", + "roles/aiplatform.user", + "roles/iam.serviceAccountUser" ] } } @@ -205,6 +208,7 @@ module "service-account-vertex" { "roles/bigquery.user", "roles/dialogflow.client", "roles/storage.admin", + "roles/aiplatform.user" ] } } @@ -234,6 +238,12 @@ resource "google_notebooks_instance" "playground" { service_account = module.service-account-notebook.email + # Enable Secure Boot + + shielded_instance_config { + enable_secure_boot = true + } + # Remove once terraform-provider-google/issues/9164 is fixed lifecycle { ignore_changes = [disk_encryption, kms_key] From 3271acd2f2f6ae7907b7b4b214ac9c4b466f894c Mon Sep 17 00:00:00 2001 From: Giorgio Conte Date: Mon, 27 Feb 2023 10:56:47 +0000 Subject: [PATCH 03/22] Added sql and jupyter notebook to run the demo --- .../bq-ml/demo/bmql_pipeline.ipynb | 290 ++++++++++++++++++ .../bq-ml/demo/requirements.txt | 2 + .../bq-ml/demo/sql/explain_predict.sql | 8 + .../bq-ml/demo/sql/features.sql | 19 ++ .../data-solutions/bq-ml/demo/sql/train.sql | 11 + 5 files changed, 330 insertions(+) create mode 100644 blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb create mode 100644 blueprints/data-solutions/bq-ml/demo/requirements.txt create mode 100644 blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql create mode 100644 blueprints/data-solutions/bq-ml/demo/sql/features.sql create mode 100644 blueprints/data-solutions/bq-ml/demo/sql/train.sql diff --git a/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb b/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb new file mode 100644 index 000000000..58a1eddc0 --- /dev/null +++ b/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb @@ -0,0 +1,290 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -r requirements.txt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import kfp\n", + "from google.cloud import aiplatform as aip\n", + "import google_cloud_pipeline_components.v1.bigquery as bqop" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Set your env variable" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "PREFIX = 'your-prefix'\n", + "PROJECT_ID = 'your-project-id'\n", + "LOCATION = 'US'\n", + "REGION = 'us-central1'\n", + "PIPELINE_NAME = 'bqml-vertex-pipeline'\n", + "MODEL_NAME = 'bqml-model'\n", + "EXPERIMENT_NAME = 'bqml-experiment'\n", + "ENDPOINT_DISPLAY_NAME = 'bqml-endpoint'\n", + "\n", + "SERVICE_ACCOUNT = f\"vertex-sa@{PROJECT_ID}.iam.gserviceaccount.com\"\n", + "PIPELINE_ROOT = f\"gs://{PREFIX}-data\"\n", + "DATASET = \"{}_data\".format(PREFIX.replace(\"-\",\"_\")) " + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Vertex Pipeline Definition\n", + "\n", + "In the following code block we are defining our Vertex AI pipeline. It is made up of three main steps:\n", + "1. Create a BigQuery dataset which will contains the BQ ML models\n", + "2. Train the BQ ML model, in this case a logistic regression\n", + "3. Evaluate the BQ ML model with the standard evaluation metrics\n", + "\n", + "The pipeline takes as input the following variables:\n", + "- ```model_name```: the display name of the BQ ML model\n", + "- ```split_fraction```: the percentage of data that will be used as evaluation dataset\n", + "- ```evaluate_job_conf```: bq dict configuration to define where to store evalution metrics\n", + "- ```dataset```: name of dataset where the artifacts will be stored\n", + "- ```project_id```: the project id where the GCP resources will be created\n", + "- ```location```: BigQuery location" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"sql/train.sql\") as file:\n", + " train_query = file.read()\n", + "\n", + "with open(\"sql/features.sql\") as file:\n", + " features_query = file.read()\n", + "\n", + "\n", + "@kfp.dsl.pipeline(name='bqml-pipeline', pipeline_root=PIPELINE_ROOT)\n", + "def pipeline(\n", + " model_name: str,\n", + " split_fraction: float,\n", + " evaluate_job_conf: dict, \n", + " dataset: str = DATASET,\n", + " project_id: str = PROJECT_ID,\n", + " location: str = LOCATION,\n", + " ):\n", + "\n", + " create_dataset = bqop.BigqueryQueryJobOp(\n", + " project=project_id,\n", + " location=location,\n", + " query=f'CREATE SCHEMA IF NOT EXISTS {dataset}'\n", + " )\n", + "\n", + " create_features_table = bqop.BigqueryQueryJobOp(\n", + " project=project_id,\n", + " location=location,\n", + " query=features_query.format(dataset=dataset, project_id=project_id),\n", + " #job_configuration_query = {\"writeDisposition\": \"WRITE_TRUNCATE\"} #, \"destinationTable\":{\"projectId\":project_id,\"datasetId\":dataset,\"tableId\":\"ecommerce_abt_table\"}} #{\"destinationTable\":{\"projectId\":\"project_id\",\"datasetId\":dataset,\"tableId\":\"ecommerce_abt_table\"}}, #\"writeDisposition\": \"WRITE_TRUNCATE\", \n", + "\n", + " ).after(create_dataset)\n", + "\n", + " create_bqml_model = bqop.BigqueryCreateModelJobOp(\n", + " project=project_id,\n", + " location=location,\n", + " query=train_query.format(model_type = 'LOGISTIC_REG'\n", + " , project_id = project_id\n", + " , dataset = dataset\n", + " , model_name = model_name\n", + " , split_fraction=split_fraction)\n", + " ).after(create_features_table)\n", + "\n", + " evaluate_bqml_model = bqop.BigqueryEvaluateModelJobOp(\n", + " project=project_id,\n", + " location=location,\n", + " model=create_bqml_model.outputs[\"model\"],\n", + " job_configuration_query=evaluate_job_conf\n", + " ).after(create_bqml_model)\n", + "\n", + "\n", + "# this is to compile our pipeline and generate the json description file\n", + "kfp.v2.compiler.Compiler().compile(pipeline_func=pipeline,\n", + " package_path=f'{PIPELINE_NAME}.json') " + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create Experiment\n", + "\n", + "We will create an experiment in order to keep track of our trainings and tasks on a specific issue or problem." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "my_experiment = aip.Experiment.get_or_create(\n", + " experiment_name=EXPERIMENT_NAME,\n", + " description='This is a new experiment to keep track of bqml trainings',\n", + " project=PROJECT_ID,\n", + " location=REGION\n", + " )" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Running the same training pipeline with different parameters\n", + "\n", + "One of the main tasks during the training phase is to compare different models or to try the same model with different inputs. We can leverage the power of Vertex Pipelines in order to submit the same steps with different training parameters. Thanks to the experiments artifact it is possible to easily keep track of all the tests that have been done. This simplifies the process to select the best model to deploy.\n", + "\n", + "In this demo case, we will run the same training pipeline while changing the data split percentage between training and test data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# this configuration is needed in order to persist the evaluation metrics on big query\n", + "job_configuration_query = {\"destinationTable\": {\"projectId\": PROJECT_ID, \"datasetId\": DATASET}, \"writeDisposition\": \"WRITE_TRUNCATE\"}\n", + "\n", + "for split_fraction in [0.1, 0.2]:\n", + " job_configuration_query['destinationTable']['tableId'] = MODEL_NAME+'-fraction-{}-eval_table'.format(int(split_fraction*100))\n", + " pipeline = aip.PipelineJob(\n", + " parameter_values = {'split_fraction':split_fraction, 'model_name': MODEL_NAME+'-fraction-{}'.format(int(split_fraction*100)), 'evaluate_job_conf': job_configuration_query },\n", + " display_name=PIPELINE_NAME,\n", + " template_path=f'{PIPELINE_NAME}.json',\n", + " pipeline_root=PIPELINE_ROOT,\n", + " enable_caching=True\n", + " \n", + " )\n", + "\n", + " pipeline.submit(service_account=SERVICE_ACCOUNT, experiment=my_experiment)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Deploy the model to an endpoint\n", + "\n", + "Thanks to the integration of Vertex Endpoint, it is very straightforward to create a live endpoint to serve the model which we prefer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get the model from the Model Registry \n", + "model = aip.Model(model_name='levelup_model_name-fraction-10')\n", + "\n", + "# let's create a Vertex Endpoint where we will deploy the ML model\n", + "endpoint = aip.Endpoint.create(\n", + " display_name=ENDPOINT_DISPLAY_NAME,\n", + " project=PROJECT_ID,\n", + " location=REGION,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "ename": "", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[1;31mRunning cells with '/usr/bin/python3' requires the ipykernel package.\n", + "\u001b[1;31mRun the following command to install 'ipykernel' into the Python environment. \n", + "\u001b[1;31mCommand: '/usr/bin/python3 -m pip install ipykernel -U --user --force-reinstall'" + ] + } + ], + "source": [ + "# deploy the BQ ML model on Vertex Endpoint\n", + "# have a coffe - this step can take up 10/15 minutes to finish\n", + "model.deploy(endpoint=endpoint, deployed_model_display_name='bqml-deployed-model')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Let's get a prediction from new data\n", + "inference_test = {\n", + " 'postal_code': '97700-000',\n", + " 'number_of_successful_orders': 0,\n", + " 'city': 'Santiago',\n", + " 'sum_previous_orders': 1,\n", + " 'number_of_unsuccessful_orders': 0,\n", + " 'day_of_week': 'WEEKDAY',\n", + " 'traffic_source': 'Facebook',\n", + " 'browser': 'Firefox',\n", + " 'hour_of_day': 20}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "my_prediction = endpoint.predict([inference_test])\n", + "\n", + "my_prediction" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.9" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/blueprints/data-solutions/bq-ml/demo/requirements.txt b/blueprints/data-solutions/bq-ml/demo/requirements.txt new file mode 100644 index 000000000..829539737 --- /dev/null +++ b/blueprints/data-solutions/bq-ml/demo/requirements.txt @@ -0,0 +1,2 @@ +kfp==1.8.19 +google-cloud-pipeline-components==1.0.39 \ No newline at end of file diff --git a/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql b/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql new file mode 100644 index 000000000..0d67bc7c8 --- /dev/null +++ b/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql @@ -0,0 +1,8 @@ +select * +from ML.EXPLAIN_PREDICT(MODEL `{project-id}.{dataset}.{model-name}`, + (select * except (session_id, session_starting_ts, user_id, has_purchased) + from `{project-id}.{dataset}.ecommerce_abt` + where extract(ISOYEAR from session_starting_ts) = 2023 + ), + STRUCT(5 AS top_k_features, 0.5 as threshold) +) \ No newline at end of file diff --git a/blueprints/data-solutions/bq-ml/demo/sql/features.sql b/blueprints/data-solutions/bq-ml/demo/sql/features.sql new file mode 100644 index 000000000..024343297 --- /dev/null +++ b/blueprints/data-solutions/bq-ml/demo/sql/features.sql @@ -0,0 +1,19 @@ +with abt as ( + SELECT user_id, session_id, city, postal_code, browser,traffic_source, min(created_at) as session_starting_ts, sum(case when event_type = 'purchase' then 1 else 0 end) has_purchased + FROM `bigquery-public-data.thelook_ecommerce.events` + group by user_id, session_id, city, postal_code, browser, traffic_source +), previous_orders as ( +select user_id, array_agg (struct(created_at as order_creations_ts, o.order_id, o.status, oi.order_cost )) as user_orders + from `bigquery-public-data.thelook_ecommerce.orders` o + join (select order_id, sum(sale_price) order_cost + from `bigquery-public-data.thelook_ecommerce.order_items` group by 1) oi + on o.order_id = oi.order_id + group by 1 +) +select abt.*, case when extract(DAYOFWEEK from session_starting_ts) in (1,7) then 'WEEKEND' else 'WEEKDAY' end as day_of_week, extract(hour from session_starting_ts) hour_of_day + , (select count(distinct uo.order_id) from unnest(user_orders) uo where uo.order_creations_ts < session_starting_ts and status in ('Shipped', 'Complete', 'Processing') ) as number_of_successful_orders + , IFNULL((select sum(distinct uo.order_cost) from unnest(user_orders) uo where uo.order_creations_ts < session_starting_ts and status in ('Shipped', 'Complete', 'Processing') ), 0) as sum_previous_orders + , (select count(distinct uo.order_id) from unnest(user_orders) uo where uo.order_creations_ts < session_starting_ts and status in ('Cancelled', 'Returned') ) as number_of_unsuccessful_orders +from abt + left join previous_orders pso + on abt.user_id = pso.user_id \ No newline at end of file diff --git a/blueprints/data-solutions/bq-ml/demo/sql/train.sql b/blueprints/data-solutions/bq-ml/demo/sql/train.sql new file mode 100644 index 000000000..0f5517b8a --- /dev/null +++ b/blueprints/data-solutions/bq-ml/demo/sql/train.sql @@ -0,0 +1,11 @@ +create or replace model `{project_id}.{dataset}.{model_name}` +OPTIONS(model_type='{model_type}', + input_label_cols=['has_purchased'], + enable_global_explain=TRUE, + MODEL_REGISTRY='VERTEX_AI', + data_split_method = 'RANDOM', + data_split_eval_fraction = {split_fraction} + ) as +select * except (session_id, session_starting_ts, user_id) +from `{project_id}.{dataset}.ecommerce_abt_table` +where extract(ISOYEAR from session_starting_ts) = 2022 \ No newline at end of file From 17b8a461f08dd12dd3f2d4fa8c0a5d661fe4603e Mon Sep 17 00:00:00 2001 From: Giorgio Conte Date: Mon, 27 Feb 2023 15:28:49 +0000 Subject: [PATCH 04/22] fixed notebook with dynamic model name cleared output from cells added creation of view instead of table --- .../bq-ml/demo/bmql_pipeline.ipynb | 19 ++++--------------- .../bq-ml/demo/sql/features.sql | 2 ++ 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb b/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb index 58a1eddc0..07719fa95 100644 --- a/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb +++ b/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb @@ -98,7 +98,7 @@ " query=f'CREATE SCHEMA IF NOT EXISTS {dataset}'\n", " )\n", "\n", - " create_features_table = bqop.BigqueryQueryJobOp(\n", + " create_features_view = bqop.BigqueryQueryJobOp(\n", " project=project_id,\n", " location=location,\n", " query=features_query.format(dataset=dataset, project_id=project_id),\n", @@ -114,7 +114,7 @@ " , dataset = dataset\n", " , model_name = model_name\n", " , split_fraction=split_fraction)\n", - " ).after(create_features_table)\n", + " ).after(create_features_view)\n", "\n", " evaluate_bqml_model = bqop.BigqueryEvaluateModelJobOp(\n", " project=project_id,\n", @@ -205,7 +205,7 @@ "outputs": [], "source": [ "# get the model from the Model Registry \n", - "model = aip.Model(model_name='levelup_model_name-fraction-10')\n", + "model = aip.Model(model_name=f'{MODEL_NAME}-fraction-10')\n", "\n", "# let's create a Vertex Endpoint where we will deploy the ML model\n", "endpoint = aip.Endpoint.create(\n", @@ -219,18 +219,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "ename": "", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[1;31mRunning cells with '/usr/bin/python3' requires the ipykernel package.\n", - "\u001b[1;31mRun the following command to install 'ipykernel' into the Python environment. \n", - "\u001b[1;31mCommand: '/usr/bin/python3 -m pip install ipykernel -U --user --force-reinstall'" - ] - } - ], + "outputs": [], "source": [ "# deploy the BQ ML model on Vertex Endpoint\n", "# have a coffe - this step can take up 10/15 minutes to finish\n", diff --git a/blueprints/data-solutions/bq-ml/demo/sql/features.sql b/blueprints/data-solutions/bq-ml/demo/sql/features.sql index 024343297..63b79d821 100644 --- a/blueprints/data-solutions/bq-ml/demo/sql/features.sql +++ b/blueprints/data-solutions/bq-ml/demo/sql/features.sql @@ -1,3 +1,5 @@ +CREATE view if not exists `{project_id}.{dataset}.ecommerce_abt` as + with abt as ( SELECT user_id, session_id, city, postal_code, browser,traffic_source, min(created_at) as session_starting_ts, sum(case when event_type = 'purchase' then 1 else 0 end) has_purchased FROM `bigquery-public-data.thelook_ecommerce.events` From 32808f93ea14034980f2d061806050a4bdfcf0cd Mon Sep 17 00:00:00 2001 From: lcaggio Date: Fri, 3 Mar 2023 14:52:33 +0100 Subject: [PATCH 05/22] Update README. --- blueprints/data-solutions/bq-ml/README.md | 64 +++++++++++++++++++- blueprints/data-solutions/bq-ml/diagram.png | Bin 0 -> 57434 bytes 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 blueprints/data-solutions/bq-ml/diagram.png diff --git a/blueprints/data-solutions/bq-ml/README.md b/blueprints/data-solutions/bq-ml/README.md index 42e4832c4..b6f2bd2b3 100644 --- a/blueprints/data-solutions/bq-ml/README.md +++ b/blueprints/data-solutions/bq-ml/README.md @@ -1,6 +1,68 @@ # BQ ML and Vertex Pipeline -This blueprint creates #TODO +This blueprint creates the infrastructure needed to deploy and run a Vertex AI environment to develop and deploy a machine learning model to be used from Vertex AI an endpoint or in BigQuery. + +This is the high level diagram: + +![High-level diagram](diagram.png "High-level diagram") + +It also includes the IAM wiring needed to make such scenarios work. Regional resources are used in this example, but the same logic will apply for 'dual regional', 'multi regional' or 'global' resources. + +The example is designed to match real-world use cases with a minimum amount of resources, and be used as a starting point for your scenario. + +## Managed resources and services + +This sample creates several distinct groups of resources: + +- Networking + - VPC network + - Subnet + - Firewall rules for SSH access via IAP and open communication within the VPC + - Cloud Nat +- IAM + - Vertex AI workbench service account + - Vertex AI pipeline service account +- Storage + - GCS bucket + - Bigquery dataset + +## Customization + +### Virtual Private Cloud (VPC) design + +As is often the case in real-world configurations, this blueprint accepts as input an existing Shared-VPC via the `network_config` variable. + +### Customer Managed Encryption Key + +As is often the case in real-world configurations, this blueprint accepts as input existing Cloud KMS keys to encrypt resources via the `service_encryption_keys` variable. + +## Demo + +In the repository `demo` folder you can find an example on how to create a Vertex AI pipeline from a publically available dataset and deploy the model to be used from a Vertex AI managed endpoint or from within Bigquery. + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [prefix](variables.tf#L32) | Prefix used for resource names. | string | ✓ | | +| [project_id](variables.tf#L50) | Project id, references existing project if `project_create` is null. | string | ✓ | | +| [location](variables.tf#L16) | The location where resources will be deployed. | string | | "EU" | +| [network_config](variables.tf#L22) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | +| [project_create](variables.tf#L41) | Provide values if project creation is needed, uses existing project if null. Parent format: folders/folder_id or organizations/org_id. | object({…}) | | null | +| [region](variables.tf#L55) | The region where resources will be deployed. | string | | "europe-west1" | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [bucket](outputs.tf#L15) | GCS Bucket URL. | | +| [dataset](outputs.tf#L20) | GCS Bucket URL. | | +| [notebook](outputs.tf#L25) | Vertex AI notebook details. | | +| [project](outputs.tf#L33) | Project id. | | +| [service-account-vertex](outputs.tf#L43) | Service account to be used for Vertex AI pipelines | | +| [vertex-ai-metadata-store](outputs.tf#L48) | | | +| [vpc](outputs.tf#L38) | VPC Network. | | + diff --git a/blueprints/data-solutions/bq-ml/diagram.png b/blueprints/data-solutions/bq-ml/diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..1effbebe280f678442884a58e2dd85828f8d096a GIT binary patch literal 57434 zcmeFZbySpH6gP?piXfoUji4xvz<@N0fTYyWCEYa)9fAVVQqnn6LrZrFC>=9&h)4}3 zjpX+L`o8+!@BZ=KweDK?uJ2(J;>+0e*$A zQ2cW(apfV})rhEiG((^~@EIEvT zj~jN+H6Unc#P`pCuShA~`vH)NW3Hm^s4geVZ)j`H_R`4Kz?jX&+U~p+nxG3maA|Gq z_>$Je+R6sP?;=EZ@dQ6`eSVvrj`rdaM@u0(bvXrEF?S{^nIHV!&rY+71cK?frf zex+v;znTO83DKE3I@n+Vg&28rVV|h3M$c2m0sdl1@i+lYd9Df&7{lFhTb7Z`dEP zaj^f-ni;#8|KBt_|K_sU#k?+u6Fl#XU&$C^YXv>03T$KUD9kB%F~)yy|4XFcd0+er z<}Suo>d(xrjcp)6Q{hLPAPzzH|LIr%)215#HU)9<{Mqu4Z+^8DWIt!uAMCqan~S@^ z;s|34vj4Nt!r1lZq=RT^qG(djo~XE7S)ID>LAmI8_H%8Cmj8(_hC4~#!$87Zi7)TL zEP0O|1SSGupTm+NNotX`pMGWw6rCkU9S~I>L)gCPSbd3P?fhs2mihQBkWM_}PO$nX zd9-^2?o&MbIyWC{ud@+T2{d`_-5+{wn&P?;zt2LvJ(10a+iLgYDcTi`TcZE|X`)7F zb60t-N&5;79qW%jw{FwQ&U^mnrat|Z78Q!An{eO19~O=4bpN-@i>oFHbZQEuA6(Yn zhojV9Zd2g-?(KhvjWfCu%hi12hB86gJ8piTVPTknXbwNWVAvnz(XL@&b81&NV5rGu z&tN>fclU2%K1jdf0D-rXNdBG=tqxp!z~hAr*&XQ7V z=BhZBAEyI5DmnaAG8eE%9IIwE>Nc??{q^CWUBqDlT?8s*cH7^2m<*Ck&(%=Svwfij zF+BMR_YVfC>6DgxtoxRv(l+Ki7T|6CV6`_I9K@IoD|ek)VK-S=u&&u%kJCAR`{&w= zVc*KalX~Wlz*1q;)!^O^@*jdnNs5d%F7{6dY?%-)d?yJh(e%rl*AjM^A;ppMsCtQk z?qy5+&V@bqPtkg(@`^D!a;tbq9z7;lExI=)j~q0t!@Xj>)7ZV1vR=3zwikEk@B5_O z8Z{HQbI=Oo%0U$Fse)!D9<*CAzi^h+@}SNPz!i%>IulW9jJx{t_Mh?`iPN8uFd0z4 zcjw{5_%XZmn)2Cib@!Q7N=H(?WY^x0Ogk*j*4sKk%S`9L+tklpDQ#!PE$^6iug!zW z`D6?&clWTXH+U&Ok&1^3k@ThvDjJtA7S?+u#*kwB$y0hPwp1&d7$ru;)Z4X<^Wns5 zl9kPFtOy*gYso=Lx&_CoU5U11yY(70Yc1qyL=M;4xJvy;8tP6O%RT3&@J)MFM z>UrWfPr=itQGD-kbjjK;*yli2cU!PVyG&50ZH?PTMYOA&=P}JBz zF%_exuC1*~{o{y&O*Pf!Y-&1%&9501ntxDTkeORvqWiN)eSB@ck|iPD`VV39zxDNs z8y2n8gd{EL4_@sYMNjU0?iy8#JcN;WgG`B~?Th`u*Y3&kBQ>lOL8f@!?>_L1Uaj?V zKW?J#>E>3SScr#RCxX52OZG;V^TMyFW-nP;x*j1-`@HPTdNYg{f*wD>Z_FkwH)#GA zqh|YPrGI3qe6f6yh_p|m%1XjM_jJX~eEg)d2qZEzSUw^#67?r>w@Aa#43z_5&r%4n z4vyRk&T7Ud*i@6JYv=Amzkwf2#DUCJbxM&zir@iA#|*xoZ|NfYM+24naE1 zAjB*~;TFZ^cgFtRY(&v~6amTOx8TQ1Quh&aclwl?mX_nV_TE}oPhzKVvW{#(q@t30 zx#K|GvU8EG&SkxI!*==96j6LqF039*uLBYup4-(R zD^sj?oL_at(V_NA%m9x{{JUZ3N23d>hu;ZnWzZBYG26_2qYKV6$#ji7itm>hUKXC80lN3B@eN6HOnMc%G+rYlS#~#L348h z(bRf$jDIRS$6K_QC|ZttFEplvCQCSx^S zbMgiC3lb)0vMmETzJ{DX=>Oa_LUi=so<)z~9=aTVrCu3rm`)jz#5cR0;QLL9YalT`6W3cWoM=Wf{%d=47ixlQy|(ir zx3I%UCI;1Kh7oB*$^{x8Bn9JXf{T4gt7hhGEXhRJhb!!=NAB*6tQrw&3P$hq&4y#+ zb3ZxHxG+AjZTBUzysniCElzkB_ODqN<;MYl#r9Nc#fY%!?BL|FFaBQRl)Ag_S`I2V zkr!q+cDRBa)D@{++A<(lIR*LFKRu2r1!adbXPXeQr3!d2qe3-v+65Knbc$+s*I^s$ z7Vz59X*c!2>YrT11=>Yi60sgg5HIEE?!>~O<_SrBRBodui2v%}3laA!$}8~oz&l|j zp9Ni*00&11gbqch1#jRL-V+r%IsRjpc4$j zWMJdESI34c2CiXd^8j&a^<>`&2N4i`Vi;R-tnN0CkNp0UsZvg5#~fu+I@ze}YA-`x7|^DN&dN=!V@xQpB4kW1^TSM^*XP|1JO;dMg2~ zEL93YbykF^la1~R(-gPwy@2H_W@{c=a;PT!!xCr3p(#ID@E!h-EhHkv>8#9PI&XfNUQ>(|Sl{A)Xat3ho5nfA<}X$${3Jz9MLf2g=m z;amJgCf0{)0LvvbUa9iERt=MX@*xV^91{rpqoUcIUZjL8~Q zBnS7eA>SxiL()M5qntPpGlKn~!!(l3!C-(*tu)()) zIQlzCMyGZIgEqYb`V_kpapjyYeNK#zIzqyxGe${)5a7Fm8;*XE_971H^F_Ocv9!~r zb@AN4`~jScH@Q4DWGa|{%?kkYG5dcmA28CZP+(Hlj$Ap|e@*HY+DoR5zZ;`np*Ww> zl8FM*Uo%1jI6i&#FHWJM^8g+~`9Mz^(|_*2$*sHoAJ@a`o@kr}Eb$5WKlibeUGe|4 znOT!MqQpwC+b@BMP#nsP9K4OOng=VD#=?&QF2M(a0RY;1-%CLx|J3CVm~ z`|a!W@$n5WCM&^YdsKqZt(~^GBr;*lRM+i=_pEwKw*m8)@)$aseDdJv@S3ky?2v{^ zT;{3iFh`Gx1wPObqb?VA^C2~uE*Ii2}Q{k7$8woI zQa{Y3mjx2Fsa9KP2?p|7vb+J6*-S@B@^$z$|WPz>dH(;=!NQ z`=y?w_sy+*M_@*+qp7pezRq9B^mk>0=%fwc~JO;|Sr_@?GnN z!tolDp1$bVL{Mc0w=l$5Jn&8*lDmC_?zbV4xGS1HqnKuob#5(Q$zgJPZ?w2_g{S23 zK?(~!Owz$JnDJ0tYx!)*tgi#}qc0MAMSAX~r5+|JdtubpW-0e#Zh0dPK{z9Yy6^NQ zJyex`DZM@^dRZ3U)j*aD)+`J|3Id z;m@=G;!T8>rnvLN9N*ytdIEG_;s(K&)~Z zL-EOF6piBtZBUe7z*tgkV6_~28uI!PHQXxSj5n`SiMT#FYx|Hluv?Kh z@}97S+~VdQAOkFORC+hx#k?7m^#^<%-*d{s93&RFl+bt^v6j+ zua&pS$dP1U+h5WW$B)g8ccIh8wM})^nY!V%haND#brl%6LuoGF*bkG-0l7S~Dnh9f z{gQOif;>8dOCq~8^w%~8BL)Bli0b;KeyE9V@AT%e40YFQih;`!X2=o*IJn13yHeqP z-EQwAIa({COw!%ZDM@~Q_x4duOQL@e)bm#*xplY~OwY=GrK#{Dr6I?XnVtD_PvFit zRhOlW`0^nVXRpzFuWe?XovoXhwRdB6+1diNw7xua$V)R;kpfljXz98%4Oq_r1v-PbD)OYg~k~hT=!Bo|bOenbEH7 z-vJX2%p_Ft8%*>$y8cN^$LlTvp~3h4Xy1jQ8&6+LEwz=_b{*vypQwd6aDtAZ(kgM% zbR72RIYpoX?vs!@y& z6qGbg7WoOX#(%CWT=*(JUx-J`7f&Y1jH|1wqdX0I1Dc*=SXi!@#F%K;+V?!iquVgT zs;k|u3|>^LnOgtym~2s0zYN{9FC|wSy6fu~EUDG>hDI`ufA&+-1r_?(HwcM|^T&#F z)6(>@k@4`XbWt!EEOS!AU1xWs=b-C#x|>+E?c&m2aaJ$ALQYKho%gyGYtyKT)(kaM zz+XuILn7ee)J)kI{vGcx1~34;z!pD7?DxDcY~Y|A;5|*PKOXrjo`*H63E=DD%7D(l za|z1~KwRI(IceqZc)t}Z0Nht5#O8nP;g=)$I08PQT%|zo?l0@}i_8kZl9%tUr8`{g z@W20v)|@8?Ca4zO{&t?U00TRIxTnDPYoEKcU9?jGKpR>MgVg_Sf6@7?0sxd+d%EXP z{^i5GvjO(nWv7nnzvKN)!@%u|1|gLTYxxgbp7R1upJVNu_TTY-eM`W-`G3gn=huC> zpzS3%v&9j3Bb}gM2%ro=W&IN0e*wU67;I&4yf>Y`rh1L-ccv@QKF{H|AB+w;pBRTR zhM;m*mI~p8Bj#iN;1;$39+*Pl#Wj9+sJMXV8L67(Gyes-#TOHK=NBBb7qGg_-`J`ha>gfo1LS#J8lH~)sa!2~FhuPwR z(;CTRU#!{-?9i}dFety`fO3}e{R%#)VX-hdpEmPs^i4a|dGe30g|AvHKRTTmIoc}a zmtJl#PLCgh;EJJa<=ep zu04Aax+D_-;glY2(O>w095(1~r`js(hux~FBWoVp&zeG1*efVRJX`gm=d{$<9dvB^ zNGqe1ZUL^R@>gxcgSOW`?S(!Ih)C_l8tGU!Em;Ta*DpJYW^h$_@l^~>Qic|H(BOHu+? zx=!jaKrrm0KVL>Jn&L2g@51UkZe#I#=t_FMAM!LV{p8EJcy+|>(6@bksJTJ(`?COc z|CDi()gUsq^f!iy}I-w6Xj)-)sF84ywh^^ z)9TtvwP_}n)|MXOCE@!$jrbHBjuG4EG*!||Krj@Rflhgz_6(74T64_d8-dL8TEdu_ z7IGFA7Jwux0~gmej^2T*Z7G7Qr z{*3@iO?CimM)V9i)1@>eW8$g!Ij3SfU8l4Nd)R5u10|>O3{%6uE|2sF^%U)P_6@H# ztcT+0AVX6YoLjA@BCsmp_m8_9n$(298NPvIwgCaZ5Jgcc?yZkWm>LmM+!)NZB9~Wt%kRDtrMuFWbN^>AW+t1bzVy&5L0&&B`%NtPXy8NS!M~R!5P4I#V1A3yaEj7L?y|F2#Mlq+*U2kBWb1DD_lD zG4rW1$Y&Udtlt@w3DN|A9WKzC+s1#t^_f2Wp^Bs;7Ly=U3#Qs|uaeZm5B{5nPS@$gq=B*I#c@zGH*ea6+%O}I1dv6vQ zI;9JcI#5^N+QYvoTjlZYS{Bo5Lv&$JL`CipAt9l*@Ys(E-{!BSeZqNdMvXg}?D5tG zL0qvc8e8mVTkMTLdu+Lhl%1UR?4;pvxRN%UcqD+vOFguN|CyiJX5#*gxA)_KDAPN@ z2KeAgyz)f=>_d^Lec?*T1Xe%@uP_#}(hnajX`ou3ao8H0s3e!XU^7=4@Vcv0M~!`F z^_X_jgKi05Vyh0ysWA1ufaT>Ii^GQCDhG^vX~ucO-Bium61HBqf=;6CAAjW1ZmSO3 zVhgsAZSF>2%;gVg^iJ695}?G3?fM+_Q-alPpR@v`q$24qNFVmBH;E5c9a~s~f-jvm zVLS8MOy;=04yWYH2%Sa9@Wsg02ltR71)qhMyvc1HNK6u>_kzgEmp{h2v^PUyUlTA< zpC~L)4oPtQWLx{4Ym{!?EWLn!^FLAJ?a7HxobT+h%URiIM(G=xeo8Ct+dLfTH?{rx zVsFPq(vAo#L`ihh$bK5DMJ0fX@Tc@@XKaQ%u2iV2aTU^lJQg&#+`7CgE?)L1Aaque z>oNChmZxC#dLxw960@_z{^y0Fzv9%4CHxdA&ANR=ou5oyd zY_~y#?QqXhI;G7zb zq*$`CXU#_=@~FGL64z)HCbuho9~^5FH%pDPvbWKbVy6m~h9!{SvCotSw{(gd*t-(q zvQ(1A31}Igsl>snqr0)V4iPT|H{X~Xdj4p>4s+j2APWi#D!WY+mcY0$y_9inf+wKO z$PKDzh*B`He??VQED7buz1MFtVy&31j=8={*e=ZnTFB=SN^AOvzn=6FYe|@@r3h zd#g*+C8+SyB16N4;5~oBvru*76Z-cL5YyQ6d{UaNyR)b+0jqr$)RgsfI6RKO4Htpik^4F=-9mV z)TUXs>Q~^5U?66eW|U@Fynro?5D_UBx_#nxRt;I~F45ft9anlERW_a-HPW=vmK|h7 zW+M=YrDq~XV|~}UvfNOL;4_BqG!SXIbMZX=Za?MBE0{kml|e&UWkrRm*c{vaXQv_y zPxxkCrJm_?Vj_yvEcKFea4kO_1qVD$7+=#?_T}?Jj`0qykjCM;7@+pk@jXAU>^5w5 zGH1U?t>{f_f&vpxS1{W}AvM8B=}Ml3p2TdQT>g7*h4kWy*YT-Ulu>YSUi8qKQ=YiW zBhwuPWOj;MC(|J2*LvhS1DK z{TPNDJluMwV8g134F%J$&AM__5|ZJHir-ZP1=UmRU#K}Aevf*=B*+C5fKF>&;-DgG z&*5sqQ^q6;mgCZop`K#+Rs4#PpMUI*=#4}YWoBl2Z$0q7Bht21*?3|tRD6omJ3)q! zaTzedaE^y78#15yH4nt@g`XXS^P9^qEYa-doQ(B(@$mauW318w5E&7j@PXm**1(7b z!Qmi#f-o~-^y!kMh-Qsr?t-WijXq}5AylivYD`k(aG<%@YomVu^y_tz{=r;LuXb%- za6wU!d5q18t{$WZ1O(ph>p4$B57i$-Q7Vp~;gLig({`=6MfH2*`;BKOAj@Gh>1=%A z1ARh6wnn{0{2%gf5#%;OwNR8M2X=_4Z z-mVOGQwAd)McXE(?Zc*YUc+1XFGNntQt)<*bM|nzHGVYtTn!Bk#TVL9ICd!w%xQht ziI7cq^QzzO6$tQNovwH9=|0-un5lCug{DzM3Vvv*S+m*O&xP`8l9}`*7+aV6G!fqE zA2Avu;Nd_zC`4uX`wMS(b3Kqx>MsfxmgU}E@8O;CI9jXB%N^<@E`4O$n^fevW=Y-% zpOHMVK|RFG3v$^lic$8<0YUfIr_P}iKU$u;=R-z|a!xZb8gvwOzHQO8$F|)3K)|S4 z!xoWv!GPokGy5d#o0zYAI}J?5OJwBI+!iR>WGpi^ zhmtUGVHK=(M#E|0HcT@#*k|4XuO->vG;Ga=X!yxH`%*(-IpW{M${f?F*>>OU`>}g8 zYf^rzn6OcvgHz)YHF%+vQ>8sV;3_$ZlnFfygNwuKIOT7~}bCDQU*meErx!Z^8>>!&>R$`D&YZQ9mr zJ8?4*U*x#hi+A^lin_WPC_{5B+5rHO%dv_&_f+z$Pvt!L0Bl)1|6a=xb$oo>yrpjZ z@zIy_IPcBgCn{@!w!Uk2jD-z>s74V;9AFsG*uIx zTcp9S7|{B@I36Oq8qZ>)X1jOzg`!47X9SBLtUl~=-o>~Ek4Vf-$1tqUPY*}h*0{~# zCvS989-hc~MGLFn>1`#{2^m-HS&q&i7^`^QgweRXY8QyO%wGCP-R=*jV?FRT5^axrZOa*$woH}A8`lp82x)uVLo^UqVS zCN=8GejCSKJVDjr3JUN2H)lgyf;1W{jJ-pKqcIvCa}O|W-993!isD)jbS36P46F?4 zXb#18@)w}F@KN3Z0JhesQKzOvy8xm@Adx3MGski^Hia<@fT*}%f+oM-sf^=-%6!=BVv_z|;eZ5S# zU$vOx9!coF)PJ|fGb2YSJvTv=D9qwRPKGK`6?A3z-u3oXrH6S68rIc8@Zo4(BIst{}Gz-TUHdv{sU-&vWZVurM z6L9hfXA1A8HwjJ%*?=mJ6W*zsj5WaqC%`Y`9QtBDY>mWsLlBlG4J-7Pf~gazWM$Hy z0(q{{g7n9lqcqfU2_1Qi(T92xqhwfQnluIn;<>EOBe`G0b+<<<6OBVA8W+a7$TvZQ zcV1tDbWugv0QYC*Cs_>N5oPx+6p)m4sz<1IZK_}}qnnDeaOexIMz52|`!a}@Tw~Gz z!T#>FSIUsGoVm?&$LhQCF4IHsHBcr>w4Ay~t5W`M4f2gZ_fUlO0eW~NGW+b#0*yoL z)3JlAJurzHnV--!GOmtjzVC(GR3%*E9nnqUK=_9e@GfT`MqIy)?4)DX*?=@9GCFM< z?X#L)&OCDJ8UaDrN~aLLlEdPVY>owT;@9Q>Q0U(~eaiRTWL{wLyZbdw@qmsGQf`n# z!!GKxCVTnV^e-xeuJ6C|*{Uae?W??&1yoR=f_Cge>a^Jw{i_SZ^S7sqCkxVz=Z-jE zi*cz_DSja@h95sFWn*khB4spr*?RAOVo!B~J0pKM8Bb*;e%_jTb9T+CMS%*GPq*9x z+`^}VFv(*oP%mlG)Rx>;At)SHLuhc@D({SqQP;krYdYKf(#5I2j&97Qv;B*K#Lb*~)UgqO$C4av zT$s;P>?7*i2ub@@tAj5sXA4k#H;4F^ zw>A~qqLNL|OV7zQ6ojoNuDJSH2w9Q!XJrZqgFUZghaUY2TNY%JG%uB*hUEjsmwWciZCsjgox>M&3K^QJ3zV#%Gp zWP$xIz0<*Q%l1L`x*sjC@rBLqfAndp!cqZ|ANBe1vxf?)89grg1p2r<(hj3WecYUm z*H`Xn3V5OW9@53xhs0s{-fAR>k|7&9kM+4ts}*8jq@T7h5s2!vZs?-s#^fC}Ps_)o zPK1m6v3(9jId+G1-nNq5G*b6UC2E}XGzp$Emp-&ZG9o(R#!TOe3P*}SR&-ysgS9J* zwmp{Mw~KrU^cM6P6VRmYX0)i_BFZ$$3%W0T4lp6fRO!fMUYq_ek3%;vDK2y=B5$Es zy~mc&JLq<395OQS9Obf;DYlUHRDx?qd}=|i8@-{lY2YrwK)7fUiNG>AKy+E4i_@wl-C1t7l?j(lUHCV-9oEL~h1y7zN?v19dHha#5Qx^|Du2b+4SLm+mijz`9RCG^P;d83?s z+uE}6sGsS#e=>!Upq~IZrtbD-grC&Z$10V6!dbkVtA-${5a&9*b#Vj_F7!C>q3j-7 z>^u~R$(5E(mgh`nc60`MnYRt>PrGl#sOb)f4*B7bBxEa_HFzF5^`zzb=^A7RbG6pF zZb$1`zHc}?JsSR07qpM~kej6vdK*+~&Y*igg&!*2)fC5QTfh6h`sHb!xN7-v2ss~U zKXSNDF~dZ{3zeIlv3(Qb4j47SP|aBu)e}d}Bt4KCy4}HE1AA6Z%6#>u#m(_0FO?9t zV*VTssAB*G@dkTof+93LIjvl4_awUEE<88)5&M&w7t8cdWx(-`2m&`9LK|-&=6^Ip zZ*fsW{w@nWppSVy_*0IB{=*&!(EB%qjPcdXa->{4z(lNe^g54!n|QpqZ(qeBCmJ{Ql|T zNTVu6aF(S~wsw66j!NG&g-z9yl6k2mAR=( z49t_;f|_7`g;7x!)FwS zC~cNGuW2ny7vwJRYt8ABd49>9`8ab_Kcm&)Q4Oy0zq8c(q_*LBo;@*;Tr%zSO!XSi zprmLi7T(Qp`BcjV=?Bl#sz-{m4&&046auL*cArRs>vm`0MwdWRa9|gVM-SwT z)w8nOzDCLY#Pc=MEO**dmEJ;Dl_>8f>N=^2OjL|=_#^)EVSlMgRk?D3as-W5?cvsT zy)ow(ii_fa4>KTkW&M%7z8U9{wb%7|QOs+C)NXtAqC14rD@;0b0)CMOh-&eO^ebbU zUpt93jH5a0eB=1xt%LJG^G)v`*S*t<0?z%ZWS{A4StRx>f^|js7ERNouP#`xYDBzIP7?>g`$jJf^k#45Mt9Rvw)AG8-UdZtFWPuzSqnm~1bH zdmkG@mKoPhX7Rmqyw5dp^0;4t^k-D+@iVcT`By>MqCzbBUX*|{)<+~Z4e1ji7f4fA z(-GE2}kh%zmLIKuR@i>fN1Rwp=Bxp~uBh?W=ZaN_4lNgr##%L5KZ@L^uOO zph!MPM=tmUd)XU2aKK5kPj)vh&g9P$nb39`X`oD0*7S;8rF&1tZEkbnN)gf-bET_t zNKx{W!43sQAOpt3#yaBWn%!k=6xv~B7b{a&mbs3PTFGlh4-!YtPm>42Pb`wSA)f1y zRGe$C$vX>a$_vbfR~Iw<`)$3CCdj*LtS5MgG8h|(JQ_CJDWugZWK7>4pAEebsW)iD zmV;)QsGsEA-e^2T77;EnaU+4H;guot+AB>pSN8#7PlYHF4cUx+6u17+5k{pPLaviF zt_e0}&GuL;AF+5<-&D{m86O17r!G zil&mF!XCB5bLwB{xEkud2BD)IQ=ZLawNVP!(1>gXzA!`QX>*@B9IhmkDqQmx-~XYO zA+0Ky+gl$@!d^kHlj?oy3SC0}fc8{dr)ZYsFz62M$RzNq^A#MLExGI*ws)p^)uo=S zn)d9z-z_re?V$12w$3(*gI`m(W)AZ_m<{N$$I#r8W~~{(%?^TsO=I$x#swo5-VNP1 z<8dfx%^y}DU<68M{pDNp)^&H ziDqnCLoN1gB(C}+BgZ&6*;KyZ_`&g_2+awsqDNH~R}3^1PAvov1#(~F;ZU_w3&%JM zn;FL;IaS4)TWw!zoi|jY^WGt)O$|fP34X3q7$P52UOG{^>}g+z`p;U(o0gYukPchb zO9y)`cg&YgYQ1;cIXD>A^1&&%Dwpe~biA{hfV#)eK7l36?6W3tqR9`AI4SsiQrtYv@=LX(xa?^C>^h zBDrN;VKg^(_~YQQgeLya0ZE#Z;KqaCrN{l9xg}}iVe1rD>zwDucoyPt{i;wD`SNu^ z=EymwzjrvLGAgPlJOKw(vR$9=|sZB0X=1Lqx@Cn zcndv|*j*k;u}HqFZAO_FT=vFAC(QN!x|W)&pyJBlcN6|jAkrVt^%3t=W)^~_Xs(p^ zP|%P%l;;`ISSWC)5%IPrqqF{FWRW4mG7rYnFnGHia$fu`LGsA*=dIjwXa>K7nM<~> z1SsADOuDjzs?F1Q%Uu*nF22ht`A*ZlCp062&RX3%aafnjjeB1+@DT-Yf@dI0)dEbT z155yWxLAofacpcfih9M-wFzg#pWenJ3@$g@{5Yngm z@)6t9c|whRgm6pzT&mU;dlkZ}b@i13mO#%_Wz)LseM~&$n{Gaz-8U|CoEu0S1Gnq= z_WVxt9P=26FllA@)Vij3c=(O{XyQ}ka~0f&^>bkY2J(i5A#kZkgTWpeA~ra4kwjE` zzL6nMkVGFTC8P=)p~ycOq!@EIE3uG_D^JF5=hjdhgKF}(o#mlUL*UpDgwBhS{mSV< zXzacDOE>nNy8al;2#L*}mqXNkLtcIA*!PqA#5zx0 z4O2(C?316L*9~1G=~s6v&%$f)&UYug4hJd8RGX<-IBEbD34<<*;&9YetE@zVV@>8{ zR8E0bC6ferUw=gulnDn;#F*F6Ge0sBGd3|YdYFy8lP6sBLTESoqj_;&yarBpPQ6Oh zU2|&+F7euKK~?hveW{5);iG(%{(Xz}OrQ=!X-GhW(JDme)i9!jLJ9rSbrZufI-<;5 zxH+D!_y}bg@S2?(3*n3>$V#;MSfqMPOEFG=`R(1_Pd4WTmEGvVm0a8lPa_G_c%)c~ zU~T#h+sPaG+SMO3GM-*ZM@Imt+;Yz6IyJ|H3fR{Pi9lG*Gl%%% zcJugfrO5m2XhyHpBy`~joDxT@MwBtc$c7v*SQ#lA5mhbEx5%X^{{(LyhVSVkX*~}B zhpn}e?!o0W=h*wljqAL)ck=3e;O~9LJRw|^&IQ}LrK1JZEp@j`rb-h&vci_hOUOhI+?jTcE&^=@WurYh>E@!6-J1)H&8&MNe5;<9Z!Td`0RQ2a*hj3Q= zV7>c3(ON2T+qvz{7Q=ew<)|M(kX~kfW3;5T8}>X|-TC*s0KU8z-j44R z1oHe5<`L}mUi;48YU!&Kwso6ZUF^(q>A8jCFbjg-ddFqjQ}0jc@_F&@p8_1v(ABWH z_sOSXg$D{JTVI?fBKj|CYqymGM$$`PsDx1o>=%vM8AgJS+V?-nQMCtO7h|WyN(xWl zS*`$2zp&kW-}}~$g?y$|X&gb)-{0{5(9lwi{l(8r-~f)MZ^W3fOFVk5uIFh(tqqM% zUc%vVB~;)kHo=h|btPZ8BDPuWvBZ)R1U_N>P`|e;DaXQ0h_u`SrH4xys$tSDn3)li zmBj!GU=nkz%5eliErO&zEz?!Riyy^mw3D{r#70#P1O?)4`(=!cAeD=ZFs&yhxAmaAsjq4XP@`d^ zvrxTHZYw4H2uS-gS8bIMeg0y%)OdP7Psxgd{>Wi&_SuN!Bi$`BH=>01(b1oR9}2l; zd_A~%L-`tWPj`Nn4p_o7vVbdrwcf%ll%2Kmj3`yUv+CT0;n4eCS_TespFxiiFl0)y zH^5!Y|9z@r$l<1?5F|KRQ3e^3r5^SKD0sA*A&VaZ;|y}lY!KF14t<0tT$OsQ58{pF zr)FkbTVGdIMm4{_4oV&R)|P7vO?tl|#TLW?x`9`F30vdTeU4c=+Qrwef}6Q;!?aZ0 zgln3WHNH8#s9!fE$a}Utr1jMdx}*>NK@fJc1+nZ2Tl0pZ;3xfJ_$L5>@LsQ8%pbRy zjqme5-A0{QW)?r=Sx(16WxM>NKdi1FPa%uOm109CRZX-j6Q77ejvC5Bv&w&>OjZJT`jKmq_ zW`X4VMF{mO`c3cIg8vteEip#B{Bg$WgRK6Md}D6`rnx#V$aw!10lmHmaEM~f`9;p{ zOG1EB1|D?DJTV}I?fii$nn$hMkM3NnfU_Ic&+slQ3IRs?+_=L*p$IVR$-MHpZ!8W( z&2n8va=3o?!Xbx2J^C$?L9N9Y(8_}w{mahBER6qoxdUK#-{cxHd_eoB-%AX)d4A|n zMCRi7+vTcV)KZd`0xXQX!FVD3|NhZGK0nmc7x~{eUbM3`$u|DgyUzf&)y(opy5_rf z?Rb;2Hano4RF)ZFKsIviyTR{~d~)yGmGSQ!Xn9J)pDO2#BZ#kHFsK0N z@*)&!gzk7q6;D$AZ*6P4;#Y4EPD@G|81T-6IBH;mLvz9oL>H?(N8djWlwAwu9`5;zsf92LHd8;jUF64f4^HKiargh+(S?h*X7dtt?l;M01pUe+0I6i6(T1V3%5W%6E;*KJ_KLBC4v=x60|e~0eFWJ*tiM=)eFMz6 zWoHA=A-JQ-pvP#gnPUPnjyE6Z-dmfT} zCEqx-sOxb5_wbk6$qk$$Ouoh&A@lc%imY%z0C%cmwQu|u^UJb8TtEP`o8`X$^1OdF z@>!<;8ZR8%67wrN0CX)2@_`|*GG`UWt)y5$vm1bkDFUh`AI&mneCka=De(Y7TDFU=zNmHkhY9+fSdVdA<2Yk7Gxmth)ylSJqjIRw zd5AC9lc$gtEQaQTae1udr7zXRTVz^qJ^XApe|YCR4D|N`U9jQ*3on1scfe@aRPTFz zHd*4C?d>4n0hn)~JnvSPeBi{h`~Rxjxh-h7bQP(#6#WS)hL2cqfVUA6{Q>xw^pZ={J9# zdp@Ti`n<+HB!%Fz==dKq{s{=kv?sp%&%1)&aRRa&JpeKI$Ljgm3J!076>$cPQvfqVCQ(yS;ZA6MY(j@wQ(JR< z+7Kvs)tDZrqQ>;|@DHJ2z&zIZ`#$_PS@pLA-p%^wHtYZRxtN;DYdhVq zmPE~W?B#7~xpH0-O&u!`Q@AyHbaXUM9xNl1t(2K-d3d9&vT`3InRn97{ZOZ7s_5W* zZbHH~3B=*!s;(j`b{btrM~GRCg^ih2*7ll!?ny&(Q4bqCw06oVpA!1?YosW0tTcF7 zw^o1lo6L=hqcea2xXa6zvp~71wKFCoTmdM01+oS2Gom2lM(7$E8fryV6@W*l!LWDt zorO+5Uo=BYi* zeH#k{opsldEt$bNstb6+lo33hk;6L`rPqAne5DUHaJQFC%VcO;6{cW9qgX95%jOZZ!thk!Xk$){D={)RAd zHR!VG+|UgwkmSEwX?dm9u56!_L@U%E4^}_1v?uWNW$T)Q(P0wTN6Jjs!ZE*Iva#MW zo^Ri>vw#f+QD=)F|IPn6V)O~;!*WMB!x6HRd+3`l!yH}9OG&X>`!s%!4-7e^NE*(k z{`6LlZ;IVEADy-1OZM+twiBSDt}NxAfr9-kdegGJ^p?*~-k6r`*)c!o}Nu^aQ(rB=1?5MTv_R-}A3UNm~_ zk+OMw^M*g=?%M2o`2gXB9h$zh<^6?(Ewy~oU47^~kh#ArJyPARqSThsWS3hVLxRpj@-=XDf%h$HO#JIq zX?g;~ZoYjPG{9DSfvh9DyPc;)L@@VYq-0Sg4aPecl_(@=JU(BjjE~VSC7Mb_qScy> zqEEdWBVOp5DPQgoGO}V2yg8KI4EOIxN$*SYx-7mA(Hocpi+4Vu&3#3#sHRe_ruxy* zLWuzsNgn23@q=pvb4G{+4;ApTk?y6{=)k-)W&RDSdQxq^3C|Aljbl#FTbuTete$bo z*8mIjAqvvM_%KZbv%DZ;QKr7-Wp+c|PUuikSm!DZA0Z6BQz#7m8hAW58{SUn5w2j2 z5qV>Y)45PkvVXI9A$EGT?oY^2O%b(dvUV zCzS-HfI8NCuMPoI-xq5*PCE&G^kUQzkHu)etEkOFw@a6NBS?b~(KsPP zrB0niud!)4BGBEELQWGo!cv!*_(Zx&+$t7jmF=YBkA}f1_Ev`R&+l|f056}wTDtln z&zZKdAqrc(VV69W1MFwwlH?j67N#p{Uy(;^N{K_=j2Q`70tR^?#%(YTN7K2o1`%( zwynnepXXic{XA>Uhcow_d+%%iu4_Zr9-;U=9WB`H9=k(5BfPg%@-!l3-$HcD45Zou zcbBa*O}Rt|0{TAJN3HzJo6j9DO?HO{l zs}-M`8yVKt4Cmc(yZnker;cJ>+S_(Y;N2cZBv#LIta<-Q<5YXWAoDK?1`M-tufd!! zhQdG!7X8G=F-zT#@rllOEwSsW)_smH%8;XaCUnOy^5QSuQmtI3{0NC>L*IIQ8e7I;@x3t>f3W)wu`X(hpmNU2n=0MBj_bo!>xoela6!m^imi( z<%#DG792>>=nT>r+0YRHN8&Nb&CyH|FE1~BT4iyu$@xli7Q5}}#Dwg|@T`r{4BE-d zWzXgc=xd0WpWnNxx^7AMOa_ZNs60}v_|*lq?WV{-e;}9NBHNk~q$QN0H+A7V9hVbll5>#$Zdv+fR(yCn^NPD{z>2;7!ODH8B!zpWlYl!ed~!Z~ zanfDCahfd_?f3TzBtU#H_y*myH=c9}4SZ%aS@N0L4699T%F$M1D_3<(YBrK|?odN> zTZ?ZMymq|d{iEqa=TXSI{9!Z|Oor9ZCx2zEUio0(P-t`lkX)G+wE+% zRle(H98qnr0uG=J4}K!(T(<2Z4`$=w;C!Y-pSxIYJZxUnQ>GB33C@k%^g5;Ewb}#- z#KXF*~cLqx97XahFayY{NO5A4BtG2Z)#A??z48@G1M;Sd+KkBVFNWe2R~sb8en|fN`U> zwRIMgvDoc40RaIZ)ZTb?Sua#Y$`K8=`0-ZjpD5z<6$;bq{q}f&_4KU-1IUrbT}|f& z?##A{m_Z@TZHnd5Y|%kU9ICi-c!Air7_NV9F-Wz-QxTG&g7Kauf+c-|Tdtg(u8qx} z!OHO{-DcK7`HidA2V6K%KYf||Tw))8SYq-dJy6YL1auGj9V=V~M;lHwT6Epw6!@X3 zPh8vR?kqg44A^|A;8!Y&5_OC=`lMbTz|{n^3C74akIoMTW-1uQ{;P47n1MOFdjGKI`8vR*7bBhvV|%oDBn(R<+w*r+M_U^&N3*9#YJRy0ZMSuz zbY9-?Cg}tK!%xt!JtYK*RUy5^bl6#LxNE;OhiZ+Hx!>dBBBCN?1P_Nj-Yt?#Vrlhz zIoTcqK0Z8;@KdPbG5dK}Y+i`rj4*}0Rn}9M@2cu!o?o#Gdhc`m-patr;vQ=*lQsNC z-cuIa8TtIw65wzNyVQR}fS=KaK8JS=8*DV&7!}sA**@v}&q;nc#o5qeAKWSj{g9^& zXF$V^2Vbl~JP6QH$tp?5fWCL9!IK_B=(*|hZ=%0n=nf)wvPhUUNY2+c3BOX;Dh_OX zTHU&l$rBPm55)QHz}&v=JWHkV?@jMPFKhtC5zP0CY@~rx=1~Byf)mcL*W4eux73LE zH(=o_w?|vKLOp@b*5voPOQhf9*Y#B7MPJT6 z@Sht|U#!{WL|v7@uIFX_QY+kAdk**QPXD7hBe9tEE%ANP|1B-(KUKRD0|UeCU7XNU zPHCXm?qn*%?k58L5YGtKf*I08kX~CV zi%69aBIkHd)sx);nMyf>o>7KLU1nll+HKb85nMSac5+vT+M)m6WS9l-msJHIPWn!^ zDDZJ;rH$7MPfD+9&(ta$W*4*N1aWW zmPA;DVv2r7t=^HA;`ZDXOANK-$_-aGZVcNLpjBZg0S;UXH^853)W~tyyGZHyR!FtM zqKxk8cj>*UjChjczEL5JUPVp$pMj%PGboKg-%eHN4cBZcx0>&sOKp(v^=jY*;d;q9 zjw-mNdc-v;t@Ql~@WOrge?0hq+@$rqj;ePTmaF#sj#4}v&v2fT{i%5vwaMYM7lEsr zA|6iJtCkKLQthAjpj1+#%n$Exz)6UUJ=%QBmN^9f)7AB@L6V)vIUdM5OG!C?+_ZmK z@0m$T+NbU*tn_OrS_%(%gBd?^F*Oa!I6T*XiYzbq(YWMG?NU!yB_?-T5*|(`J&X%j zO7~4UYqa+kkb1fK;G6Dh4{b`VrM7>@!tm8E|NP@_rk1lLU_c6^gkK6GaQ(}noK~x# zj52H$5{aygvD4DjgajuZHwN&2RBqs-*iz&P z-u(C8gk`UZffU=#Wd-RpW4Qun(ZSvj=uw{z`l>$TQRe}tRxQQCJT*A(%ReQjw|ML} z9{VZk=m?h|GW-+N8hqqjd4e1& zz`S|kGUppcr`2jl6p8d)zUEz)0vo!ymGIm@tC4ItBA;pMWol}#fwODt1%;TJ%vQ;y zdMfe-1@MuL4OzSpgzGWC0){4|t765DYTDCjZ5)!XorTJYgWhP>5A?fR>2vX62apsY zv>5qSZ>q`z1FoVg1z)3?IgcVyI)Fvld zhhe*5uvHVrYQ?uURvvoMLv`s%#;K}Ez40K70kl7|38j_|gKci#0%?B92~j zUT7VHX~kzEjn@{e&a*JB3%6)S&(5Neln9JR@M8h=x43~#bt+*du6?9doJ@u7b$qnw zUbUngn&$gRSi_*qb0RrQC#nP`W9=W;a;_=~rMbMeb+2E)e5xL9Eu$tKh>zeokYqy( zF%;I{j@jtMbiMbORNl;+>ht|=(FdDUWk-o5H=U_(x6Fh^ExA|CX=$AE%26g5+-!Bu za>v~gXJ==DjIau0Azj^^4!k4%36)~O0yT|tzTRh1<=Pkh*BvYn=#JUHfYmcmiV+PE zV(iZeY>d-OsKiu@K_nHpT$Q71PGfYz7#5{8_~6Sn3@sYP0cEzvOl0{DRSwVkGf@M4 zm|N>jtY5De;(B6xjr~5l7EGYbm|Q+_cl~nK#R6BvyMiqgY`}$buP*w&cg}M`Ipt1Q z8R4Tg&l(D3r2U=--I4cy-2A%s_y57nCCfdJq3{7S@XV;VTBXVO0SF2z80GdqLAb_? zzm|XhVd-(nUQN9{uD2+4-nUUV;RTlq*1gDuqklUj*E2K*jJ}r@l<}jCs=mnJ9Q$;u zQJi%(W5P=RaQsEkjkrsp;X=3#psoY7*)1nH78F>KD?P?*p;r=wuWoLyQ8a00qEDFH z;ZZ8Bj9$)PXLg&T{>`&{URocdF;(j4tL%BqsS4wFCzrP%M?2MXDOwcQ|)($`?~D^HSJ?&X?|lv@6_D-wj$+Q57X2*Sequ84+q`#bQV2aJJN}Z_;wy2@}(IL_DRlooN z{s`~9bn-~B0{M$hk7o&CaCo_oYm#FP6TWBfk-sPJ(1PiLx$KT{^Z3n*r(Jc{3fQH6 zyR3S6`f%jgbHG^CY~-5*+ikS1?)y47v)jX|h|kfC*C#}Tt1HuK7@22nRMJKq0v3WBjtQRni&JqBw#A~=ALEqRF7;fUD;hM zNJPvn-vt^7@fWAubF7^zHHUr`wlRya*(2j+ zF@s{r!1&$uzpPNPqef|ba}#mCZ~AIs!j7d{5`}rl&AOU9NTKUiC^$DIk*Y11$Y*?C zwOFR!xU+by!U%5S9mn%Gk=b;ue%R7c6fH=+NuAJ;RQkNlQmM^@%AQ4|^~AuVjD<0V zscz4=mUJmq7u|rv?qvk1a)EA{2f&nrbN*)Y90OG|%kZ++auf2upWWAAbkm+uPNS)5 za=rk0_W9Rl2;8o2zIGaO)fLuZ3nmMI5UStYeW(P>%f{yuEl0 ziSEEL5A%#_$?Gw7{iOhJYs%9J^lNakJyl?QE6POxTMXLsCw9dospY(K{z*6Q7e z{~7H6bjObCIln~Y0BH+n=p7!~;TA?u8tnh#^4%SZK{(9yxsgoCKRe&qQah?5U^zKm8)PmWWM6 zjfLm9HnW3%_|E;hS30hc7owvbDnG@-{X2?<*9VXn&BoutaTB6G(t7vD_>9w)h<)4$rOV|F&O z2;o+k5G=0YTpY386k^V05#g$r2Sjwgvxy`4XH-_Xt#xq!A$~}_1?-sO=d~^ZNvY9P|b7jY@$FzZX z-*PMI06yakUEn5X6sR(CZx|g?v9!WQ`!ic$Z8Br72j`Q($^iTP5-APz$Z#vdmzifo zDzywNM$J9-oHZBKJK4H<$#)f2MO; z-+NzQ#XgUBKy7U3Rwcnbmf#_G+DYiVeM`*DA@8E9?KwJ2X=Cdir+alJV7I{z)P~;- z(&N3<>K%5x9>ls_X-2&MQJS0VR4z%ZjOlCi7419QxXN}6KHYW#9^_iPv$>z^QTxWm z)<8Dz3=6-?e9!C67)~rBy9~3&U0Q@1qpF6*Cvnxdd<*IwwSo!+)F(Vq=sK*WjD0dG z66e2kGq8_U7Wau^MRwdLV{{9gPh7?UF&wHUr0(lUB|Xh6x+(?o1V`yC3o|9uL}N{v zrbTM)ScO^2#rBn?z0g|O@fUdM2aKs{W?m+&=tHpt(MnwnSn$}P1EsdBEeV!xHkb+& ziQ0M&M;A?&a~eNMD}b0vDemBEI_>DDChARh*-I6#h);pPWHwV}E&(U2kTOa5kwylC z3i&r3T7scfdwL#TOCc_bY>?|8J@VS(WlK;d?~wS?b%K~-;8N1-5t`^tX_nniww5Y! z6#*X9K)Z3s)VLT7?yn8FFli`QB!SXU6vQAf@AFa7=F_D6JRwQ}`hD2@@ z<2MP*_~@|t!flPvO->K9zeykPk)pF?>Uujh;j3`V%t5>@u#~3(rQs`r1+Xz}V8?%v4=QS^>S7=(+szN# z<{$Q5+fb6cWO9p{-ueNW-a&I#yeVktxzzKve7&4e9Ze@fs@pG{AACDBlc&q}*kp1U z%=>^ZJ&XaH^X`WRZ+uZhfB{;Fa||NG?|9!q6u&E|p>2g+0#WQIwXPiLP)uRs(OH$d zYzLK=#`1fTrfu<{yq_Bq_i3+x1qH*A3AEDg5GNFWcAPbgxm~PDI{#*JwCevESIwAO zDdE?a-#`~uiZmbgRYImtkE<^-7~(s_mycp)IBqw5F!H|D?#uG}QHzwR6FSYXl0G&Nl?~plo-aPhD8%r2GCAzMU>B*y zimTn2ZuCE21mEO1?_LEk7nixZ2@Dx!*Cr@yX^pnG zXg&#+HBF$(VFvdQm=Sj~_g!ihNht^5hbZ;BAo44TbEx^o$`Zuo- z2&?i5!l8D6sIzSw>sZ%G*DnLU_||7S&YXg(s@g4=KW5H={3a4>L>B=%^b?GsQ|cEu zWBP3UM&_&T##{v;azFyr6%!gP(!Q!-r7wo|jm2y{z?!(uG((B{kAmjVj?d_s4F>gC z7#xm$Z&-K+Bc+-st=EZ!wCkkF(n-898g;ywNOn$+ER|Hw5aeLB)80(LNg5~v0= zkx!JQ%Iqm-i8u{(V{ZglZPU+eLW|{ABI4wPxcn6BiEmtf)JeI~qi+q#9%(RND+{pk z8xASWoIKF|lCYfYswPP=TyYn*|B{d^qgr6VqlP$m$aB9;d2KqTMkI8dsplF}xu3pu zF6B3$Kx}s;!LDo-Rxutl!@$J+6MHG*3}-UrJx9GpAX5-kgbV~_ls4$@Zlx3HyIBt9 z+pOq$pR+}jj0&2kJ2MT8+w6~~0IFODuQxllv|G3hX8n=k!p%HI)jW`r6+3Lt%Aw-? z94@J(yskv_Ca|v;T7AMY#WOH@W6uAQu~OXtSHda2lx-EN*|)?aAiKQj=*<*8h(}9i6g$jU zd6&z^g$B96gQy^Ai0~cAf!e(jQO4R%I>BrV0;EbP!b-b>qTiR$ian<_omV=-KkTR; z5vf!$#{B`B*~WsJxBNRijBz%bkyXeVxO}ML9(Y?6jt`bFYc3N{0pk$XlWESf+o)m# zzsH?uh6a~jA=OWitTBLC<~v&@$B15pVU0z$;dNRK0lidC{Fw!kNl+Oa^0>Z()z8!Z zcg`l1lXpMrx8$5PA^q9yjnN05)4!@-|ne#hw~uGAdmDUOqlv1w%2{ zrD@WGsffO=1e{Wf7eoC}bregDpykiD1L&c&s@1q!B5wz6j1<~kQ~pRnWt&WIXWKtB zJ?^;lLbTDlqzk{ZRcQ6q&K5Cf?CQ2H=oV;R@bJF}F|mufM~~i$^E>%>ZF1MtzO`u9 zknrxmc60LeYMlL(Fr4kixQN#ie!RKlIPif{IrPpZAA7N(MhJF%FC5FjpVH8I|>}sW#d-3S0fa;9zkEegB5avs)VEfTcRs9 z5)t28I<>AK+cIr~H`xc!Rhx@PgaDYFOX;u82O#3Ek-(%iUKj8U;MO?L6>PFtVl=1r>b)%`&lC z`$9EzG1{L7nsy>H8`#6L_hu5gg3G~(#G~kckjuYR$2JPccg394_RbE#mTJvm-04|w zdE$%?@sWu+tNKq`C_vBCM@S6or<>~AUX_2+Rh_YxFsivb{7YM$)*3oxpa&GilLp@RnxAG?(Jh{E z4-Pn;N>B)rO+ao3k<6;T0?w>;gqNuYLlDuL2LrCkANm-qm>+$$=DSfxQm&5*J;)F& zhBvn(^(t2yhEWzbhbcZ!IsPAN&B^0OZgV8V(qpe3dS9pu-%<3hYXK3n%*e-IIIX&4 zkx>*5QE6Chk<0djL-hF}N}s zD0B7x8pHW1EFhbrZb&=5l?~Vz1_}uYXng)DHR@5wNbRam#8=bdY0}z|ESYujv4y8U zzcj}ESmtle;~sb4{5=~G`F6!G*%`Nrt93j4!0ppK#?;h#|9}UjKfov&=VPGX(uuvP z+t;(3SM~6_x&Nc_VTU)6f59*c^ng^rL(OIffi!p0#1O!9HnSs!plg9=jxxWsPy3=m zw-pe_Rj57R8GE43L0iPGWvH7-Ynfv;_2>Faq`PA@l^v`$ZgLFSNykVEpmr|@@Wngt zc>N%rYuh(L*oeSDDONP?ha&RWf+&IvH=_JxOqH)f{mUoCz}E2dAceAkU%K{QQA2OR~>eLj{z2&9*sXq+#1(hdrn$OJM*8*d*aT;o3o8#T={AAHw}LK0a}rd&Qa1xa)FjFiNmku7za8qJM^Diup9GWb-a0$0Re; zG0@T+{wpa@Su1JNNE8kMw`Y*YbhZ?)Cs&VZ{u`R?hZ`zGCJ0sAU6>7OQ{v=qn8S9= z(DuphvIDAH)%&;j4TwhFgAQI!Q={DB$mA|Ik=Z3!KKba@r%51uG&9WhzIexV6g#Ca zUYm5oBrS}eosHd_dHyh-5u3VuZ`T-2TIG>$#diqZu;tEfy)IZ3wm((HQ3b4~Xw>vL zPlu(;ieqD;q$EYv~_K~OQ1 z#!~r72$hga`_1YspcHaq`^YGl*wq0o*Xd5svYzuG~f^q?k3 zQL(AApG=@yf4F_{-337Mgt}H(Tz)FBgW~_Q@lyO0G}GA92CF3lnpjwYS&^5RHMd0#C1w zFT?T~gg_N8|3WiRQ$r699>QRn>fkbIjEG3@>5IcGjTmjs-EGvi(#uuG+w_xb{xVy! z{NCvm9VA*IRv9k}rGyYHZfS&kf>q?T+G_Hw9!xLUT&7}4l^VpnS`Vo|46WU|NB#Xv zPb-PFU4_`RN=?6$w4&zS7A@Z)l6$I}7o$aGf`DRlzt(RBY*VGNnx${(wz1kh)OF!Us{wG)f-Eq~P^MQ}IT$%x|YB(Pqjj ze8}PmxWnW)Z`^OkVi<7rVY?u+##{y~eO~-YS z8dJBz63O62fv%kp)X}fMqTTVg` zi=t_%{8}Nd{IzACkp1^-?`OlklvOcV_UKo{C)(}kNrEFsO=0|efgi#*l40=5BF7Uf zujVHtwTz<2RLvn2CaYp5H{B01^)4q{9}_XoM_`}iUAHn<5r(n+3%T%#9$IYmm?p5j zw3!Xzb;ErO3;tnDL!+`(C6i3ap`UIwf6ut;MGHB_sp^sO@m75E{j`M;nPo0^vC z|GAQc_vjqNg(;4;#zCAF_F>&t^`9nKHTB`5&!b%$34_aT5ai!>2NY$g?fDDsdA6_;vRV2Q zHmcd(NI9*12j%VzVu#|CKj_tptD2+j{ywphu-pj$`GG_#3j-mTj|tKCJvISUqr71g zGfz!NPanDPDV9MoI^j_rh*S+j791cDNRtmtJLi>U+ws8MI=?`YQd!f((& z#LSA&tgtNHdT<(O+|biQsM_H`JscmbV*-$w`QBAd$$WYO$3Vc1QAR23xSHMSC&=_P zv@itlCTY^_X_#;~qS&fxz$Z+EG=7f;jGkXH-g>t^iD*{psH}b{xwk1fpa$OG>Mrm> zUZmrT1r=@Dh@}-*8YlVrU$4&oQjajPThWS7tB`B-@e<<|nR#Sd@r|1Ige)Nqy=(=e zglS4b9u(P;j4Pnf?j;mary@pw)>LK|CY>(a6pF>?fCQc$tD|3@K^65Hy?9O-ija5}r`K8?^Lgb_)KR?g@ntmm) z%=OJAa{r|<=M@fk97fe7!-_Bx#tzPoh4o z-HhCFzNkEGkKM=A<=@iQ&UzXzO?R>btX|Y_;;5#i)lXjxF;V}ug&3PIZ%b6_Ap+}@Qp|o#t03PTLEynH`@#sL|0uiQ zbxjxh!VpAX$P=t|0fco7&-@_jxt6uEFwaiBX}kug3Zu;a&-BQ6Svj9U81u1XVCV5TP!XpDgL(9PPYrFiwJv9MbBmr zQ&Et0bZI|6Lk0<)isa|dH&AE;viG&l4OmV-K#O21Ii*Z_bOimnGQtSsJ%>NB(st|? zQ|0Js|CuEZ&dTf?ip`AMCZhBjpzEujUDI#wSHf4#(h@s~{W7AP(cydALrY;tff5gG zYk(1{U(g___|N3AOFpN=!cpCHH@+&4oVmK={*cK4DJPfwDcvp1Jm0 zO%FIL`p5R3_%iS^dG%!#FZ=Nzq}>H}k1R>cd|x$F%Msui1kJFW&OI^xs&ka!$nwH0+e zbs_EdxbI19!wL-wu63>yf#Qys&?y7@KO&RZi&7GZ;w9weDOy-gk-iyxErAVCzOSLdQeQ8>?`a9ST*t z>)@|S^ObPte(n|aT7oN6C4wy>*P=ji+d5CW04^zJBfK3w`(t|BJ3R(y)f}_JO1+YQ z%In%$!=OCdy-%+9Gx!WJ?ZSdBZZeI+AN^i@4fZA}OowTo0DSecRQIABA= zhw-<0G7t@XS%+kUvRz*uFW{&t(0^*ijE0FiIe9dPx($2F(CSJu{f#x{5-z+d>@c{a zbT+~#*1n+Vk!K5T1hwd&>o#Jap-ezNTfYCT=VQO18bi;klS?Y3)V><(Cy4Cp*dtSl zLK!)dp}BWDve|j3Ry7qnf~9pisv#1njwjy?2cgKz3EWhx;{>A&{s~RJ3U`?%H`yAq zER|fQHKwX?f9Da-a{kAVK_Y_duDgw2v<&Y&V9~ZrDyVbwFU{cedey@*f1rzs zJkl$e)j1R?5kTcwhyQh9yXr9=6+d?-VR=k%pQp0g_Wte~Gzf_oNWyzgspsgWdif}q z)$FfMdI+|oTbeb&Z?0OuqyZt6THP|{SK6A?t|&8-^*nvE*=;`}+DAk#_8Cr5` z#fWU}|MG3JRY_`h;E~v))%Y)|YTG>}`^+c09yZ2Ci(*H9Z7?crb+Ow?>Uf_wr2}i`y!^;=JkwvVqpBn$Wua$fTZ>oZkCp;15bcoXH-2K`=hPWXCbJ zF9hyu1vU+26}2b5PEQlceSgU8!7adgh9g&)V#^V@sig)(6R)YceZ3Ud-%Ybl9E;%x zdNIlUmuJvZFCm3YlonS|4elb*Qw-lK2D*bc*a>F&mUlQ4%f3#_=<;%DMM|=C_vD0u zxfix-zL&`o_txHkWO4rcZnU_it#AqFgOJ6u(#6Om_cJf3IhcQgv{S9h$BsrghfkR#-q4DaAv!CbDtBbEJxF`9U&UslNVab}&1QxT)UjO}$2uh{<>YQT1PWS_hWuwFSJ1? zFBxH#Lp$ig4ZD8+u-yX`!rY7omyfrFqZEAZz+7Bg$NI0qt(D=x^E>RNcI7&poSL)B z?Wvn*PNUtVWm|}(KnJi@yj?8UpUivz_vWc4_W@Q@W@GUWY$Y2SkPt*j+5{R>m zbPQe83F4<5_sd;_iFy@B)PvI8#uSU51V@)9;y!r?T)z?ntaCGG;t0w=Yb zzeY>rlewmWq@kgpD)xl`JeQD@N-iRvs~VxqH@luUHzTSdqU-mUOE~zG_~?Fulka&h zL9pUoU@Jb>j}H!8qr}2%>ml-fmtA=;QvBX=LNc4#2&-&c%2opq9fd}M#I~xbj3)f_ ze1u?$!!>=qHlWnuv+vS{-V1aaNCmHn%>MWO??9Dsd3tj-KX{q~A*qda8+}NgPk?g~hq)#t9>Nva^@Y8+ zv__8qiF@1n(l%vrP3NLYm75ZNKe5C=EF6tiA7P< zPFZox5+dG%%rI+6zy}d;kFVSGhoy}Cx9YDDa#&w0;`{9(j8^Yn2MOv3h;bqe4%^f>+?OcsTrjHrXWZ(yGh4$r4%wBejZ|m`wgB+kS?` z7m__P-^?o_nwYy<`m8FNEY*gKP7mwS>?S!RknaU^_3Q#&U1H~7OA*zSO}d_6!Aj;e z@hrSzuFW&ViM2Ya*oA+H>~6nNzI$;>le7*Fv7bLeb5Q4W3?7_zG6WUD&m?A945M~et+aB|Ix?W$Q> zg`yy4$259UK>4^PtStuo){2RRy!D~TSm}V4X38SOBlWZQ;C{Da*Ez3prD;KgeB3_Q z{lQ~VcFehU*ILnFt3?(+W$4pGsXORy^55=K0WNy0>K1VlIy7{k7;!Je!q$bFNm;nu z^VM+JJo-vxkeDX0!a?1;b`m`4mk_vRT5V2&$Ah`J6ZVDYhsjH$YfCyF%2To+cT{%{ z+Uqm!T-cIh-FyUTSY1R&7cNZaPWa4r|8qK3<1f|yqt#hV&q)*YD{y+GqCwL6Le=%^q3Xnr7N>+D`_2e&EjVN_LL!QGyUXdx5|&|6&4t_vhq~Gtj?BUT%`}nA%k6eBhigvC+RWx>#e8@~7LRXF zu({a6Kk0N;99x3J5Bo|+D%4O#dezYR4v5T z%rd%~CIk3her6pvnuPM5Gt9Utp>9oBZRNM*4^C?a%6_+s%py=jaC`U5M!CTk5>CEagxn?j2h4)qpIF*k4}l>Na0zLARrkYF3qHWwc>MC9HLuN*HN35m)H>* zfxC6Du13D5`=>1V3u$++cG9e>>zP)MGU_JTu!t{Fl&*g9-3p7LYyM9rYbJ4+bns3y zS=6@vO%Z)7&<;A*q;yKlxHy$_n#Z`Dj>FP`MAzPK0N^xUnY+2Q(QksBrqp6R zA(g_cUk-n0kSDSk#so?;dpE^qn4}q^&J5k*t~eiycJtUID7Jbz=J*5=Y0~L4))b2t zx?S;B`8+~`(yurNm!6mmJ|U91+rc3+nA-$7xL#i}~b# zdp)*hG5?ZytPp)X4(q?)q^0S3St24)2*X2~L8B@=SnNBkB%n`sV$FncF%ZU8T${rVtsY5=I+KxXZ)O4WlIJwBkowwyl0u7H=o7) z3#Ow&%A3Q~%xa84-~|rN7KOk*dabS!mMXYP0_g{{mAkO#!EobH<;CC0-7` zgKMVXtN7D58-LN_iIxpDTLqywSr`FMrcz$0)t8UA?$^Vcdj$69+ySE0w;&`nOA|oB zNdiF8#m$fR2LL!2{`-qZQBhHl{NncZHeMEuB^m*`9v16&j+* z3=?=C@M*Ne_(k(NqltU0`@H-?B3DnPX7_LFnt4S`trs=wA2a#bMz=GAQ}75oh@qP> z60D;eygpWf_ExZN2+|Z}R!!z;Aq4AboIOpp%Z+b+uX(9x8`W(j15Qr!a-IMX8DRtk z!5&Nb$zA&9k~mT(7!Cbf7ni2DOT$6haw)PK(#cnCTrYs1(&%(M2WaIOV|S?jFL>Lg+?im(S$*t;SpNIEX-z}7nQ<(2`QtPWhg0$=H>8kq7J(5? zzAgzPaf~lHpQ4ZTgQac~oi+>Gt>0YdlQ8iRF7otb&)c~@jan6>#Yus1I5RVIIt6PA zsd0a$TD6pVUe7Yi-9+#uw_>6qj$)3@RrG2WGAQee#caHp<` zOQV*3%xc`o()?SZhd2E3_oE%9(Us;^*#4ffT+WdSVG>FPIr4dAO`>$pM%l;LZ{Ji z6j)%DMyCy1O~7J?WZ36*cVg;t1Gshn^G6Ti*@v8xh7De@F5PD#sGQE0i~(w{E$jHi zgfFn&$?a^3$>eW*G(&eAfPRZ5NR=b!$Q|OCX9vnPFcGos09I3k;rG{nQyGlEo`JZ< z)>Zi@LDT8l?q{pT8eQm>Tux^*z;_PyT(%9Y=wu25S&9xIL>A8>jRKs1%J2%5Q)512Fr7qR` z`2`2+d@g*&c$&f-@SBm?0FLZyuP)?pf2x4e{t3VLe|f^nfC|v~UyQp6snmXL!F9BF z6X4MQkoSK^%TQ4(t^NR_@V@jv0hpN4I>P1v+gDeySbWb(PGG!GFTp@t-`4gBJS|3M zAlZ@KaYrPZ?rR+$5Vfb*=CIvCcl9fFdQp0&Mw-skqbj{i90d^+RsK|$GjuispA z&NbJJ>+);L0>+f0PM42n)BU*^u(1vd+>ZPJDKJj~)v)_Cm;C|=;K9cC_OP|C(31tT zGEIkRMPb9KNs(p0}K8vZoz1TNh`U^@>KkRDX)vXJlfsmn-?{DbB@XMFFU% zvtlMDCV|hV-$6f-PXpYkGz6B_cDA;wuEB%DX*|TKCaJFp&H#giK$Ha9 zX)Mu_rBYwd(=626kQdkI@4Em3uJoi=c0P)Q2rq^JV-Q@Ss*H^F$!5Ry-M+HS3%JmhZm`Q6e>jzd zjEIQ1HVR=9sI002a1ydv3ZQW|gw!i)zZ-)f!j1l@u)j2yFEVAbYsy8=oT8(nA>gMGPnp((s8id0M|XYEV)0&*$s|R$ zIUFb{UN^1WAtUJ;S0Sjgg0k6DMmrC{dm0AeJgxg;i`Y5$U-E$yOgBLF zDyZ~wv#LF%j_2HwD9@wne^g=M6)y|q)9!OJd&pF~XxJ@zXw;^cc~D3@LZk;pz7j%1 zO2Wb_);?J7!~29pC%3@(wsR{`d!YFRl8{^@z;1P-ZS@>|eJU|Weg%-H{Y^Wc$|kYq zM#R|u__ijtYx=J9X;`(`*t9{srvDzTo9%M5GgJ;dJp49+BHvqMOVbPJ zPz)rt(&2Jci&-+_A3gLXmJ4QK*3sHb=S8$)`GS0<8I$V5pi<6jK+^NN=Ch{}i-OPq zHOBL#dc~0d;^Ibyg6}v%^qHDf5{g&_$c@U zIP{+-VnHPo&f!&0qtsz>$B#`Zo$2-~z1@)$jHA@ScKAt+^_XO@;MV@_Mjr zpHc8iSg-VHF5XMwV;tqT`E8jiTowx~u@Sb6@LRV1`9a5&G|!N z2nQ|8o9_AZK2So}CKvbW}Xwi(Ac~OIrVkL%C^I+TdSain*403C>W%SGo4UO!cKh%=A9D z86%UQDAIMTiF~$Wbh`oZIb_Zc8cPK47L519l9r$yu)5jNV*QWEAd)lI1qWp9i&f@i z5>z|5^dAV4tY){&LgtdX_JVRq=>-^65|9|r5x_Y-CUSaEK^>PSBk7xVj+cuUfmA^u zN}J*d5C{YmicSCyY}$qze2ETOE;{<7wk9zY)mVPUwTW^`V>%9ndN54|AA<)klJiDa z=ex2*bld%T+0z0&UDs|bWfB;w+OU1?dxZaJ`7cp^#)O`MYf-Hv?*3}z)l8Y2n(asZ z;bxLjJWj)YT!`Vd+O%=1{@9o zGK9izK{=+t_6tI;lh@%;4VV`zLR34M!3bJOY>C zf7`@AKV=4bjZM+`Fbe#S6Z!XhzkIUsq(9aNbl{f&i$O_%vE%Y$piODSL=lw1mOw3aCDmq$PS|w_lQ*@YHxOc!S z(KBe32lBy8JVw&4b1z%6jOb|SkAZ=D=)J!FQTEOszm6%Pe7}+C!{kTW8evdWKVyz? zOHtbpQU00ylG$AWZ(P~K{-9UPlm*Y5#VCAGZGLwLhtbYpzq(ujSBxM&+J8zbAuh>y%8j#9ErmegRg{|n7V*Zeq95$>Y2)#hn>OJ zTx}mVR;xc8}OxfqxJQjeJ|?{h=qdW zox(`U_aZ-dT5RfWGW}j~5-}v96j2chVr*L&>|$%4AHukv>Q{DsXW$hioN&!RlHtne z+G^OuDQgA_|JZ@I62zpGeq{8={Kb@wo8VL&G_9JXsikK!;Wia_4T!EdYiPEa6G+D5 zP(cl_zBtxmFKAO02BZjh3*r)vB`r{-=|*XQ{SABbiX}7jV`b_`<@!Gms{D&wQDHJc z+&X1U%DC~TBmQl@zP!A~rf@P8;{RLf;EB{A=8k>BT>2Nm`tZVmQ|ABlv7Z;hf4z1r z2^C0|X$j5!Yl+DOvogf>A;GW_7A~8I9v_E=;r2iKy>dN5sHohYYif#944BODb|0*x z$15}&R#Qe~7{GaX&jhxM53V0OQQAzB8kr?G16cB3K8=J9vQC6U=KQ~|(-@|>u{7X$ zlm=hA_kSeJippB>9Egw=F8kNPki>PR4c<)cG4wy?#31RB0&Ye z`-}dOzzHG(y%^`$>iP2+{_St!yuk*bri;e?b#c|Gs7ZJXn+6(xb(*N)!Tt{)>%AYi zIF9%0bG?1|bup08< zB_@!d(*N=e-@=&hdj@>p5do_jD@1iNQN1NBqp;-MG*_&O>;J7df*4?PgfZN4n>bKu z=oyAbN6o@h7cJ7dyWz3LHG~Yd#>uG{8Q_M$uY&MBrODnRJT+%Dm&E$kDSYv(i5SWR z+XL@`;CpQGfXvMGp&<+?A=^bvm_~;K0%GEo7^3WwU-DrJ@lu>UPe@6YD0U!?goFZ_ zKOGjzUHm51Q{&=h8popNYuRBC5D}{o#SDj9>sz>;5_TkinTal4wZwQ6@w2@zCEB*O zwu4Wov*4h^1W4%Vo5ZQz2XOWAXKz_*>88<7TMF_j1op(hhA&`2gOi*>IglFL1Y!q8 zisW9trthxX0e5uWeqLmMoimKBo!uatLG@Lyc^FDUQWCS_;g6V@n3-43iFP-9>2~p5 zSJdxo({^I1!S;J!_Y!lk!dw2IeN=`I?#IFWbF<56;iUMe66H^#APC zzt31+3370L=JF6FCjUi`z9In){Q4Cme&;VLNRkU2B;F)(goM;zmIM|&Hg$0kG>JdNwk=Pl>IIJQ6&;2wI;ddZ{z=6G8m7mOD7^^@lRI`aQ{ z+|MjP5c<~}QT=~0x3OhlF$>FYY@D_JB11hqAPA8e%k2V&{<-%&AK^f5=0jFG^okg1@aoSf{N zcHc<(fz(kWooMTSmX1mU9(C<|lpJ3=6Wng2fppx(xbRAaTbTV5Es(Nq1CUv_XOq&8 zhb+LI8b{N9d?H7D_^N!qKt69IfhmiYB^G&pir;aV`_m**L;%7!BRfNd34nVqY=|D) z8_%Z?1vL}K5Kw(N>k38=pK1s=sT&O^bpy^NHPST*iAguYDaZ;2Jd2h1&pxDU0U^yG zMk%1ekO@8D-=BOYa#}O1?3{LAc{1OP#p984lH#y@E7tS)4zFh|55z;+B}%%ywia}&GERHH`;{cOFMF+cHGX!MK*dPAP}PH&KLy{ zuMTl@h7dI}d?V%@NkiVg&0MGQ(>CBTQRs-0i(baSS@EbqZIX^2*PG@+Hgq}c^HVW_v>ZXo+(U|;}XPbH+KQJ#mr4w$US z%*@nT^zCzvO!}>eB{)z~Vb#l{QieqIa3r)0bhDR4v#kT)4!JXKe0+(xD&5*N(^I&n zJh~M`(nc<_VM16J>>N2$L`B-}uMa{)L&vj3iuRtVR{DNcHA<<`>kiHmiMTjgQs1(g zEB6+BAlbcd4iIDj|LapGz0aVkarnNea7a7&;&=r>K+Y0zyR`eh)pps-5ia1cS#5`^ z%_t}+0D0b*JI9N4R)9xtGXkIeIS0qQM)m{^%H&JDPKnf7B{SB{d`j}0uVWcqk)#3% zj~pUVqS;c%zhI?&gOl*Ovj=uDEF{dy#pODgD`}5daXgt95yc1(DGslmt0D}ZeHC#R zcf4jsWZ~|6~^G$!0T9O5`EQ>A~xK$3e6{{k}nCyNhPRWpC`3VHt}MU>!|+28*PuPE$@eq@p4cV za`}FOy2?IavK9ET2~cWsJ~tF13qg`FYeSH##Qc($`}s7eT{VKtOBj(TAks?VJ8nLe z;=GznyN~(!(Z_VX>A2Nw>S{c~14e`!)H!U(y0@`T`ZpmdW)`e4PEhe7ioWV1zW)X) z4HaGfL{$~rL7@mBH37U0=jK9ej(dIoB|rj?BX4~{pokKW(SnMDfEyDsc?PXJ(j2ijZ>1-s88=7#RmKu~55~0b60Q z=vnT*{(=}s0mPumiJC+zT9y5!@xw-PmELWLxO{=>>sue7MIc zfh5+e@RtHA!E|MXI>xu_a~M7WsSLIepoo){&J;4iNG9_xLgvk>h~Hz(i!8ccRCC)9 z1mZI+lPAy;RRqG*8uiw?dU|9gpR>)?Ns~dw`w6;&ZDju{7pB@E0qShz!)d{MpR-5s zFZKrtAK#O~3i%dUTCm|);?TKm;IEK879Qxqn`g|bnx_mXVR(kwDIr5ZKv<-7>%NoX zICZ;+fQLfe>&DK@GoSwqo{z*9=Czi%VQTG0eyw2}d(*d=4La-(PI;6SrbugxL9wnT zep)J8MR14T_ZGulP_|vD^SDL{O^C-AXE^#zL;Q7O9ifhT(L4A)@JyUil4)1*d~vs?YgGNsZY-v4uxpj% zkM9bEH(q#IS=F*x1nOVQ+rDGP}rJs1xEj#@#Of@;agYW}tnD5V~Qf>mfU>^3|P3r<1xdOqE- zRvZ$|3C3sEu?6WjS1?y}K4{&tiiCYeFCs~Wib(#X<@)L1sZ8 z0|yGfvsMX({jhCE0cua?exxXp|L~O;)I;OGz1WOgKG+XgGpy7j3FZFX2<>iaf&xnL zorLsB1AO?OfB55D0^qL4W4wlYyg5EwrKN8uA-7%-DwD85j|wuG~?E|Oqa^1W^>0 zqCD-5A7CRPe!`@5KEM5u*F<%xu&rLz1;l#896cZMC8mR35wZ$U?D>=Nn$tazVJAMb z;MtZBcMBsz*Np2}`ie2}waaBe4&z}?7EJcMhoE}RVhHZ-_x1frrnryAWM5OluHz)W z7H5R_e=vVIS^QiiWIz!Vly3l4S_Zo}EiED}ywe`2uv?-^g_k;+MTXq3UOsjb5$9{a z6@K|?DNNyO5C$X^b(A7ui?yz0&1v+C(vQ;nuO)-2pU4u62PBXy{c57vE;jO51O z9@$SbEi;>V-5q9(EEmkAv$t!rFB>GXjIm`0RB0(t;&179OLA))MISJOcE3+ zLWjHrHCrlp6Tj-4_ysDCE4;GfrT&j_$LJuTKHVFt#CKDu?N81{pLZf0*OD(pxv?BU ztL6_{5>5}Y85LJK8hN4_>d#-reV}%C&T;XmfUPh+9;s!z1I;NVk&14&RNbnncl5mY7KH@L7}u_QR?b8s1|{8?YSS03T?QdP*MB+Nuod^10KBBe@XzRBUN zj8p~Y-S&QfU5$@iTR*SQAT%pz)tM>yP^16&z)zp>yVQYaTvJ5i8MCqCc5N}&?&2|z zq}}RcrvV*%uZK8ANl?zC&#xPHhYqBJ2+fH3!jG=AslUff+j+zdfX& zQfA#8- zB~R*&E8p`FdAX~jaE;(u^5{pld{}ua^HNKce@tankkj=Yh-L#aNm5;hS&>YF1_T3x!=Iljd^c41 z*=Lc&FCY7nbX`NE;iD)5Hx>c-ROl)3_#>Y`hp^TtwtkX$95YKIppytDBFMqnacM6-@O!iq(QgkB zOrHmh^|&H6ux778pV7N_^x=4VWH#>yH~B8^HSf-_!W+%wkE2Vp6mI68xfC|juf3$L z;#F^4yjj<~99Z(u)DBpJ*5aj=W; zZs3eQaoFWHP&b=(YT6{c%}vZj|KZ(26{Qp7muKp8xl~;IER!T@BRt9V<%HPNP6j^O zIUBOAcK1@*LUf6qK2j386U9!VLIQM^qV+EM#J)lapU)%-it8F#x(P55UZ)g99f3`K z%Sql&2p5yOcw$i`s2tF6va^S$EiFA&#!W^mZ+(Vgh-?c=IUUP0j}~7}YrAC1{M2i@ z93(t)@0ER0TaI88>&)=@v|0!^)zf2d*}6T=Pq-)X!`F=)|NGMQ%plgC)33`rt@&>6 zqwqh#dNK|>SIDHRi0e#1%9`0>#`&fa%e3nQt9v*+!iFsxovL>`JU`)i7<0Z!t7qYv zp&2O>MALe9(h+}vO{QM!lw8u2Y(EIayzJ$d)NS zB5s$M^vE2HPdCz;@j6MH48{bU)!tXIS@~cX(G3^gAxcuBB2K>KiWYyOTpBPlisQ&h_cMPiMYo6+pkowsE$gb9C zI3_{v7BB&*NS(h|dH%VG8cFvPoM~htY95+r#>(B1BN8%!tspr@jiY=#xhY6~jp^{?Rtcg$ z$I@4JYt}7jH6G7=_!zl7lvj_>ckR_;*l%&&!Mw%qGF$mhg?X}AK>I$VIt41} zf=LUVUW3*ppm)*_#sh9tmeBYh=gQsoM6@9%4#0nrauj}VPe^U8>liFyRw10LNkziI zb5EtS2_6@)qj<8kq|@7c_zHKt7`KSZ3vt-FrIb1Dlr;OMM@&Q z?wX5;jgKcs+$u8->Bn^)07Y>KsqdjdP+@TW^kEo0XD|zv@W8H03Yt=s$B#iO){}8u zKbr6BSy=WMo5atxhd3TIGTkg-*lSeAA7R;K-bQam?@sWmhOt1XmN z26LX+$KU7;bat1^idv-eTs%4|p7G`{LdWZdB4{HKg-286Q^`tjD0KS0Dr19|cG}R7 z;KugPnm|F9-KFWA@B|zYFic9^`mzx;ST&y4@1WLdx;=;(v<)QN@x%13B@S%(iuVQl zIlRd}0!+#dyJ&*nt|>vN-BhVFXkK;Tvl=Y4GnR|$DSg5;qEJs968WU7|K+)w6}9nz zdu8|K+BUD_n{EH+O&;CZ^`{~?^Hh4#J2*`t^k$U!jY2%Kyk8XT5XCRwEz&zul5mhd z?>^Ob3KX^zvl$!4V5dZtt5MA<;L~R#$*Qi#>}~UngnvJ}=%vN4XrS3i`l4y^#k~_Q zp``xMn#@DwX%RtZ-yYIRJ&M#cvi(xhSQ!qj8e09^m^~zush)#htphThpX1z*a_HH@ zQeJ&1)0X#VD64L69B46YOH-V2u<3K?rr4?OxMzKL^_<0b$sm`~n{~7A?eN!6); zI;rlTZUV2(Iy*3~PebV2mT^iw8{Icbkq2JNYuXw2yPa#DcG9(pp&^GULX*kIojsUT z;-}G<3s*HyP$cUlRWo5dBs`HmBw*i;nNyQ>j+0|xTKR+*n_(aWQPPKS*`Mb>@;mFJ z<*cxMTN@x3d8QMq9bUoieGEj#tp?qRM_p(W;ut?3UznABQY27fBdU?q# z5@QyKlJ0sw>5X7|l6oeNvI-~F5AFwf z?|GBdP$ZG&rK_3Aoy$J#JmNwM*AQYY)*6RFE@kBSowl@i=4izb$F-*1-M+kKbx2R~V$m>XdU;9~)#iUJEe7(@=~(14d((EoMM7C7{WC}BxX_;$ zOZ2bjU*T-fm5fPa_RA{M-=7)m|Nf9Q$i<3?7_{i@3qJT%zBG@gOI_bcq27C0$MtpX zeHgsdprZV32C2+9dYWl?Mp>g-IY=So>jNmkf&eMg<-*u>ZRLaVRB-;g9!?1k9#(in zpeIKWGR`1x58$CAKd1d&UooDB^Syg(J}pfZjRTW}3#HkW5>)#&Uz;ZBsS)nt?3>z% zEwDZBdj~{90L@g!e2khl~5e#JY9! zd{LFdh)Q19(NZiD>~vUU9-J%}^@3rT?H6mM8XDG@A~#NUv8c6_+cf%+u7yJ~>q-^% zx|P|wkgzNOX=s6CB9*t~`YpSxdSbmqc4T+8(q6oB827kdFZd_*r^pf9-fFBS1Eip( z2t`Ax->PB{st;_00(%*EWM2DA9RwD8Qpd$n$nO*HnpDe@*5}@Tj(xSy}6q~vivtqi)O0Btq7^)6tc5DCMZK2v1?*>jQr z&}1ku+4PYJ#+xmIH)pnTBcW{612U=Y6mRg3SiLTML^9y3Yo&WPmP_R%qsG@X!!*`T zDKz6%2NO`9xpx`uuf)T~=MFtNJ#G83j+p7l47*MZ20B{65g?fsUQ8<4E93GjJV-|( z0TnVNx4;v)al!N{3)R-aPpj2Zc*X)25uQdUHPVFdy5Ok?TC8WYoy;7*;Du~ihqvm4 zx$dRzhVa>IdhX@KgTm1W*Ksr8ajzVA14Jj!n+_`hfM->Z@x|_V*A@a+c0pD>U$=O9Ial-^L%DqriL zl0_=#dNvNr%N!-ltt`_|qwiE-KOCh{lauJ$j$eKG_QpQT=xTZQm@#n{^bdm~#Lt#v zcdM~tHLe6|sC{K`mC5qfQy`rO| zL&v~)c+#T9h`zLX>tzbAgVoM3;h4s0=DJAZOg6{M+E1@6uL z7>ES1i53ghL6_Gcx9-4sxBUcAO}TF;8pScF8_s0Y(9*sY71hDz*n0a6o8^^p4&Eb7 z8~iIgxmi4{FL|54+bXI#A?3ox2ni~J5ID@3J#psO0k&@BE{m>tAm>n$5S)O(bkXe?Owb+ptZK#LZdK6nWh(xFf5*Q*7R;cwr* z6$vM}0x_kx*2?~L5y0e-M8@(s%OWlydh_#^XN}GA!2?2xw}N~Mr=XIt1k&}0IM00_ zt~ktUZdQf3-+ZmBvsx)NQHy`BS=w^3aCT;Yvep2}3ZLVag6!?ArlaOctGgl($hR7& z!M(%TOS3vZcO3Th<_B<@I5NS5MxR#0m6|K%FY_q@)HrZa@ z&xLQ^U_YKgJ!4go&Xq`HwpxDON2l}%f(-k1zS;Q{OIf4Bc547!WG2N}ok68eUQP}& zhjb4T{6&sP@D0K>qgLU+`j$ZVE*2`P{`@NSq%i|W*7boTp3u_^Qvx3v%E2o^S<64Z5 zGZ&X!&mUBi$;632aL$SAa&GN>`a+o1ldSpJdIaQIPe2v5x0tX7Q3>n$emkkQ)0*FW zl~J0*j4J8~j<#FW`MaJ_Tudw+oYUMD=be;d(r1jtq%Txv!T=!*B#f<12p}P}MGA_D zK&cXjJtL|6eejfELbZuMg1G8cd(zblwMvAV5)n7;<5Q1{t}3TINBvyJJeJy5bSNRg z#|(8}I;qgDC_R645kt^bQ@9-IP(=xNJZ=k%O8E)nOE&#*v=XACylqqHgvf#)P_VLM zT~Whey%)RhkEX7wssgpc{QdkkKR>^OqTv)SoCDU{SV+wo%v~t}(>K zO48wZft)K_U2HYUm?c{NH(s%t;Q;rd3m2Zn?R(RB#{)F0VjG1wgd?et;#G552L-Xq zZU$tA;w9~Q0fTmHa*jTr7PD5yqi@RWaF>59>Jj3f&*v4SNyAz9G<1z8=l}pIceP>y@XPXnxN#>o ziNVNZOiawD9CY33!jDliPRH;)u zy>HTjx!e8a$YU_)t1YgVBZ)72H880(fs@P)0$82DRP0B?I7{P;mZInxsRsF zYyK%A(v#r8@yE*@vE$g&DeVy2gL1Mi`bQUj6r_M=6X~uq~$K z8s6ZD zg~I?f7|~f53eRUfq+#8&Gm>fBsQLHEVjf#A(}wZSSA!g6@n!F=yfc=XZjx0G&cIBqx;lkn2BzhDQIqp{qE!8 z4JFqx2*BHrkJ+oihO`7%!bvcFy0KxdHGi78{8GiUm~TT*{<mq`;scpFAh+f3Cw3QaxuC z?)o2#lnY~(Q$rpW4+qrv9ely6u?Z1CiH$9C?ws+?O=TUj*?LDUhHKw(?5vx(HC)Dh zY-JsPqs%yn@93~QQ{l3Pz%3kY=iJo5-`Xdt%5mY-!DbWLI1@QpBMhKA$4$erCGLY# zHhP{!%)o@&h}V2}vnsTqhm3Nf!r=Krnl@>tQ2a#Y@nNzQEVbe0N@iX@n%fR_xR9t> zD17g72pxZP)>5@B>D%Q})+~p#fP2r`Fc~es+himB#TUwhGbvf8!8EwVai`wXW9^L^ zs@f=2dMpyR8F&|3om=)#jf~p=O!5cepZr0T5+A z4p{fT%R}VEB&+AYz&`0lQGMxVsA<->rkyr%B$* zt(ovm-QguXK<{P}kklblD6_8Il#RqB(^KD!~gOLv;;^$)1>|Fl9>+p?~m+^6N;*4in z=HlYCTS^0jFi@JPd80Qikg3u$y zu~m7|%Im+V(X>3U;Ip(aGW6^Uisl@O!Cv6lIdh}V<~RjRn5jFb)hIijse?>xWs-_) zkB&{>rzh%p!k{`?L$xs)oJ(jIG8W@YyA&Xq9Eo1_9xWm%2V@7F+N*yniAVdw=P&{K!f{{!Z0G>TI%O4 z!9-u9KYd1qdi>_X*BFbNqiQY0H(XpYA2AZU*r-wt#Bpv5Wk2iErSG0udf{EsAML}Z z)uT3yI&2*!pOVs5Am7({E}2+fmKC0cwQy~yBf_GLH@tcV{p&*o7z%2$zBR12dG1;^yrFO>X>q_r`FRR}O^lrGeA=`1q$h}^Cb>||$-UZ15R9n`sq&9+ zII2%oX3sN-y^CNy>b@Q~&Jit?zepD`N~C8)(zAJm`On7yqOBQ>nU6{&Qoil0@Lx0| zU)G=Y9?fJNJ`CzV&N%J(>e_@?_GL$xjcH|_0UB;jhF3af{?g8R_WQff`Qz)y1;_K1 zO+}~Mkt?bpsZmHUsJsuLU><3DEl=@LZ+Ufz@Y)IX8sOLDEX&99$8!oeHp=;V33-3=YMH0UcaTbE0rc4@9adG7WI*9LO%7_ zy?P;jqFgM8 z!cAA&b~RFT!jbqEPMFe69kkr&UZY=-n%|` z%4M~$AfH=`5E*|GBL!u;Of%{WeS8bw+JF%%OKu;{@V0C+PrB`L*%i`;8B|-|B4|ZA=FsC=EYw5DO|R{i3dJm{4>> z)$olS+$}wBe}MD@3h|rUDS7EVS}i7A(J3Vgu{IFXAJ33%8{4cDcJCe?jr8>edWhkp z=jc3yf~|zM1-ykI~{If?_(}VS=^9HzVro%Ull0 zb%Ic)=1*U#T(ykaBz^kCs8L@Ig39o4nDH&nCUd#av9uzH221z0YdW^s50ehn(snKU$Q*-u}Nv= zF8>Y6N3rIqGdXun&YPvRB+ijOLj7UM=ZXPG*0hBRlQxHQ6`+R>P$*HVi3BYK3+}ms zfhRCBol6P>?d^w%Q>s$Y*~}N0k975|3k9I=6Cl$|&5RZn7KY)o*Vfif_zM z9HJ1VZgzSHLlSQhxc_Z#B{5vN&8!JW+A`lj>TEhPi>4X(ug<`(~5~ zXqzoUo`zsFP8TW~vTx^z$AfwH%(~OTf!a4%R2(!L9ZC(ZjmQi879 zpPSLrwHAwziPkesZZHX~s66z^s1)Hd`aw^Wa`7kgSHl z6eUT%*t5j9z2z@Y>+*7lrf2VZS@k9FlLo6$YIC+UMj#ZA{jQ9Kg@u}0(h+w7caVR< zFPZFzDq>_~r|0Hw-(H;(d)`)9I)S_9>uq}`D=d7T%P)2yIl>LH2O`Ao*&lYppIV59 zqFbQ>?-hWcG8MET!%!K<%LtXmj$Zt{=zB>b)(>}0Q$=sIZx`n-{$?E^xzsptYNBC(&9BcsJ} z3%rB{g!TDoZDO;d*Visc^Q@O|A=U(G0|sn&xzcH+pX6g>V?opUq*q8}Dtz1zhaIU+ zzG}Oj5yh%3G=j#ZnTZL0{EtsVr6K+;0DkP#emMep(|Zh=vcT> zXoA(LzPg&Fv8}bpY&hZbZe9y}W|bCemZFXAam1W(I}&u#Qt3rT;TTPSc<4yJRd_Y2 zBw4nXRidURrGa-#~=e?HCzZK2N)g{~ShzrnF3uM-kSvFcG zf~pUoKqBI{b#`vphBi*{w{#W4&}XwhmM|+3!kG5or5~>||8>&8()4;Uy~iqgZ$TQl ze;KotyJ>`nL4J_sgp#^v(v03PB!QAW^jx_upo~nr=!Xjn!|)qMq~Yo5wMDBo3&Z}y z`muLYz++HQM>pn6js(3zgl5B$DK8wyR?{qaF_=#qEaCEVG%vUk9dg- z(>QM;MT35RGn?}%7h|@9HxplE?H<>V;?nam>22Km9mPiG)ldP z-mb7iIyKKmquG1d4U#W=dvpXoIlj*+W9g0N9uS5qOtfRr$VMu0%cYDN!n|Aayxz-8 zqEb^*ipwt%jVaZVcONJCl96SiPR=}pHDxRyq2)ma0xC<4a`~O`#Y5U-Ys|nWYp1Tk zRiS>0J&jl1&KRMc5{upWKC^SL!{X%y)qWg^N6XeU=5{KSBo>7A^au~Xn&#DDaL%N3 z8>Tf1n!xHX9ZHQ;ouF9<$SP((0d&u&F0b=Q{F3>i0BPfeIioc9nKVvtxK<-)!t> zCeEY@RG>hIW4!n?qeAL zL@&JXYF@DMoKu%`h@FRLzFpwQ=~%PaCSEod!tbqhH2nfUlPGygm7%4UeRu z&-fXk7pcYXQlsohz1pqO`y^O}AFX2sE>FFpp&<40qK%A-LW>8_mg?woeBZdP(C%#I zLQIQbA6@%Vo_Kzh90dQO-hviFnPTw)ko9UB`P@9yr_ zuY|jWd&NRaN=mj`$H#K4egX1omq9?yMb3HNTg#uWrzg45P{p1E|DbqmC;1M0_h_#m zm9u4~sY?F@X5X|(eet=*!3FhkOftN;H)1y4QY;qM!=FSe5UC}TuEGa&jr7qrRJXKG zFRp!By5*d2P3^+`kUw^p1XaZA z1uHVb+y83EukVx~cNxl2=RY?cW*zK$4<9M~zZ3y{WIG*D)uK%fKzje{(E9xcF9B5N zKgRLLN2*ZrCYZ*)-Vyq9eSze>33!kFDE@pHs3M@)B_DU1|9IyI&^F+oH_{`0|9EE- zomimniYLi9f3A-t{DH110p`d5TG6lXiqT0VYQ$1c|1=J`&uAnPe--wdBB`O@0@-C4 zTS!^FVOadVtW`BA7kN$fb4-FiKDj&#tcTFd?=|Tk_E(Ap?9XCa-ujPcZ4U?g_P!YF zk5&0MHZ>cjX$=3pJ23bRVwup)m5_mBpNyjP7>`VT33nZd&S z|7X!&)C9 Date: Fri, 3 Mar 2023 15:08:57 +0100 Subject: [PATCH 06/22] Update --- blueprints/data-solutions/bq-ml/README.md | 15 +++++++++++++++ .../bq-ml/demo/sql/explain_predict.sql | 18 +++++++++++++++++- .../data-solutions/bq-ml/demo/sql/features.sql | 18 +++++++++++++++++- .../data-solutions/bq-ml/demo/sql/train.sql | 18 +++++++++++++++++- blueprints/data-solutions/bq-ml/main.tf | 6 ++++-- blueprints/data-solutions/bq-ml/outputs.tf | 2 ++ blueprints/data-solutions/bq-ml/variables.tf | 1 + blueprints/data-solutions/bq-ml/versions.tf | 2 ++ 8 files changed, 75 insertions(+), 5 deletions(-) diff --git a/blueprints/data-solutions/bq-ml/README.md b/blueprints/data-solutions/bq-ml/README.md index b6f2bd2b3..407ce1466 100644 --- a/blueprints/data-solutions/bq-ml/README.md +++ b/blueprints/data-solutions/bq-ml/README.md @@ -66,3 +66,18 @@ In the repository `demo` folder you can find an example on how to create a Verte | [vpc](outputs.tf#L38) | VPC Network. | | + +## Test + +```hcl +module "test" { + source = "./fabric/blueprints/data-solutions/bq-ml/" + project_create = { + billing_account_id = "123456-123456-123456" + parent = "folders/12345678" + } + project_id = "project-1" + prefix = "prefix" +} +# tftest modules=9 resources=46 +``` diff --git a/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql b/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql index 0d67bc7c8..7d36e0387 100644 --- a/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql +++ b/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql @@ -1,3 +1,19 @@ +/* +* Copyright 2023 Google LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* https://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + select * from ML.EXPLAIN_PREDICT(MODEL `{project-id}.{dataset}.{model-name}`, (select * except (session_id, session_starting_ts, user_id, has_purchased) @@ -5,4 +21,4 @@ from ML.EXPLAIN_PREDICT(MODEL `{project-id}.{dataset}.{model-name}`, where extract(ISOYEAR from session_starting_ts) = 2023 ), STRUCT(5 AS top_k_features, 0.5 as threshold) -) \ No newline at end of file +) diff --git a/blueprints/data-solutions/bq-ml/demo/sql/features.sql b/blueprints/data-solutions/bq-ml/demo/sql/features.sql index 63b79d821..2b55185f6 100644 --- a/blueprints/data-solutions/bq-ml/demo/sql/features.sql +++ b/blueprints/data-solutions/bq-ml/demo/sql/features.sql @@ -1,3 +1,19 @@ +/* +* Copyright 2023 Google LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* https://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + CREATE view if not exists `{project_id}.{dataset}.ecommerce_abt` as with abt as ( @@ -18,4 +34,4 @@ select abt.*, case when extract(DAYOFWEEK from session_starting_ts) in (1,7) the , (select count(distinct uo.order_id) from unnest(user_orders) uo where uo.order_creations_ts < session_starting_ts and status in ('Cancelled', 'Returned') ) as number_of_unsuccessful_orders from abt left join previous_orders pso - on abt.user_id = pso.user_id \ No newline at end of file + on abt.user_id = pso.user_id diff --git a/blueprints/data-solutions/bq-ml/demo/sql/train.sql b/blueprints/data-solutions/bq-ml/demo/sql/train.sql index 0f5517b8a..597623d0b 100644 --- a/blueprints/data-solutions/bq-ml/demo/sql/train.sql +++ b/blueprints/data-solutions/bq-ml/demo/sql/train.sql @@ -1,3 +1,19 @@ +/* +* Copyright 2023 Google LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* https://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + create or replace model `{project_id}.{dataset}.{model_name}` OPTIONS(model_type='{model_type}', input_label_cols=['has_purchased'], @@ -8,4 +24,4 @@ OPTIONS(model_type='{model_type}', ) as select * except (session_id, session_starting_ts, user_id) from `{project_id}.{dataset}.ecommerce_abt_table` -where extract(ISOYEAR from session_starting_ts) = 2022 \ No newline at end of file +where extract(ISOYEAR from session_starting_ts) = 2022 diff --git a/blueprints/data-solutions/bq-ml/main.tf b/blueprints/data-solutions/bq-ml/main.tf index 77ae55ea5..e91c7424c 100644 --- a/blueprints/data-solutions/bq-ml/main.tf +++ b/blueprints/data-solutions/bq-ml/main.tf @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +# tfdoc:file:description Core resources. + ############################################################################### # Project # ############################################################################### + locals { service_encryption_keys = var.service_encryption_keys shared_vpc_project = try(var.network_config.host_project, null) @@ -160,7 +163,7 @@ module "dataset" { project_id = module.project.project_id id = "${replace(var.prefix, "-", "_")}_data" encryption_key = try(local.service_encryption_keys.bq, null) # Example assignment of an encryption key - location = "US" + location = "US" } ############################################################################### @@ -239,7 +242,6 @@ resource "google_notebooks_instance" "playground" { service_account = module.service-account-notebook.email # Enable Secure Boot - shielded_instance_config { enable_secure_boot = true } diff --git a/blueprints/data-solutions/bq-ml/outputs.tf b/blueprints/data-solutions/bq-ml/outputs.tf index 2b62074b0..1a39e19ca 100644 --- a/blueprints/data-solutions/bq-ml/outputs.tf +++ b/blueprints/data-solutions/bq-ml/outputs.tf @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# tfdoc:file:description Output variables. + output "bucket" { description = "GCS Bucket URL." value = module.bucket.url diff --git a/blueprints/data-solutions/bq-ml/variables.tf b/blueprints/data-solutions/bq-ml/variables.tf index 3bd0ca65b..160b46166 100644 --- a/blueprints/data-solutions/bq-ml/variables.tf +++ b/blueprints/data-solutions/bq-ml/variables.tf @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# tfdoc:file:description Terraform variables. variable "location" { description = "The location where resources will be deployed." diff --git a/blueprints/data-solutions/bq-ml/versions.tf b/blueprints/data-solutions/bq-ml/versions.tf index 08492c6f9..a467162be 100644 --- a/blueprints/data-solutions/bq-ml/versions.tf +++ b/blueprints/data-solutions/bq-ml/versions.tf @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# tfdoc:file:description Terraform version. + terraform { required_version = ">= 1.3.1" required_providers { From 6526dda8c721e7216d4ba446d192383dfe025b53 Mon Sep 17 00:00:00 2001 From: Giorgio Conte Date: Fri, 3 Mar 2023 14:52:35 +0000 Subject: [PATCH 07/22] sql linting --- .../bq-ml/demo/sql/explain_predict.sql | 13 ++++++------ .../data-solutions/bq-ml/demo/sql/train.sql | 20 +++++++++---------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql b/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql index 7d36e0387..86309815b 100644 --- a/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql +++ b/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql @@ -14,11 +14,12 @@ * limitations under the License. */ -select * -from ML.EXPLAIN_PREDICT(MODEL `{project-id}.{dataset}.{model-name}`, - (select * except (session_id, session_starting_ts, user_id, has_purchased) - from `{project-id}.{dataset}.ecommerce_abt` - where extract(ISOYEAR from session_starting_ts) = 2023 +SELECT * +FROM ML.EXPLAIN_PREDICT(MODEL `{project-id}.{dataset}.{model-name}`, + (SELECT * EXCEPT (session_id, session_starting_ts, user_id, has_purchased) + FROM `{project-id}.{dataset}.ecommerce_abt` + WHERE extract(ISOYEAR FROM session_starting_ts) = 2023 ), - STRUCT(5 AS top_k_features, 0.5 as threshold) + STRUCT(5 AS top_k_features, 0.5 AS threshold) ) +LIMIT 100 diff --git a/blueprints/data-solutions/bq-ml/demo/sql/train.sql b/blueprints/data-solutions/bq-ml/demo/sql/train.sql index 597623d0b..72ce3bb12 100644 --- a/blueprints/data-solutions/bq-ml/demo/sql/train.sql +++ b/blueprints/data-solutions/bq-ml/demo/sql/train.sql @@ -14,14 +14,14 @@ * limitations under the License. */ -create or replace model `{project_id}.{dataset}.{model_name}` -OPTIONS(model_type='{model_type}', - input_label_cols=['has_purchased'], - enable_global_explain=TRUE, +CREATE OR REPLACE MODEL `{project_id}.{dataset}.{model_name}` +OPTIONS(MODEL_TYPE='{model_type}', + INPUT_LABEL_COLS=['has_purchased'], + ENABLE_GLOBAL_EXPLAIN=TRUE, MODEL_REGISTRY='VERTEX_AI', - data_split_method = 'RANDOM', - data_split_eval_fraction = {split_fraction} - ) as -select * except (session_id, session_starting_ts, user_id) -from `{project_id}.{dataset}.ecommerce_abt_table` -where extract(ISOYEAR from session_starting_ts) = 2022 + DATA_SPLIT_METHOD = 'RANDOM', + DATA_SPLIT_EVAL_FRACTION = {split_fraction} + ) AS +SELECT * EXCEPT (session_id, session_starting_ts, user_id) +FROM `{project_id}.{dataset}.ecommerce_abt_table` +WHERE extract(ISOYEAR FROM session_starting_ts) = 2022 \ No newline at end of file From 98e17bb997daf2b237b338158169c3a2c0c81788 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Sat, 4 Mar 2023 08:09:29 +0100 Subject: [PATCH 08/22] Fix readme. --- blueprints/data-solutions/README.md | 7 +++ blueprints/data-solutions/bq-ml/README.md | 50 +++++++++++-------- .../bq-ml/demo/bmql_pipeline.ipynb | 34 ++++++------- blueprints/data-solutions/bq-ml/variables.tf | 14 +++--- 4 files changed, 59 insertions(+), 46 deletions(-) diff --git a/blueprints/data-solutions/README.md b/blueprints/data-solutions/README.md index 3da4e7e92..651b87a47 100644 --- a/blueprints/data-solutions/README.md +++ b/blueprints/data-solutions/README.md @@ -69,3 +69,10 @@ This [blueprint](./vertex-mlops/) implements the infrastructure required to have This [blueprint](./shielded-folder/) implements an opinionated folder configuration according to GCP best practices. Configurations implemented on the folder would be beneficial to host workloads inheriting constraints from the folder they belong to.
+ +### BigQuery ML and Vertex Pipeline + + +This [blueprint](./bq-ml/) implements the infrastructure required to have a fully functional develpement environment using BigQuery ML and Vertex AI to develop and deploy a machine learning model to be used from Vertex AI endpoint or in BigQuery ML. + +
diff --git a/blueprints/data-solutions/bq-ml/README.md b/blueprints/data-solutions/bq-ml/README.md index 407ce1466..8390d7955 100644 --- a/blueprints/data-solutions/bq-ml/README.md +++ b/blueprints/data-solutions/bq-ml/README.md @@ -1,14 +1,14 @@ -# BQ ML and Vertex Pipeline +# BigQuery ML and Vertex Pipeline -This blueprint creates the infrastructure needed to deploy and run a Vertex AI environment to develop and deploy a machine learning model to be used from Vertex AI an endpoint or in BigQuery. +This blueprint creates the infrastructure needed to deploy and run a Vertex AI environment to develop and deploy a machine learning model to be used from Vertex AI endpoint or in BigQuery. -This is the high level diagram: +This is the high-level diagram: ![High-level diagram](diagram.png "High-level diagram") -It also includes the IAM wiring needed to make such scenarios work. Regional resources are used in this example, but the same logic will apply for 'dual regional', 'multi regional' or 'global' resources. +It also includes the IAM wiring needed to make such scenarios work. Regional resources are used in this example, but the same logic applies to 'dual regional', 'multi regional', or 'global' resources. -The example is designed to match real-world use cases with a minimum amount of resources, and be used as a starting point for your scenario. +The example is designed to match real-world use cases with a minimum amount of resources and be used as a starting point for your scenario. ## Managed resources and services @@ -30,15 +30,21 @@ This sample creates several distinct groups of resources: ### Virtual Private Cloud (VPC) design -As is often the case in real-world configurations, this blueprint accepts as input an existing Shared-VPC via the `network_config` variable. +As is often the case in real-world configurations, this blueprint accepts an existing Shared-VPC via the `network_config` variable as input. ### Customer Managed Encryption Key -As is often the case in real-world configurations, this blueprint accepts as input existing Cloud KMS keys to encrypt resources via the `service_encryption_keys` variable. +As is often the case in real-world configurations, this blueprint accepts as input existing Cloud KMS keys to encrypt resources via the `service_encryption_keys` variable. ## Demo -In the repository `demo` folder you can find an example on how to create a Vertex AI pipeline from a publically available dataset and deploy the model to be used from a Vertex AI managed endpoint or from within Bigquery. +In the repository [`demo`](./demo/) folder, you can find an example of creating a Vertex AI pipeline from a publically available dataset and deploying the model to be used from a Vertex AI managed endpoint or from within Bigquery. + +To run the demo: + +- Connect to the Vertex AI workbench instance +- Clone this repository +- Run the and run [`demo/bmql_pipeline.ipynb`](demo/bmql_pipeline.ipynb) Jupyter Notebook. @@ -46,27 +52,27 @@ In the repository `demo` folder you can find an example on how to create a Verte | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [prefix](variables.tf#L32) | Prefix used for resource names. | string | ✓ | | -| [project_id](variables.tf#L50) | Project id, references existing project if `project_create` is null. | string | ✓ | | -| [location](variables.tf#L16) | The location where resources will be deployed. | string | | "EU" | -| [network_config](variables.tf#L22) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | -| [project_create](variables.tf#L41) | Provide values if project creation is needed, uses existing project if null. Parent format: folders/folder_id or organizations/org_id. | object({…}) | | null | -| [region](variables.tf#L55) | The region where resources will be deployed. | string | | "europe-west1" | +| [prefix](variables.tf#L33) | Prefix used for resource names. | string | ✓ | | +| [project_id](variables.tf#L51) | Project id references existing project if `project_create` is null. | string | ✓ | | +| [location](variables.tf#L17) | The location where resources will be deployed. | string | | "US" | +| [network_config](variables.tf#L23) | Shared VPC network configurations to use. If null networks will be created in projects with pre-configured values. | object({…}) | | null | +| [project_create](variables.tf#L42) | Provide values if project creation is needed, use existing project if null. Parent format: folders/folder_id or organizations/org_id. | object({…}) | | null | +| [region](variables.tf#L56) | The region where resources will be deployed. | string | | "us-central1" | +| [service_encryption_keys](variables.tf#L62) | Cloud KMS to use to encrypt different services. The key location should match the service region. | object({…}) | | null | ## Outputs | name | description | sensitive | |---|---|:---:| -| [bucket](outputs.tf#L15) | GCS Bucket URL. | | -| [dataset](outputs.tf#L20) | GCS Bucket URL. | | -| [notebook](outputs.tf#L25) | Vertex AI notebook details. | | -| [project](outputs.tf#L33) | Project id. | | -| [service-account-vertex](outputs.tf#L43) | Service account to be used for Vertex AI pipelines | | -| [vertex-ai-metadata-store](outputs.tf#L48) | | | -| [vpc](outputs.tf#L38) | VPC Network. | | +| [bucket](outputs.tf#L17) | GCS Bucket URL. | | +| [dataset](outputs.tf#L22) | GCS Bucket URL. | | +| [notebook](outputs.tf#L27) | Vertex AI notebook details. | | +| [project](outputs.tf#L35) | Project id. | | +| [service-account-vertex](outputs.tf#L45) | Service account to be used for Vertex AI pipelines | | +| [vertex-ai-metadata-store](outputs.tf#L50) | | | +| [vpc](outputs.tf#L40) | VPC Network. | | - ## Test ```hcl diff --git a/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb b/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb index 07719fa95..592f3d107 100644 --- a/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb +++ b/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb @@ -53,18 +53,18 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Vertex Pipeline Definition\n", + "# Vertex AI Pipeline Definition\n", "\n", - "In the following code block we are defining our Vertex AI pipeline. It is made up of three main steps:\n", - "1. Create a BigQuery dataset which will contains the BQ ML models\n", - "2. Train the BQ ML model, in this case a logistic regression\n", - "3. Evaluate the BQ ML model with the standard evaluation metrics\n", + "In the following code block, we are defining our Vertex AI pipeline. It is made up of three main steps:\n", + "1. Create a BigQuery dataset that will contain the BigQuery ML models\n", + "2. Train the BigQuery ML model, in this case, a logistic regression\n", + "3. Evaluate the BigQuery ML model with the standard evaluation metrics\n", "\n", "The pipeline takes as input the following variables:\n", - "- ```model_name```: the display name of the BQ ML model\n", - "- ```split_fraction```: the percentage of data that will be used as evaluation dataset\n", - "- ```evaluate_job_conf```: bq dict configuration to define where to store evalution metrics\n", - "- ```dataset```: name of dataset where the artifacts will be stored\n", + "- ```model_name```: the display name of the BigQuery ML model\n", + "- ```split_fraction```: the percentage of data that will be used as an evaluation dataset\n", + "- ```evaluate_job_conf```: bq dict configuration to define where to store evaluation metrics\n", + "- ```dataset```: name of the dataset where the artifacts will be stored\n", "- ```project_id```: the project id where the GCP resources will be created\n", "- ```location```: BigQuery location" ] @@ -136,7 +136,7 @@ "source": [ "# Create Experiment\n", "\n", - "We will create an experiment in order to keep track of our trainings and tasks on a specific issue or problem." + "We will create an experiment to keep track of our training and tasks on a specific issue or problem." ] }, { @@ -158,11 +158,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Running the same training pipeline with different parameters\n", + "# Running the same training Verte AI pipeline with different parameters\n", "\n", - "One of the main tasks during the training phase is to compare different models or to try the same model with different inputs. We can leverage the power of Vertex Pipelines in order to submit the same steps with different training parameters. Thanks to the experiments artifact it is possible to easily keep track of all the tests that have been done. This simplifies the process to select the best model to deploy.\n", + "One of the main tasks during the training phase is to compare different models or to try the same model with different inputs. We can leverage the power of Vertex AI Pipelines to submit the same steps with different training parameters. Thanks to the experiments artifact, it is possible to easily keep track of all the tests that have been done. This simplifies the process of selecting the best model to deploy.\n", "\n", - "In this demo case, we will run the same training pipeline while changing the data split percentage between training and test data." + "In this demo case, we will run the same training pipeline while changing the split data percentage between training and test data." ] }, { @@ -193,9 +193,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Deploy the model to an endpoint\n", + "# Deploy the model on a Vertex AI endpoint\n", "\n", - "Thanks to the integration of Vertex Endpoint, it is very straightforward to create a live endpoint to serve the model which we prefer." + "Thanks to the integration of Vertex AI Endpoint, creating a live endpoint to serve the model we prefer is very straightforward." ] }, { @@ -221,7 +221,7 @@ "metadata": {}, "outputs": [], "source": [ - "# deploy the BQ ML model on Vertex Endpoint\n", + "# deploy the BigQuery ML model on Vertex Endpoint\n", "# have a coffe - this step can take up 10/15 minutes to finish\n", "model.deploy(endpoint=endpoint, deployed_model_display_name='bqml-deployed-model')" ] @@ -265,7 +265,7 @@ }, "language_info": { "name": "python", - "version": "3.10.9" + "version": "3.8.9" }, "orig_nbformat": 4, "vscode": { diff --git a/blueprints/data-solutions/bq-ml/variables.tf b/blueprints/data-solutions/bq-ml/variables.tf index 160b46166..13552a385 100644 --- a/blueprints/data-solutions/bq-ml/variables.tf +++ b/blueprints/data-solutions/bq-ml/variables.tf @@ -17,11 +17,11 @@ variable "location" { description = "The location where resources will be deployed." type = string - default = "EU" + default = "US" } variable "network_config" { - description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values." + description = "Shared VPC network configurations to use. If null networks will be created in projects with pre-configured values." type = object({ host_project = string network_self_link = string @@ -40,7 +40,7 @@ variable "prefix" { } variable "project_create" { - description = "Provide values if project creation is needed, uses existing project if null. Parent format: folders/folder_id or organizations/org_id." + description = "Provide values if project creation is needed, use existing project if null. Parent format: folders/folder_id or organizations/org_id." type = object({ billing_account_id = string parent = string @@ -49,18 +49,18 @@ variable "project_create" { } variable "project_id" { - description = "Project id, references existing project if `project_create` is null." + description = "Project id references existing project if `project_create` is null." type = string } variable "region" { description = "The region where resources will be deployed." type = string - default = "europe-west1" + default = "us-central1" } -variable "service_encryption_keys" { # service encription key - description = "Cloud KMS to use to encrypt different services. Key location should match service region." +variable "service_encryption_keys" { + description = "Cloud KMS to use to encrypt different services. The key location should match the service region." type = object({ bq = string compute = string From 0d4b599e99be6334ffa899cbb66a7dde9a884962 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Sat, 4 Mar 2023 08:13:53 +0100 Subject: [PATCH 09/22] Fix README --- blueprints/data-solutions/README.md | 4 ++-- blueprints/data-solutions/bq-ml/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/blueprints/data-solutions/README.md b/blueprints/data-solutions/README.md index 651b87a47..a2060560f 100644 --- a/blueprints/data-solutions/README.md +++ b/blueprints/data-solutions/README.md @@ -70,9 +70,9 @@ This [blueprint](./shielded-folder/) implements an opinionated folder configurat
-### BigQuery ML and Vertex Pipeline +### BigQuery ML and Vertex AI Pipeline - + This [blueprint](./bq-ml/) implements the infrastructure required to have a fully functional develpement environment using BigQuery ML and Vertex AI to develop and deploy a machine learning model to be used from Vertex AI endpoint or in BigQuery ML.
diff --git a/blueprints/data-solutions/bq-ml/README.md b/blueprints/data-solutions/bq-ml/README.md index 8390d7955..53bfdca66 100644 --- a/blueprints/data-solutions/bq-ml/README.md +++ b/blueprints/data-solutions/bq-ml/README.md @@ -1,4 +1,4 @@ -# BigQuery ML and Vertex Pipeline +# BigQuery ML and Vertex AI Pipeline This blueprint creates the infrastructure needed to deploy and run a Vertex AI environment to develop and deploy a machine learning model to be used from Vertex AI endpoint or in BigQuery. From ccd68b2fa6cae816dc8025c3a4c8269f0313d3b6 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Sat, 4 Mar 2023 08:19:47 +0100 Subject: [PATCH 10/22] Fix linting. --- blueprints/data-solutions/bq-ml/README.md | 3 +-- blueprints/data-solutions/bq-ml/outputs.tf | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/blueprints/data-solutions/bq-ml/README.md b/blueprints/data-solutions/bq-ml/README.md index 53bfdca66..79a73832c 100644 --- a/blueprints/data-solutions/bq-ml/README.md +++ b/blueprints/data-solutions/bq-ml/README.md @@ -45,7 +45,6 @@ To run the demo: - Connect to the Vertex AI workbench instance - Clone this repository - Run the and run [`demo/bmql_pipeline.ipynb`](demo/bmql_pipeline.ipynb) Jupyter Notebook. - ## Variables @@ -69,7 +68,7 @@ To run the demo: | [notebook](outputs.tf#L27) | Vertex AI notebook details. | | | [project](outputs.tf#L35) | Project id. | | | [service-account-vertex](outputs.tf#L45) | Service account to be used for Vertex AI pipelines | | -| [vertex-ai-metadata-store](outputs.tf#L50) | | | +| [vertex-ai-metadata-store](outputs.tf#L50) | Vertex AI Metadata Store ID. | | | [vpc](outputs.tf#L40) | VPC Network. | | diff --git a/blueprints/data-solutions/bq-ml/outputs.tf b/blueprints/data-solutions/bq-ml/outputs.tf index 1a39e19ca..a23ba484b 100644 --- a/blueprints/data-solutions/bq-ml/outputs.tf +++ b/blueprints/data-solutions/bq-ml/outputs.tf @@ -48,7 +48,6 @@ output "service-account-vertex" { } output "vertex-ai-metadata-store" { - description = "" + description = "Vertex AI Metadata Store ID." value = google_vertex_ai_metadata_store.store.id - } From f8a7aa865acc8239dfefbcc7f4bcc28567ee8ffa Mon Sep 17 00:00:00 2001 From: lcaggio Date: Sat, 4 Mar 2023 08:25:29 +0100 Subject: [PATCH 11/22] Fix test. --- blueprints/data-solutions/bq-ml/README.md | 6 +++--- blueprints/data-solutions/bq-ml/outputs.tf | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/blueprints/data-solutions/bq-ml/README.md b/blueprints/data-solutions/bq-ml/README.md index 79a73832c..2ea841901 100644 --- a/blueprints/data-solutions/bq-ml/README.md +++ b/blueprints/data-solutions/bq-ml/README.md @@ -67,9 +67,9 @@ To run the demo: | [dataset](outputs.tf#L22) | GCS Bucket URL. | | | [notebook](outputs.tf#L27) | Vertex AI notebook details. | | | [project](outputs.tf#L35) | Project id. | | -| [service-account-vertex](outputs.tf#L45) | Service account to be used for Vertex AI pipelines | | -| [vertex-ai-metadata-store](outputs.tf#L50) | Vertex AI Metadata Store ID. | | -| [vpc](outputs.tf#L40) | VPC Network. | | +| [service-account-vertex](outputs.tf#L40) | Service account to be used for Vertex AI pipelines. | | +| [vertex-ai-metadata-store](outputs.tf#L45) | Vertex AI Metadata Store ID. | | +| [vpc](outputs.tf#L50) | VPC Network. | | ## Test diff --git a/blueprints/data-solutions/bq-ml/outputs.tf b/blueprints/data-solutions/bq-ml/outputs.tf index a23ba484b..8299ce2ff 100644 --- a/blueprints/data-solutions/bq-ml/outputs.tf +++ b/blueprints/data-solutions/bq-ml/outputs.tf @@ -37,13 +37,8 @@ output "project" { value = module.project.project_id } -output "vpc" { - description = "VPC Network." - value = local.vpc -} - output "service-account-vertex" { - description = "Service account to be used for Vertex AI pipelines" + description = "Service account to be used for Vertex AI pipelines." value = module.service-account-vertex.email } @@ -51,3 +46,8 @@ output "vertex-ai-metadata-store" { description = "Vertex AI Metadata Store ID." value = google_vertex_ai_metadata_store.store.id } + +output "vpc" { + description = "VPC Network." + value = local.vpc +} From 652495e530e7ba1dae3e9d08ad616d33d142aee1 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Sat, 4 Mar 2023 14:12:50 +0100 Subject: [PATCH 12/22] Update versions. --- blueprints/data-solutions/bq-ml/README.md | 1 + blueprints/data-solutions/bq-ml/versions.tf | 8 ++------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/blueprints/data-solutions/bq-ml/README.md b/blueprints/data-solutions/bq-ml/README.md index 2ea841901..ca6a1676a 100644 --- a/blueprints/data-solutions/bq-ml/README.md +++ b/blueprints/data-solutions/bq-ml/README.md @@ -84,5 +84,6 @@ module "test" { project_id = "project-1" prefix = "prefix" } + # tftest modules=9 resources=46 ``` diff --git a/blueprints/data-solutions/bq-ml/versions.tf b/blueprints/data-solutions/bq-ml/versions.tf index a467162be..a7c764bff 100644 --- a/blueprints/data-solutions/bq-ml/versions.tf +++ b/blueprints/data-solutions/bq-ml/versions.tf @@ -12,20 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -# tfdoc:file:description Terraform version. - terraform { required_version = ">= 1.3.1" required_providers { google = { source = "hashicorp/google" - version = ">= 4.50.0" # tftest + version = ">= 4.55.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 4.50.0" # tftest + version = ">= 4.55.0" # tftest } } } - - From 2b8ba16a9a5db926117df6d03c6337abc40d5686 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Sat, 4 Mar 2023 14:32:54 +0100 Subject: [PATCH 13/22] Fix typos --- blueprints/data-solutions/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/data-solutions/README.md b/blueprints/data-solutions/README.md index a2060560f..5fc832b3b 100644 --- a/blueprints/data-solutions/README.md +++ b/blueprints/data-solutions/README.md @@ -73,6 +73,6 @@ This [blueprint](./shielded-folder/) implements an opinionated folder configurat ### BigQuery ML and Vertex AI Pipeline -This [blueprint](./bq-ml/) implements the infrastructure required to have a fully functional develpement environment using BigQuery ML and Vertex AI to develop and deploy a machine learning model to be used from Vertex AI endpoint or in BigQuery ML. +This [blueprint](./bq-ml/) implements the infrastructure required to have a fully functional development environment using BigQuery ML and Vertex AI to develop and deploy a machine learning model to be used from Vertex AI endpoint or in BigQuery ML.
From 9e19f8960861fe61830801eab27111422f1d7a4e Mon Sep 17 00:00:00 2001 From: lcaggio Date: Sun, 5 Mar 2023 22:02:41 +0100 Subject: [PATCH 14/22] Implement PR comments. --- blueprints/README.md | 2 +- blueprints/data-solutions/README.md | 3 +- blueprints/data-solutions/bq-ml/README.md | 22 +- .../data-solutions/bq-ml/datastorage.tf | 32 +++ .../bq-ml/demo/sql/explain_predict.sql | 16 +- .../bq-ml/demo/sql/features.sql | 71 ++++-- .../data-solutions/bq-ml/demo/sql/train.sql | 6 +- blueprints/data-solutions/bq-ml/main.tf | 202 +----------------- blueprints/data-solutions/bq-ml/variables.tf | 2 +- blueprints/data-solutions/bq-ml/vertex.tf | 104 +++++++++ blueprints/data-solutions/bq-ml/vpc.tf | 64 ++++++ 11 files changed, 285 insertions(+), 239 deletions(-) create mode 100644 blueprints/data-solutions/bq-ml/datastorage.tf create mode 100644 blueprints/data-solutions/bq-ml/vertex.tf create mode 100644 blueprints/data-solutions/bq-ml/vpc.tf diff --git a/blueprints/README.md b/blueprints/README.md index e7136d9ce..49b374eaa 100644 --- a/blueprints/README.md +++ b/blueprints/README.md @@ -6,7 +6,7 @@ Currently available blueprints: - **apigee** - [Apigee Hybrid on GKE](./apigee/hybrid-gke/), [Apigee X analytics in BigQuery](./apigee/bigquery-analytics), [Apigee network patterns](./apigee/network-patterns/) - **cloud operations** - [Active Directory Federation Services](./cloud-operations/adfs), [Cloud Asset Inventory feeds for resource change tracking and remediation](./cloud-operations/asset-inventory-feed-remediation), [Fine-grained Cloud DNS IAM via Service Directory](./cloud-operations/dns-fine-grained-iam), [Cloud DNS & Shared VPC design](./cloud-operations/dns-shared-vpc), [Delegated Role Grants](./cloud-operations/iam-delegated-role-grants), [Networking Dashboard](./cloud-operations/network-dashboard), [Managing on-prem service account keys by uploading public keys](./cloud-operations/onprem-sa-key-management), [Compute Image builder with Hashicorp Packer](./cloud-operations/packer-image-builder), [Packer example](./cloud-operations/packer-image-builder/packer), [Compute Engine quota monitoring](./cloud-operations/quota-monitoring), [Scheduled Cloud Asset Inventory Export to Bigquery](./cloud-operations/scheduled-asset-inventory-export-bq), [Configuring workload identity federation with Terraform Cloud/Enterprise workflows](./cloud-operations/terraform-cloud-dynamic-credentials), [TCP healthcheck and restart for unmanaged GCE instances](./cloud-operations/unmanaged-instances-healthcheck), [Migrate for Compute Engine (v5) blueprints](./cloud-operations/vm-migration), [Configuring workload identity federation to access Google Cloud resources from apps running on Azure](./cloud-operations/workload-identity-federation) -- **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder) +- **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder), [BigQuery ML and Vertex AI Pipeline](./data-solutions/dq'ml) - **factories** - [The why and the how of Resource Factories](./factories), [Google Cloud Identity Group Factory](./factories/cloud-identity-group-factory), [Google Cloud BQ Factory](./factories/bigquery-factory), [Google Cloud VPC Firewall Factory](./factories/net-vpc-firewall-yaml), [Minimal Project Factory](./factories/project-factory) - **GKE** - [Binary Authorization Pipeline Blueprint](./gke/binauthz), [Storage API](./gke/binauthz/image), [Multi-cluster mesh on GKE (fleet API)](./gke/multi-cluster-mesh-gke-fleet-api), [GKE Multitenant Blueprint](./gke/multitenant-fleet), [Shared VPC with GKE support](./networking/shared-vpc-gke/) - **networking** - [Calling a private Cloud Function from On-premises](./networking/private-cloud-function-from-onprem), [Decentralized firewall management](./networking/decentralized-firewall), [Decentralized firewall validator](./networking/decentralized-firewall/validator), [Network filtering with Squid](./networking/filtering-proxy), [GLB and multi-regional daisy-chaining through hybrid NEGs](./networking/glb-hybrid-neg-internal), [Hybrid connectivity to on-premise services through PSC](./networking/psc-hybrid), [HTTP Load Balancer with Cloud Armor](./networking/glb-and-armor), [Hub and Spoke via VPN](./networking/hub-and-spoke-vpn), [Hub and Spoke via VPC Peering](./networking/hub-and-spoke-peering), [Internal Load Balancer as Next Hop](./networking/ilb-next-hop), [Network filtering with Squid with isolated VPCs using Private Service Connect](./networking/filtering-proxy-psc), On-prem DNS and Google Private Access, [PSC Producer](./networking/psc-hybrid/psc-producer), [PSC Consumer](./networking/psc-hybrid/psc-consumer), [Shared VPC with optional GKE cluster](./networking/shared-vpc-gke) diff --git a/blueprints/data-solutions/README.md b/blueprints/data-solutions/README.md index 5fc832b3b..9cef8bc26 100644 --- a/blueprints/data-solutions/README.md +++ b/blueprints/data-solutions/README.md @@ -73,6 +73,5 @@ This [blueprint](./shielded-folder/) implements an opinionated folder configurat ### BigQuery ML and Vertex AI Pipeline -This [blueprint](./bq-ml/) implements the infrastructure required to have a fully functional development environment using BigQuery ML and Vertex AI to develop and deploy a machine learning model to be used from Vertex AI endpoint or in BigQuery ML. - +This [blueprint](./bq-ml/) provides the necessary infrastructure to create a complete development environment for building and deploying machine learning models using BigQuery ML and Vertex AI. With this blueprint, you can deploy your models to a Vertex AI endpoint or use them within BigQuery ML.
diff --git a/blueprints/data-solutions/bq-ml/README.md b/blueprints/data-solutions/bq-ml/README.md index ca6a1676a..3efa457e5 100644 --- a/blueprints/data-solutions/bq-ml/README.md +++ b/blueprints/data-solutions/bq-ml/README.md @@ -1,6 +1,6 @@ # BigQuery ML and Vertex AI Pipeline -This blueprint creates the infrastructure needed to deploy and run a Vertex AI environment to develop and deploy a machine learning model to be used from Vertex AI endpoint or in BigQuery. +This blueprint provides the necessary infrastructure to create a complete development environment for building and deploying machine learning models using BigQuery ML and Vertex AI. With this blueprint, you can deploy your models to a Vertex AI endpoint or use them within BigQuery ML. This is the high-level diagram: @@ -30,15 +30,15 @@ This sample creates several distinct groups of resources: ### Virtual Private Cloud (VPC) design -As is often the case in real-world configurations, this blueprint accepts an existing Shared-VPC via the `network_config` variable as input. +As is often the case in real-world configurations, this blueprint accepts an existing Shared-VPC via the `vpc_config` variable as input. -### Customer Managed Encryption Key +### Customer Managed Encryption Keys As is often the case in real-world configurations, this blueprint accepts as input existing Cloud KMS keys to encrypt resources via the `service_encryption_keys` variable. ## Demo -In the repository [`demo`](./demo/) folder, you can find an example of creating a Vertex AI pipeline from a publically available dataset and deploying the model to be used from a Vertex AI managed endpoint or from within Bigquery. +In the [`demo`](./demo/) folder, you can find an example of creating a Vertex AI pipeline from a publicly available dataset and deploying the model to be used from a Vertex AI managed endpoint or from within Bigquery. To run the demo: @@ -47,6 +47,18 @@ To run the demo: - Run the and run [`demo/bmql_pipeline.ipynb`](demo/bmql_pipeline.ipynb) Jupyter Notebook. +## Files + +| name | description | modules | resources | +|---|---|---|---| +| [datastorage.tf](./datastorage.tf) | Datastorage resources. | bigquery-dataset · gcs | | +| [main.tf](./main.tf) | Core resources. | project | | +| [outputs.tf](./outputs.tf) | Output variables. | | | +| [variables.tf](./variables.tf) | Terraform variables. | | | +| [versions.tf](./versions.tf) | Version pins. | | | +| [vertex.tf](./vertex.tf) | Vertex resources. | iam-service-account | google_notebooks_instance · google_vertex_ai_metadata_store | +| [vpc.tf](./vpc.tf) | VPC resources. | net-cloudnat · net-vpc · net-vpc-firewall | google_project_iam_member | + ## Variables | name | description | type | required | default | @@ -54,10 +66,10 @@ To run the demo: | [prefix](variables.tf#L33) | Prefix used for resource names. | string | ✓ | | | [project_id](variables.tf#L51) | Project id references existing project if `project_create` is null. | string | ✓ | | | [location](variables.tf#L17) | The location where resources will be deployed. | string | | "US" | -| [network_config](variables.tf#L23) | Shared VPC network configurations to use. If null networks will be created in projects with pre-configured values. | object({…}) | | null | | [project_create](variables.tf#L42) | Provide values if project creation is needed, use existing project if null. Parent format: folders/folder_id or organizations/org_id. | object({…}) | | null | | [region](variables.tf#L56) | The region where resources will be deployed. | string | | "us-central1" | | [service_encryption_keys](variables.tf#L62) | Cloud KMS to use to encrypt different services. The key location should match the service region. | object({…}) | | null | +| [vpc_config](variables.tf#L23) | Shared VPC network configurations to use. If null networks will be created in projects with pre-configured values. | object({…}) | | null | ## Outputs diff --git a/blueprints/data-solutions/bq-ml/datastorage.tf b/blueprints/data-solutions/bq-ml/datastorage.tf new file mode 100644 index 000000000..ad5910952 --- /dev/null +++ b/blueprints/data-solutions/bq-ml/datastorage.tf @@ -0,0 +1,32 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# tfdoc:file:description Datastorage resources. + +module "bucket" { + source = "../../../modules/gcs" + project_id = module.project.project_id + prefix = var.prefix + location = var.location + name = "data" + encryption_key = try(local.service_encryption_keys.storage, null) # Example assignment of an encryption key +} + +module "dataset" { + source = "../../../modules/bigquery-dataset" + project_id = module.project.project_id + id = "${replace(var.prefix, "-", "_")}_data" + encryption_key = try(local.service_encryption_keys.bq, null) # Example assignment of an encryption key + location = "US" +} diff --git a/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql b/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql index 86309815b..0c8c20635 100644 --- a/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql +++ b/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql @@ -14,12 +14,10 @@ * limitations under the License. */ -SELECT * -FROM ML.EXPLAIN_PREDICT(MODEL `{project-id}.{dataset}.{model-name}`, - (SELECT * EXCEPT (session_id, session_starting_ts, user_id, has_purchased) - FROM `{project-id}.{dataset}.ecommerce_abt` - WHERE extract(ISOYEAR FROM session_starting_ts) = 2023 - ), - STRUCT(5 AS top_k_features, 0.5 AS threshold) -) -LIMIT 100 +SELECT * +FROM ML.EXPLAIN_PREDICT(MODEL `{project-id}.{dataset}.{model-name}`, + (SELECT * EXCEPT (session_id, session_starting_ts, user_id, has_purchased) + FROM `{project-id}.{dataset}.ecommerce_abt` + WHERE extract(ISOYEAR FROM session_starting_ts) = 2023), + STRUCT(5 AS top_k_features, 0.5 AS threshold)) +LIMIT 100 diff --git a/blueprints/data-solutions/bq-ml/demo/sql/features.sql b/blueprints/data-solutions/bq-ml/demo/sql/features.sql index 2b55185f6..a28ba85bc 100644 --- a/blueprints/data-solutions/bq-ml/demo/sql/features.sql +++ b/blueprints/data-solutions/bq-ml/demo/sql/features.sql @@ -14,24 +14,55 @@ * limitations under the License. */ -CREATE view if not exists `{project_id}.{dataset}.ecommerce_abt` as - -with abt as ( - SELECT user_id, session_id, city, postal_code, browser,traffic_source, min(created_at) as session_starting_ts, sum(case when event_type = 'purchase' then 1 else 0 end) has_purchased - FROM `bigquery-public-data.thelook_ecommerce.events` - group by user_id, session_id, city, postal_code, browser, traffic_source -), previous_orders as ( -select user_id, array_agg (struct(created_at as order_creations_ts, o.order_id, o.status, oi.order_cost )) as user_orders - from `bigquery-public-data.thelook_ecommerce.orders` o - join (select order_id, sum(sale_price) order_cost - from `bigquery-public-data.thelook_ecommerce.order_items` group by 1) oi - on o.order_id = oi.order_id - group by 1 +CREATE VIEW if NOT EXISTS `{project_id}.{dataset}.ecommerce_abt` AS +WITH abt AS ( + SELECT user_id, + session_id, + city, + postal_code, + browser, + traffic_source, + min(created_at) AS session_starting_ts, + sum(CASE WHEN event_type = 'purchase' THEN 1 ELSE 0 END) has_purchased + FROM `bigquery-public-data.thelook_ecommerce.events` + GROUP BY user_id, + session_id, + city, + postal_code, + browser, + traffic_source +), previous_orders AS ( + SELECT user_id, + array_agg (struct(created_at AS order_creations_ts, + o.order_id, + o.status, + oi.order_cost)) as user_orders + FROM `bigquery-public-data.thelook_ecommerce.orders` o + JOIN (SELECT order_id, + sum(sale_price) order_cost + FROM `bigquery-public-data.thelook_ecommerce.order_items` + GROUP BY 1) oi + ON o.order_id = oi.order_id + GROUP BY 1 ) -select abt.*, case when extract(DAYOFWEEK from session_starting_ts) in (1,7) then 'WEEKEND' else 'WEEKDAY' end as day_of_week, extract(hour from session_starting_ts) hour_of_day - , (select count(distinct uo.order_id) from unnest(user_orders) uo where uo.order_creations_ts < session_starting_ts and status in ('Shipped', 'Complete', 'Processing') ) as number_of_successful_orders - , IFNULL((select sum(distinct uo.order_cost) from unnest(user_orders) uo where uo.order_creations_ts < session_starting_ts and status in ('Shipped', 'Complete', 'Processing') ), 0) as sum_previous_orders - , (select count(distinct uo.order_id) from unnest(user_orders) uo where uo.order_creations_ts < session_starting_ts and status in ('Cancelled', 'Returned') ) as number_of_unsuccessful_orders -from abt - left join previous_orders pso - on abt.user_id = pso.user_id +SELECT abt.*, + CASE WHEN extract(DAYOFWEEK FROM session_starting_ts) IN (1,7) + THEN 'WEEKEND' + ELSE 'WEEKDAY' + END AS day_of_week, + extract(HOUR FROM session_starting_ts) hour_of_day, + (SELECT count(DISTINCT uo.order_id) + FROM unnest(user_orders) uo + WHERE uo.order_creations_ts < session_starting_ts + AND status IN ('Shipped', 'Complete', 'Processing')) AS number_of_successful_orders, + IFNULL((SELECT sum(DISTINCT uo.order_cost) + FROM unnest(user_orders) uo + WHERE uo.order_creations_ts < session_starting_ts + AND status IN ('Shipped', 'Complete', 'Processing')), 0) AS sum_previous_orders, + (SELECT count(DISTINCT uo.order_id) + FROM unnest(user_orders) uo + WHERE uo.order_creations_ts < session_starting_ts + AND status IN ('Cancelled', 'Returned')) AS number_of_unsuccessful_orders +FROM abt +LEFT JOIN previous_orders pso +ON abt.user_id = pso.user_id diff --git a/blueprints/data-solutions/bq-ml/demo/sql/train.sql b/blueprints/data-solutions/bq-ml/demo/sql/train.sql index 72ce3bb12..2c30f2e67 100644 --- a/blueprints/data-solutions/bq-ml/demo/sql/train.sql +++ b/blueprints/data-solutions/bq-ml/demo/sql/train.sql @@ -22,6 +22,6 @@ OPTIONS(MODEL_TYPE='{model_type}', DATA_SPLIT_METHOD = 'RANDOM', DATA_SPLIT_EVAL_FRACTION = {split_fraction} ) AS -SELECT * EXCEPT (session_id, session_starting_ts, user_id) -FROM `{project_id}.{dataset}.ecommerce_abt_table` -WHERE extract(ISOYEAR FROM session_starting_ts) = 2022 \ No newline at end of file +SELECT * EXCEPT (session_id, session_starting_ts, user_id) +FROM `{project_id}.{dataset}.ecommerce_abt_table` +WHERE extract(ISOYEAR FROM session_starting_ts) = 2022 \ No newline at end of file diff --git a/blueprints/data-solutions/bq-ml/main.tf b/blueprints/data-solutions/bq-ml/main.tf index e91c7424c..6ec7766e7 100644 --- a/blueprints/data-solutions/bq-ml/main.tf +++ b/blueprints/data-solutions/bq-ml/main.tf @@ -14,45 +14,20 @@ # tfdoc:file:description Core resources. -############################################################################### -# Project # -############################################################################### - locals { service_encryption_keys = var.service_encryption_keys - shared_vpc_project = try(var.network_config.host_project, null) - + shared_vpc_project = try(var.vpc_config.host_project, null) subnet = ( local.use_shared_vpc - ? var.network_config.subnet_self_link + ? var.vpc_config.subnet_self_link : values(module.vpc.0.subnet_self_links)[0] ) + use_shared_vpc = var.vpc_config != null vpc = ( local.use_shared_vpc - ? var.network_config.network_self_link + ? var.vpc_config.network_self_link : module.vpc.0.self_link ) - use_shared_vpc = var.network_config != null - - shared_vpc_bindings = { - "roles/compute.networkUser" = [ - "robot-df", "notebooks" - ] - } - - shared_vpc_role_members = { - robot-df = "serviceAccount:${module.project.service_accounts.robots.dataflow}" - notebooks = "serviceAccount:${module.project.service_accounts.robots.notebooks}" - } - - # reassemble in a format suitable for for_each - shared_vpc_bindings_map = { - for binding in flatten([ - for role, members in local.shared_vpc_bindings : [ - for member in members : { role = role, member = member } - ] - ]) : "${binding.role}-${binding.member}" => binding - } } module "project" { @@ -75,12 +50,10 @@ module "project" { "storage.googleapis.com", "storage-component.googleapis.com" ] - shared_vpc_service_config = local.shared_vpc_project == null ? null : { attach = true host_project = local.shared_vpc_project } - service_encryption_key_ids = { compute = [try(local.service_encryption_keys.compute, null)] bq = [try(local.service_encryption_keys.bq, null)] @@ -90,170 +63,3 @@ module "project" { disable_on_destroy = false, disable_dependent_services = false } } - -############################################################################### -# Networking # -############################################################################### - -module "vpc" { - source = "../../../modules/net-vpc" - count = local.use_shared_vpc ? 0 : 1 - project_id = module.project.project_id - name = "${var.prefix}-vpc" - subnets = [ - { - ip_cidr_range = "10.0.0.0/20" - name = "${var.prefix}-subnet" - region = var.region - } - ] -} - -module "vpc-firewall" { - source = "../../../modules/net-vpc-firewall" - count = local.use_shared_vpc ? 0 : 1 - project_id = module.project.project_id - network = module.vpc.0.name - default_rules_config = { - admin_ranges = ["10.0.0.0/20"] - } - ingress_rules = { - #TODO Remove and rely on 'ssh' tag once terraform-provider-google/issues/9273 is fixed - ("${var.prefix}-iap") = { - description = "Enable SSH from IAP on Notebooks." - source_ranges = ["35.235.240.0/20"] - targets = ["notebook-instance"] - rules = [{ protocol = "tcp", ports = [22] }] - } - } -} - -module "cloudnat" { - source = "../../../modules/net-cloudnat" - count = local.use_shared_vpc ? 0 : 1 - project_id = module.project.project_id - name = "${var.prefix}-default" - region = var.region - router_network = module.vpc.0.name -} - -resource "google_project_iam_member" "shared_vpc" { - count = local.use_shared_vpc ? 1 : 0 - project = var.network_config.host_project - role = "roles/compute.networkUser" - member = "serviceAccount:${module.project.service_accounts.robots.notebooks}" -} - - -############################################################################### -# Storage # -############################################################################### - -module "bucket" { - source = "../../../modules/gcs" - project_id = module.project.project_id - prefix = var.prefix - location = var.location - name = "data" - encryption_key = try(local.service_encryption_keys.storage, null) # Example assignment of an encryption key -} - -module "dataset" { - source = "../../../modules/bigquery-dataset" - project_id = module.project.project_id - id = "${replace(var.prefix, "-", "_")}_data" - encryption_key = try(local.service_encryption_keys.bq, null) # Example assignment of an encryption key - location = "US" -} - -############################################################################### -# Vertex AI # -############################################################################### -resource "google_vertex_ai_metadata_store" "store" { - provider = google-beta - project = module.project.project_id - name = "default" #"${var.prefix}-metadata-store" - description = "Vertex Ai Metadata Store" - region = var.region - #TODO Check/Implement P4SA logic for IAM role - # encryption_spec { - # kms_key_name = var.service_encryption_keys.ai_metadata_store - # } -} - -module "service-account-notebook" { - source = "../../../modules/iam-service-account" - project_id = module.project.project_id - name = "notebook-sa" - iam_project_roles = { - (module.project.project_id) = [ - "roles/bigquery.admin", - "roles/bigquery.jobUser", - "roles/bigquery.dataEditor", - "roles/bigquery.user", - "roles/dialogflow.client", - "roles/storage.admin", - "roles/aiplatform.user", - "roles/iam.serviceAccountUser" - ] - } -} - -module "service-account-vertex" { - source = "../../../modules/iam-service-account" - project_id = module.project.project_id - name = "vertex-sa" - iam_project_roles = { - (module.project.project_id) = [ - "roles/bigquery.admin", - "roles/bigquery.jobUser", - "roles/bigquery.dataEditor", - "roles/bigquery.user", - "roles/dialogflow.client", - "roles/storage.admin", - "roles/aiplatform.user" - ] - } -} - -resource "google_notebooks_instance" "playground" { - name = "${var.prefix}-notebook" - location = format("%s-%s", var.region, "b") - machine_type = "e2-medium" - project = module.project.project_id - - container_image { - repository = "gcr.io/deeplearning-platform-release/base-cpu" - tag = "latest" - } - - install_gpu_driver = true - boot_disk_type = "PD_SSD" - boot_disk_size_gb = 110 - disk_encryption = try(local.service_encryption_keys.compute != null, false) ? "CMEK" : null - kms_key = try(local.service_encryption_keys.compute, null) - - no_public_ip = true - no_proxy_access = false - - network = local.vpc - subnet = local.subnet - - service_account = module.service-account-notebook.email - - # Enable Secure Boot - shielded_instance_config { - enable_secure_boot = true - } - - # Remove once terraform-provider-google/issues/9164 is fixed - lifecycle { - ignore_changes = [disk_encryption, kms_key] - } - - #TODO Uncomment once terraform-provider-google/issues/9273 is fixed - # tags = ["ssh"] - depends_on = [ - google_project_iam_member.shared_vpc, - ] -} diff --git a/blueprints/data-solutions/bq-ml/variables.tf b/blueprints/data-solutions/bq-ml/variables.tf index 13552a385..058284ee7 100644 --- a/blueprints/data-solutions/bq-ml/variables.tf +++ b/blueprints/data-solutions/bq-ml/variables.tf @@ -20,7 +20,7 @@ variable "location" { default = "US" } -variable "network_config" { +variable "vpc_config" { description = "Shared VPC network configurations to use. If null networks will be created in projects with pre-configured values." type = object({ host_project = string diff --git a/blueprints/data-solutions/bq-ml/vertex.tf b/blueprints/data-solutions/bq-ml/vertex.tf new file mode 100644 index 000000000..061bf7dac --- /dev/null +++ b/blueprints/data-solutions/bq-ml/vertex.tf @@ -0,0 +1,104 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# tfdoc:file:description Vertex resources. + +resource "google_vertex_ai_metadata_store" "store" { + provider = google-beta + project = module.project.project_id + name = "default" #"${var.prefix}-metadata-store" + description = "Vertex Ai Metadata Store" + region = var.region + #TODO Check/Implement P4SA logic for IAM role + # encryption_spec { + # kms_key_name = var.service_encryption_keys.ai_metadata_store + # } +} + +module "service-account-notebook" { + source = "../../../modules/iam-service-account" + project_id = module.project.project_id + name = "notebook-sa" + iam_project_roles = { + (module.project.project_id) = [ + "roles/bigquery.admin", + "roles/bigquery.jobUser", + "roles/bigquery.dataEditor", + "roles/bigquery.user", + "roles/dialogflow.client", + "roles/storage.admin", + "roles/aiplatform.user", + "roles/iam.serviceAccountUser" + ] + } +} + +module "service-account-vertex" { + source = "../../../modules/iam-service-account" + project_id = module.project.project_id + name = "vertex-sa" + iam_project_roles = { + (module.project.project_id) = [ + "roles/bigquery.admin", + "roles/bigquery.jobUser", + "roles/bigquery.dataEditor", + "roles/bigquery.user", + "roles/dialogflow.client", + "roles/storage.admin", + "roles/aiplatform.user" + ] + } +} + +resource "google_notebooks_instance" "playground" { + name = "${var.prefix}-notebook" + location = format("%s-%s", var.region, "b") + machine_type = "e2-medium" + project = module.project.project_id + + container_image { + repository = "gcr.io/deeplearning-platform-release/base-cpu" + tag = "latest" + } + + install_gpu_driver = true + boot_disk_type = "PD_SSD" + boot_disk_size_gb = 110 + disk_encryption = try(local.service_encryption_keys.compute != null, false) ? "CMEK" : null + kms_key = try(local.service_encryption_keys.compute, null) + + no_public_ip = true + no_proxy_access = false + + network = local.vpc + subnet = local.subnet + + service_account = module.service-account-notebook.email + + # Enable Secure Boot + shielded_instance_config { + enable_secure_boot = true + } + + # Remove once terraform-provider-google/issues/9164 is fixed + lifecycle { + ignore_changes = [disk_encryption, kms_key] + } + + #TODO Uncomment once terraform-provider-google/issues/9273 is fixed + # tags = ["ssh"] + depends_on = [ + google_project_iam_member.shared_vpc, + ] +} diff --git a/blueprints/data-solutions/bq-ml/vpc.tf b/blueprints/data-solutions/bq-ml/vpc.tf new file mode 100644 index 000000000..c581ed5c3 --- /dev/null +++ b/blueprints/data-solutions/bq-ml/vpc.tf @@ -0,0 +1,64 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# tfdoc:file:description VPC resources. + +module "vpc" { + source = "../../../modules/net-vpc" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.project.project_id + name = "${var.prefix}-vpc" + subnets = [ + { + ip_cidr_range = "10.0.0.0/20" + name = "${var.prefix}-subnet" + region = var.region + } + ] +} + +module "vpc-firewall" { + source = "../../../modules/net-vpc-firewall" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.project.project_id + network = module.vpc.0.name + default_rules_config = { + admin_ranges = ["10.0.0.0/20"] + } + ingress_rules = { + #TODO Remove and rely on 'ssh' tag once terraform-provider-google/issues/9273 is fixed + ("${var.prefix}-iap") = { + description = "Enable SSH from IAP on Notebooks." + source_ranges = ["35.235.240.0/20"] + targets = ["notebook-instance"] + rules = [{ protocol = "tcp", ports = [22] }] + } + } +} + +module "cloudnat" { + source = "../../../modules/net-cloudnat" + count = local.use_shared_vpc ? 0 : 1 + project_id = module.project.project_id + name = "${var.prefix}-default" + region = var.region + router_network = module.vpc.0.name +} + +resource "google_project_iam_member" "shared_vpc" { + count = local.use_shared_vpc ? 1 : 0 + project = var.vpc_config.host_project + role = "roles/compute.networkUser" + member = "serviceAccount:${module.project.service_accounts.robots.notebooks}" +} From dc034d74f730c6a5cfd54d7d9cdf62580c8574a2 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Sun, 5 Mar 2023 22:24:16 +0100 Subject: [PATCH 15/22] Variables. --- blueprints/data-solutions/bq-ml/README.md | 15 ++++++++------- blueprints/data-solutions/bq-ml/variables.tf | 20 ++++++++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/blueprints/data-solutions/bq-ml/README.md b/blueprints/data-solutions/bq-ml/README.md index 3efa457e5..39402b91c 100644 --- a/blueprints/data-solutions/bq-ml/README.md +++ b/blueprints/data-solutions/bq-ml/README.md @@ -45,7 +45,6 @@ To run the demo: - Connect to the Vertex AI workbench instance - Clone this repository - Run the and run [`demo/bmql_pipeline.ipynb`](demo/bmql_pipeline.ipynb) Jupyter Notebook. - ## Files @@ -59,17 +58,19 @@ To run the demo: | [vertex.tf](./vertex.tf) | Vertex resources. | iam-service-account | google_notebooks_instance · google_vertex_ai_metadata_store | | [vpc.tf](./vpc.tf) | VPC resources. | net-cloudnat · net-vpc · net-vpc-firewall | google_project_iam_member | + + ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [prefix](variables.tf#L33) | Prefix used for resource names. | string | ✓ | | -| [project_id](variables.tf#L51) | Project id references existing project if `project_create` is null. | string | ✓ | | +| [prefix](variables.tf#L23) | Prefix used for resource names. | string | ✓ | | +| [project_id](variables.tf#L41) | Project id references existing project if `project_create` is null. | string | ✓ | | | [location](variables.tf#L17) | The location where resources will be deployed. | string | | "US" | -| [project_create](variables.tf#L42) | Provide values if project creation is needed, use existing project if null. Parent format: folders/folder_id or organizations/org_id. | object({…}) | | null | -| [region](variables.tf#L56) | The region where resources will be deployed. | string | | "us-central1" | -| [service_encryption_keys](variables.tf#L62) | Cloud KMS to use to encrypt different services. The key location should match the service region. | object({…}) | | null | -| [vpc_config](variables.tf#L23) | Shared VPC network configurations to use. If null networks will be created in projects with pre-configured values. | object({…}) | | null | +| [project_create](variables.tf#L32) | Provide values if project creation is needed, use existing project if null. Parent format: folders/folder_id or organizations/org_id. | object({…}) | | null | +| [region](variables.tf#L46) | The region where resources will be deployed. | string | | "us-central1" | +| [service_encryption_keys](variables.tf#L52) | Cloud KMS to use to encrypt different services. The key location should match the service region. | object({…}) | | null | +| [vpc_config](variables.tf#L62) | Shared VPC network configurations to use. If null networks will be created in projects with pre-configured values. | object({…}) | | null | ## Outputs diff --git a/blueprints/data-solutions/bq-ml/variables.tf b/blueprints/data-solutions/bq-ml/variables.tf index 058284ee7..3106fb108 100644 --- a/blueprints/data-solutions/bq-ml/variables.tf +++ b/blueprints/data-solutions/bq-ml/variables.tf @@ -20,16 +20,6 @@ variable "location" { default = "US" } -variable "vpc_config" { - description = "Shared VPC network configurations to use. If null networks will be created in projects with pre-configured values." - type = object({ - host_project = string - network_self_link = string - subnet_self_link = string - }) - default = null -} - variable "prefix" { description = "Prefix used for resource names." type = string @@ -68,3 +58,13 @@ variable "service_encryption_keys" { }) default = null } + +variable "vpc_config" { + description = "Shared VPC network configurations to use. If null networks will be created in projects with pre-configured values." + type = object({ + host_project = string + network_self_link = string + subnet_self_link = string + }) + default = null +} From 16f703f336d2ff1ba555a8dc839e0fcf00f0af45 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Sun, 5 Mar 2023 22:30:33 +0100 Subject: [PATCH 16/22] Fix typos --- blueprints/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/README.md b/blueprints/README.md index 49b374eaa..ac9367083 100644 --- a/blueprints/README.md +++ b/blueprints/README.md @@ -6,7 +6,7 @@ Currently available blueprints: - **apigee** - [Apigee Hybrid on GKE](./apigee/hybrid-gke/), [Apigee X analytics in BigQuery](./apigee/bigquery-analytics), [Apigee network patterns](./apigee/network-patterns/) - **cloud operations** - [Active Directory Federation Services](./cloud-operations/adfs), [Cloud Asset Inventory feeds for resource change tracking and remediation](./cloud-operations/asset-inventory-feed-remediation), [Fine-grained Cloud DNS IAM via Service Directory](./cloud-operations/dns-fine-grained-iam), [Cloud DNS & Shared VPC design](./cloud-operations/dns-shared-vpc), [Delegated Role Grants](./cloud-operations/iam-delegated-role-grants), [Networking Dashboard](./cloud-operations/network-dashboard), [Managing on-prem service account keys by uploading public keys](./cloud-operations/onprem-sa-key-management), [Compute Image builder with Hashicorp Packer](./cloud-operations/packer-image-builder), [Packer example](./cloud-operations/packer-image-builder/packer), [Compute Engine quota monitoring](./cloud-operations/quota-monitoring), [Scheduled Cloud Asset Inventory Export to Bigquery](./cloud-operations/scheduled-asset-inventory-export-bq), [Configuring workload identity federation with Terraform Cloud/Enterprise workflows](./cloud-operations/terraform-cloud-dynamic-credentials), [TCP healthcheck and restart for unmanaged GCE instances](./cloud-operations/unmanaged-instances-healthcheck), [Migrate for Compute Engine (v5) blueprints](./cloud-operations/vm-migration), [Configuring workload identity federation to access Google Cloud resources from apps running on Azure](./cloud-operations/workload-identity-federation) -- **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder), [BigQuery ML and Vertex AI Pipeline](./data-solutions/dq'ml) +- **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder), [BigQuery ML and Vertex AI Pipeline](./data-solutions/dq-ml) - **factories** - [The why and the how of Resource Factories](./factories), [Google Cloud Identity Group Factory](./factories/cloud-identity-group-factory), [Google Cloud BQ Factory](./factories/bigquery-factory), [Google Cloud VPC Firewall Factory](./factories/net-vpc-firewall-yaml), [Minimal Project Factory](./factories/project-factory) - **GKE** - [Binary Authorization Pipeline Blueprint](./gke/binauthz), [Storage API](./gke/binauthz/image), [Multi-cluster mesh on GKE (fleet API)](./gke/multi-cluster-mesh-gke-fleet-api), [GKE Multitenant Blueprint](./gke/multitenant-fleet), [Shared VPC with GKE support](./networking/shared-vpc-gke/) - **networking** - [Calling a private Cloud Function from On-premises](./networking/private-cloud-function-from-onprem), [Decentralized firewall management](./networking/decentralized-firewall), [Decentralized firewall validator](./networking/decentralized-firewall/validator), [Network filtering with Squid](./networking/filtering-proxy), [GLB and multi-regional daisy-chaining through hybrid NEGs](./networking/glb-hybrid-neg-internal), [Hybrid connectivity to on-premise services through PSC](./networking/psc-hybrid), [HTTP Load Balancer with Cloud Armor](./networking/glb-and-armor), [Hub and Spoke via VPN](./networking/hub-and-spoke-vpn), [Hub and Spoke via VPC Peering](./networking/hub-and-spoke-peering), [Internal Load Balancer as Next Hop](./networking/ilb-next-hop), [Network filtering with Squid with isolated VPCs using Private Service Connect](./networking/filtering-proxy-psc), On-prem DNS and Google Private Access, [PSC Producer](./networking/psc-hybrid/psc-producer), [PSC Consumer](./networking/psc-hybrid/psc-consumer), [Shared VPC with optional GKE cluster](./networking/shared-vpc-gke) From f9acf61b810e6ee3faa7d2a57a0e79ac615f02a5 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Sun, 5 Mar 2023 22:42:27 +0100 Subject: [PATCH 17/22] Fix README --- blueprints/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/README.md b/blueprints/README.md index ac9367083..a9867ce7f 100644 --- a/blueprints/README.md +++ b/blueprints/README.md @@ -6,7 +6,7 @@ Currently available blueprints: - **apigee** - [Apigee Hybrid on GKE](./apigee/hybrid-gke/), [Apigee X analytics in BigQuery](./apigee/bigquery-analytics), [Apigee network patterns](./apigee/network-patterns/) - **cloud operations** - [Active Directory Federation Services](./cloud-operations/adfs), [Cloud Asset Inventory feeds for resource change tracking and remediation](./cloud-operations/asset-inventory-feed-remediation), [Fine-grained Cloud DNS IAM via Service Directory](./cloud-operations/dns-fine-grained-iam), [Cloud DNS & Shared VPC design](./cloud-operations/dns-shared-vpc), [Delegated Role Grants](./cloud-operations/iam-delegated-role-grants), [Networking Dashboard](./cloud-operations/network-dashboard), [Managing on-prem service account keys by uploading public keys](./cloud-operations/onprem-sa-key-management), [Compute Image builder with Hashicorp Packer](./cloud-operations/packer-image-builder), [Packer example](./cloud-operations/packer-image-builder/packer), [Compute Engine quota monitoring](./cloud-operations/quota-monitoring), [Scheduled Cloud Asset Inventory Export to Bigquery](./cloud-operations/scheduled-asset-inventory-export-bq), [Configuring workload identity federation with Terraform Cloud/Enterprise workflows](./cloud-operations/terraform-cloud-dynamic-credentials), [TCP healthcheck and restart for unmanaged GCE instances](./cloud-operations/unmanaged-instances-healthcheck), [Migrate for Compute Engine (v5) blueprints](./cloud-operations/vm-migration), [Configuring workload identity federation to access Google Cloud resources from apps running on Azure](./cloud-operations/workload-identity-federation) -- **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder), [BigQuery ML and Vertex AI Pipeline](./data-solutions/dq-ml) +- **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder), [BigQuery ML and Vertex AI Pipeline](./data-solutions/bq-ml) - **factories** - [The why and the how of Resource Factories](./factories), [Google Cloud Identity Group Factory](./factories/cloud-identity-group-factory), [Google Cloud BQ Factory](./factories/bigquery-factory), [Google Cloud VPC Firewall Factory](./factories/net-vpc-firewall-yaml), [Minimal Project Factory](./factories/project-factory) - **GKE** - [Binary Authorization Pipeline Blueprint](./gke/binauthz), [Storage API](./gke/binauthz/image), [Multi-cluster mesh on GKE (fleet API)](./gke/multi-cluster-mesh-gke-fleet-api), [GKE Multitenant Blueprint](./gke/multitenant-fleet), [Shared VPC with GKE support](./networking/shared-vpc-gke/) - **networking** - [Calling a private Cloud Function from On-premises](./networking/private-cloud-function-from-onprem), [Decentralized firewall management](./networking/decentralized-firewall), [Decentralized firewall validator](./networking/decentralized-firewall/validator), [Network filtering with Squid](./networking/filtering-proxy), [GLB and multi-regional daisy-chaining through hybrid NEGs](./networking/glb-hybrid-neg-internal), [Hybrid connectivity to on-premise services through PSC](./networking/psc-hybrid), [HTTP Load Balancer with Cloud Armor](./networking/glb-and-armor), [Hub and Spoke via VPN](./networking/hub-and-spoke-vpn), [Hub and Spoke via VPC Peering](./networking/hub-and-spoke-peering), [Internal Load Balancer as Next Hop](./networking/ilb-next-hop), [Network filtering with Squid with isolated VPCs using Private Service Connect](./networking/filtering-proxy-psc), On-prem DNS and Google Private Access, [PSC Producer](./networking/psc-hybrid/psc-producer), [PSC Consumer](./networking/psc-hybrid/psc-consumer), [Shared VPC with optional GKE cluster](./networking/shared-vpc-gke) From 0852ae3778cc07d4631861cd2cf0973e35fa3322 Mon Sep 17 00:00:00 2001 From: Giorgio Conte Date: Mon, 6 Mar 2023 10:16:21 +0000 Subject: [PATCH 18/22] demo: added batch prediction example --- .../bq-ml/demo/bmql_pipeline.ipynb | 22 +++++++++++++++++-- .../bq-ml/demo/sql/explain_predict.sql | 4 ++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb b/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb index 592f3d107..46e59314d 100644 --- a/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb +++ b/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb @@ -17,7 +17,8 @@ "source": [ "import kfp\n", "from google.cloud import aiplatform as aip\n", - "import google_cloud_pipeline_components.v1.bigquery as bqop" + "import google_cloud_pipeline_components.v1.bigquery as bqop\n", + "from google.cloud import bigquery" ] }, { @@ -255,6 +256,23 @@ "\n", "my_prediction" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# batch prediction on BigQuery\n", + "\n", + "with open(\"sql/explain_predict.sql\") as file:\n", + " train_query = file.read()\n", + "\n", + "client = bigquery_client = bigquery.Client(location=LOCATION, project=PROJECT_ID)\n", + "batch_predictions = bigquery_client.query(train_query.format(project_id=PROJECT_ID, dataset=DATASET, model_name=f'{MODEL_NAME}-fraction-10')).to_dataframe()\n", + "\n", + "batch_predictions" + ] } ], "metadata": { @@ -265,7 +283,7 @@ }, "language_info": { "name": "python", - "version": "3.8.9" + "version": "3.10.9" }, "orig_nbformat": 4, "vscode": { diff --git a/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql b/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql index 0c8c20635..4ef34fd93 100644 --- a/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql +++ b/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql @@ -15,9 +15,9 @@ */ SELECT * -FROM ML.EXPLAIN_PREDICT(MODEL `{project-id}.{dataset}.{model-name}`, +FROM ML.EXPLAIN_PREDICT(MODEL `{project_id}.{dataset}.{model-name}`, (SELECT * EXCEPT (session_id, session_starting_ts, user_id, has_purchased) - FROM `{project-id}.{dataset}.ecommerce_abt` + FROM `{project_id}.{dataset}.ecommerce_abt` WHERE extract(ISOYEAR FROM session_starting_ts) = 2023), STRUCT(5 AS top_k_features, 0.5 AS threshold)) LIMIT 100 From c82e7cca7b3615ca3fb2d0c848d0d0f59214aa17 Mon Sep 17 00:00:00 2001 From: Giorgio Conte Date: Mon, 6 Mar 2023 11:12:36 +0000 Subject: [PATCH 19/22] added demo README file --- .../data-solutions/bq-ml/demo/README.md | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 blueprints/data-solutions/bq-ml/demo/README.md diff --git a/blueprints/data-solutions/bq-ml/demo/README.md b/blueprints/data-solutions/bq-ml/demo/README.md new file mode 100644 index 000000000..2b5af642d --- /dev/null +++ b/blueprints/data-solutions/bq-ml/demo/README.md @@ -0,0 +1,38 @@ +# BigQuery ML and Vertex AI Pipeline Demo + +This demo shows how to combine BigQuery ML (BQML) and Vertex AI to create a ML pipeline leveraging the infrastructure created in the blueprint. + +More in details, this tutorial will focus on the following three steps: + +- define a Vertex AI pipeline to create features, train and evaluate BQML models +- serve a BQ model through an API powered by Vertex AI Endpoint +- create batch prediction via BigQuery + +# Dataset + +This tutorial uses a fictitious e-commerce dataset collecting programmatically generated data from the fictitious e-commerce store called The Look. The dataset is publicy available on BigQuery at this location `bigquery-public-data.thelook_ecommerce`. + +# Goal + +The goal of this tutorial is to train a classification ML model using BigQuery ML and predict if a new web session is going to convert. + +The tutorial focuses more on how to combine Vertex AI and BigQuery ML to create a model that can be used both for near-real time and batch predictions rather than the design of the model itself. + +# Main components + +In this tutorial we will make use of the following main components: +- Big Query: + - standard: to create a view which contains the model features and the target variable + - ML: to train, evaluate and make batch predictions +- Vertex AI: + - Pipeline: to define a configurable and re-usable set of steps to train and evaluate a BQML model + - Experiment: to keep track of all the trainings done via the Pipeline + - Model Registry: to keep track of the trained versions of a specific model + - Endpoint: to serve the model via API + - Workbench: to run this demo + +# How to get started + +1. Access the Vertex AI Workbench +2. clone this repository +2. run the [`bmql_pipeline.ipynb`](bmql_pipeline.ipynb) Jupyter Notebook \ No newline at end of file From 0ac6dd65cf30666dd3044e1a396a24fff1196826 Mon Sep 17 00:00:00 2001 From: Giorgio Conte Date: Mon, 6 Mar 2023 11:21:30 +0000 Subject: [PATCH 20/22] sql fix and more comments on demo notebook --- .../data-solutions/bq-ml/demo/README.md | 2 + .../bq-ml/demo/bmql_pipeline.ipynb | 62 +++++++++++++++++-- .../bq-ml/demo/sql/explain_predict.sql | 2 +- 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/blueprints/data-solutions/bq-ml/demo/README.md b/blueprints/data-solutions/bq-ml/demo/README.md index 2b5af642d..8ab748d8b 100644 --- a/blueprints/data-solutions/bq-ml/demo/README.md +++ b/blueprints/data-solutions/bq-ml/demo/README.md @@ -8,6 +8,8 @@ More in details, this tutorial will focus on the following three steps: - serve a BQ model through an API powered by Vertex AI Endpoint - create batch prediction via BigQuery +In this tutorial we will also see how to make explainable predictions, in order to understand what are the most important features that most influence the algorithm outputs. + # Dataset This tutorial uses a fictitious e-commerce dataset collecting programmatically generated data from the fictitious e-commerce store called The Look. The dataset is publicy available on BigQuery at this location `bigquery-public-data.thelook_ecommerce`. diff --git a/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb b/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb index 46e59314d..443428826 100644 --- a/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb +++ b/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb @@ -1,5 +1,40 @@ { "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Copyright 2023 Google LLC**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Install python requirements and import packages" + ] + }, { "cell_type": "code", "execution_count": null, @@ -26,7 +61,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Set your env variable" + "# Set your env variables" ] }, { @@ -159,7 +194,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Running the same training Verte AI pipeline with different parameters\n", + "# Running the same training Vertex AI pipeline with different parameters\n", "\n", "One of the main tasks during the training phase is to compare different models or to try the same model with different inputs. We can leverage the power of Vertex AI Pipelines to submit the same steps with different training parameters. Thanks to the experiments artifact, it is possible to easily keep track of all the tests that have been done. This simplifies the process of selecting the best model to deploy.\n", "\n", @@ -266,13 +301,32 @@ "# batch prediction on BigQuery\n", "\n", "with open(\"sql/explain_predict.sql\") as file:\n", - " train_query = file.read()\n", + " explain_predict_query = file.read()\n", "\n", "client = bigquery_client = bigquery.Client(location=LOCATION, project=PROJECT_ID)\n", - "batch_predictions = bigquery_client.query(train_query.format(project_id=PROJECT_ID, dataset=DATASET, model_name=f'{MODEL_NAME}-fraction-10')).to_dataframe()\n", + "batch_predictions = bigquery_client.query(\n", + " explain_predict_query.format(\n", + " project_id=PROJECT_ID,\n", + " dataset=DATASET,\n", + " model_name=f'{MODEL_NAME}-fraction-10')\n", + " ).to_dataframe()\n", "\n", "batch_predictions" ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Conclusions\n", + "\n", + "Thanks to this tutorial we were able to:\n", + "- Define a re-usable Vertex AI pipeline to train and evaluate BQ ML models\n", + "- Use a Vertex AI Experiment to keep track of multiple trainings for the same model with different paramenters (in this case a different split for train/test data)\n", + "- Deploy the preferred model on a Vertex AI managed Endpoint in order to serve the model for real-time use cases via API\n", + "- Make batch prediction via Big Query and see what are the top 5 features which influenced the algorithm output" + ] } ], "metadata": { diff --git a/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql b/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql index 4ef34fd93..e09fbd94d 100644 --- a/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql +++ b/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql @@ -15,7 +15,7 @@ */ SELECT * -FROM ML.EXPLAIN_PREDICT(MODEL `{project_id}.{dataset}.{model-name}`, +FROM ML.EXPLAIN_PREDICT(MODEL `{project_id}.{dataset}.{model_name}`, (SELECT * EXCEPT (session_id, session_starting_ts, user_id, has_purchased) FROM `{project_id}.{dataset}.ecommerce_abt` WHERE extract(ISOYEAR FROM session_starting_ts) = 2023), From ee55127ede281a2cce164856d1af63b8c6283919 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 6 Mar 2023 12:51:46 +0100 Subject: [PATCH 21/22] Fix notebook --- .../bq-ml/demo/bmql_pipeline.ipynb | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb b/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb index 443428826..ff0639578 100644 --- a/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb +++ b/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb @@ -14,6 +14,8 @@ "metadata": {}, "outputs": [], "source": [ + "# Copyright 2023 Google LLC\n", + "#\n", "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", @@ -51,8 +53,9 @@ "outputs": [], "source": [ "import kfp\n", - "from google.cloud import aiplatform as aip\n", "import google_cloud_pipeline_components.v1.bigquery as bqop\n", + "\n", + "from google.cloud import aiplatform as aip\n", "from google.cloud import bigquery" ] }, @@ -70,18 +73,17 @@ "metadata": {}, "outputs": [], "source": [ - "PREFIX = 'your-prefix'\n", - "PROJECT_ID = 'your-project-id'\n", - "LOCATION = 'US'\n", - "REGION = 'us-central1'\n", - "PIPELINE_NAME = 'bqml-vertex-pipeline'\n", - "MODEL_NAME = 'bqml-model'\n", "EXPERIMENT_NAME = 'bqml-experiment'\n", "ENDPOINT_DISPLAY_NAME = 'bqml-endpoint'\n", - "\n", - "SERVICE_ACCOUNT = f\"vertex-sa@{PROJECT_ID}.iam.gserviceaccount.com\"\n", + "DATASET = \"{}_data\".format(PREFIX.replace(\"-\",\"_\")) \n", + "LOCATION = 'US'\n", + "MODEL_NAME = 'bqml-model'\n", + "PIPELINE_NAME = 'bqml-vertex-pipeline'\n", "PIPELINE_ROOT = f\"gs://{PREFIX}-data\"\n", - "DATASET = \"{}_data\".format(PREFIX.replace(\"-\",\"_\")) " + "PREFIX = 'your-prefix'\n", + "PROJECT_ID = 'your-project-id'\n", + "REGION = 'us-central1'\n", + "SERVICE_ACCOUNT = f\"vertex-sa@{PROJECT_ID}.iam.gserviceaccount.com\"" ] }, { @@ -97,12 +99,12 @@ "3. Evaluate the BigQuery ML model with the standard evaluation metrics\n", "\n", "The pipeline takes as input the following variables:\n", - "- ```model_name```: the display name of the BigQuery ML model\n", - "- ```split_fraction```: the percentage of data that will be used as an evaluation dataset\n", - "- ```evaluate_job_conf```: bq dict configuration to define where to store evaluation metrics\n", "- ```dataset```: name of the dataset where the artifacts will be stored\n", + "- ```evaluate_job_conf```: bq dict configuration to define where to store evaluation metrics\n", + "- ```location```: BigQuery location\n", + "- ```model_name```: the display name of the BigQuery ML model\n", "- ```project_id```: the project id where the GCP resources will be created\n", - "- ```location```: BigQuery location" + "- ```split_fraction```: the percentage of data that will be used as an evaluation dataset" ] }, { @@ -186,7 +188,7 @@ " description='This is a new experiment to keep track of bqml trainings',\n", " project=PROJECT_ID,\n", " location=REGION\n", - " )" + ")" ] }, { @@ -218,7 +220,6 @@ " template_path=f'{PIPELINE_NAME}.json',\n", " pipeline_root=PIPELINE_ROOT,\n", " enable_caching=True\n", - " \n", " )\n", "\n", " pipeline.submit(service_account=SERVICE_ACCOUNT, experiment=my_experiment)" @@ -278,7 +279,8 @@ " 'day_of_week': 'WEEKDAY',\n", " 'traffic_source': 'Facebook',\n", " 'browser': 'Firefox',\n", - " 'hour_of_day': 20}" + " 'hour_of_day': 20\n", + "}" ] }, { @@ -337,7 +339,7 @@ }, "language_info": { "name": "python", - "version": "3.10.9" + "version": "3.8.9" }, "orig_nbformat": 4, "vscode": { From 3123d67ddfb7f0dfe02d6363ebd847cebbeceb2d Mon Sep 17 00:00:00 2001 From: Giorgio Conte Date: Mon, 6 Mar 2023 12:28:58 +0000 Subject: [PATCH 22/22] moved sql to notebook --- .../bq-ml/demo/bmql_pipeline.ipynb | 120 +++++++++++++++++- .../bq-ml/demo/sql/explain_predict.sql | 23 ---- .../bq-ml/demo/sql/features.sql | 68 ---------- .../data-solutions/bq-ml/demo/sql/train.sql | 27 ---- 4 files changed, 113 insertions(+), 125 deletions(-) delete mode 100644 blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql delete mode 100644 blueprints/data-solutions/bq-ml/demo/sql/features.sql delete mode 100644 blueprints/data-solutions/bq-ml/demo/sql/train.sql diff --git a/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb b/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb index ff0639578..4d3f5b533 100644 --- a/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb +++ b/blueprints/data-solutions/bq-ml/demo/bmql_pipeline.ipynb @@ -93,6 +93,100 @@ "source": [ "# Vertex AI Pipeline Definition\n", "\n", + "Let's first define the queries for the features and target creation and the query to train the model\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# this query creates the features for our model and the target value we would like to predict\n", + "\n", + "features_query = \"\"\"\n", + "CREATE VIEW if NOT EXISTS `{project_id}.{dataset}.ecommerce_abt` AS\n", + "WITH abt AS (\n", + " SELECT user_id,\n", + " session_id,\n", + " city,\n", + " postal_code,\n", + " browser,\n", + " traffic_source,\n", + " min(created_at) AS session_starting_ts,\n", + " sum(CASE WHEN event_type = 'purchase' THEN 1 ELSE 0 END) has_purchased\n", + " FROM `bigquery-public-data.thelook_ecommerce.events` \n", + " GROUP BY user_id,\n", + " session_id,\n", + " city,\n", + " postal_code,\n", + " browser,\n", + " traffic_source\n", + "), previous_orders AS (\n", + " SELECT user_id,\n", + " array_agg (struct(created_at AS order_creations_ts,\n", + " o.order_id,\n", + " o.status,\n", + " oi.order_cost)) as user_orders\n", + " FROM `bigquery-public-data.thelook_ecommerce.orders` o\n", + " JOIN (SELECT order_id,\n", + " sum(sale_price) order_cost \n", + " FROM `bigquery-public-data.thelook_ecommerce.order_items`\n", + " GROUP BY 1) oi\n", + " ON o.order_id = oi.order_id\n", + " GROUP BY 1\n", + ")\n", + "SELECT abt.*,\n", + " CASE WHEN extract(DAYOFWEEK FROM session_starting_ts) IN (1,7)\n", + " THEN 'WEEKEND' \n", + " ELSE 'WEEKDAY'\n", + " END AS day_of_week,\n", + " extract(HOUR FROM session_starting_ts) hour_of_day,\n", + " (SELECT count(DISTINCT uo.order_id) \n", + " FROM unnest(user_orders) uo \n", + " WHERE uo.order_creations_ts < session_starting_ts \n", + " AND status IN ('Shipped', 'Complete', 'Processing')) AS number_of_successful_orders,\n", + " IFNULL((SELECT sum(DISTINCT uo.order_cost) \n", + " FROM unnest(user_orders) uo \n", + " WHERE uo.order_creations_ts < session_starting_ts \n", + " AND status IN ('Shipped', 'Complete', 'Processing')), 0) AS sum_previous_orders,\n", + " (SELECT count(DISTINCT uo.order_id) \n", + " FROM unnest(user_orders) uo \n", + " WHERE uo.order_creations_ts < session_starting_ts \n", + " AND status IN ('Cancelled', 'Returned')) AS number_of_unsuccessful_orders\n", + "FROM abt \n", + "LEFT JOIN previous_orders pso \n", + "ON abt.user_id = pso.user_id\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# this query create the train job on BQ ML\n", + "train_query = \"\"\"\n", + "CREATE OR REPLACE MODEL `{project_id}.{dataset}.{model_name}`\n", + "OPTIONS(MODEL_TYPE='{model_type}',\n", + " INPUT_LABEL_COLS=['has_purchased'],\n", + " ENABLE_GLOBAL_EXPLAIN=TRUE,\n", + " MODEL_REGISTRY='VERTEX_AI',\n", + " DATA_SPLIT_METHOD = 'RANDOM',\n", + " DATA_SPLIT_EVAL_FRACTION = {split_fraction}\n", + " ) AS \n", + "SELECT * EXCEPT (session_id, session_starting_ts, user_id) \n", + "FROM `{project_id}.{dataset}.ecommerce_abt`\n", + "WHERE extract(ISOYEAR FROM session_starting_ts) = 2022\n", + "\"\"\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ "In the following code block, we are defining our Vertex AI pipeline. It is made up of three main steps:\n", "1. Create a BigQuery dataset that will contain the BigQuery ML models\n", "2. Train the BigQuery ML model, in this case, a logistic regression\n", @@ -113,13 +207,6 @@ "metadata": {}, "outputs": [], "source": [ - "with open(\"sql/train.sql\") as file:\n", - " train_query = file.read()\n", - "\n", - "with open(\"sql/features.sql\") as file:\n", - " features_query = file.read()\n", - "\n", - "\n", "@kfp.dsl.pipeline(name='bqml-pipeline', pipeline_root=PIPELINE_ROOT)\n", "def pipeline(\n", " model_name: str,\n", @@ -294,6 +381,25 @@ "my_prediction" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# batch prediction on BigQuery\n", + "\n", + "explain_predict_query = \"\"\"\n", + "SELECT *\n", + "FROM ML.EXPLAIN_PREDICT(MODEL `{project_id}.{dataset}.{model_name}`,\n", + " (SELECT * EXCEPT (session_id, session_starting_ts, user_id, has_purchased) \n", + " FROM `{project_id}.{dataset}.ecommerce_abt`\n", + " WHERE extract(ISOYEAR FROM session_starting_ts) = 2023),\n", + " STRUCT(5 AS top_k_features, 0.5 AS threshold))\n", + "LIMIT 100\n", + "\"\"\"" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql b/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql deleted file mode 100644 index e09fbd94d..000000000 --- a/blueprints/data-solutions/bq-ml/demo/sql/explain_predict.sql +++ /dev/null @@ -1,23 +0,0 @@ -/* -* Copyright 2023 Google LLC -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* https://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -SELECT * -FROM ML.EXPLAIN_PREDICT(MODEL `{project_id}.{dataset}.{model_name}`, - (SELECT * EXCEPT (session_id, session_starting_ts, user_id, has_purchased) - FROM `{project_id}.{dataset}.ecommerce_abt` - WHERE extract(ISOYEAR FROM session_starting_ts) = 2023), - STRUCT(5 AS top_k_features, 0.5 AS threshold)) -LIMIT 100 diff --git a/blueprints/data-solutions/bq-ml/demo/sql/features.sql b/blueprints/data-solutions/bq-ml/demo/sql/features.sql deleted file mode 100644 index a28ba85bc..000000000 --- a/blueprints/data-solutions/bq-ml/demo/sql/features.sql +++ /dev/null @@ -1,68 +0,0 @@ -/* -* Copyright 2023 Google LLC -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* https://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -CREATE VIEW if NOT EXISTS `{project_id}.{dataset}.ecommerce_abt` AS -WITH abt AS ( - SELECT user_id, - session_id, - city, - postal_code, - browser, - traffic_source, - min(created_at) AS session_starting_ts, - sum(CASE WHEN event_type = 'purchase' THEN 1 ELSE 0 END) has_purchased - FROM `bigquery-public-data.thelook_ecommerce.events` - GROUP BY user_id, - session_id, - city, - postal_code, - browser, - traffic_source -), previous_orders AS ( - SELECT user_id, - array_agg (struct(created_at AS order_creations_ts, - o.order_id, - o.status, - oi.order_cost)) as user_orders - FROM `bigquery-public-data.thelook_ecommerce.orders` o - JOIN (SELECT order_id, - sum(sale_price) order_cost - FROM `bigquery-public-data.thelook_ecommerce.order_items` - GROUP BY 1) oi - ON o.order_id = oi.order_id - GROUP BY 1 -) -SELECT abt.*, - CASE WHEN extract(DAYOFWEEK FROM session_starting_ts) IN (1,7) - THEN 'WEEKEND' - ELSE 'WEEKDAY' - END AS day_of_week, - extract(HOUR FROM session_starting_ts) hour_of_day, - (SELECT count(DISTINCT uo.order_id) - FROM unnest(user_orders) uo - WHERE uo.order_creations_ts < session_starting_ts - AND status IN ('Shipped', 'Complete', 'Processing')) AS number_of_successful_orders, - IFNULL((SELECT sum(DISTINCT uo.order_cost) - FROM unnest(user_orders) uo - WHERE uo.order_creations_ts < session_starting_ts - AND status IN ('Shipped', 'Complete', 'Processing')), 0) AS sum_previous_orders, - (SELECT count(DISTINCT uo.order_id) - FROM unnest(user_orders) uo - WHERE uo.order_creations_ts < session_starting_ts - AND status IN ('Cancelled', 'Returned')) AS number_of_unsuccessful_orders -FROM abt -LEFT JOIN previous_orders pso -ON abt.user_id = pso.user_id diff --git a/blueprints/data-solutions/bq-ml/demo/sql/train.sql b/blueprints/data-solutions/bq-ml/demo/sql/train.sql deleted file mode 100644 index 2c30f2e67..000000000 --- a/blueprints/data-solutions/bq-ml/demo/sql/train.sql +++ /dev/null @@ -1,27 +0,0 @@ -/* -* Copyright 2023 Google LLC -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* https://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -CREATE OR REPLACE MODEL `{project_id}.{dataset}.{model_name}` -OPTIONS(MODEL_TYPE='{model_type}', - INPUT_LABEL_COLS=['has_purchased'], - ENABLE_GLOBAL_EXPLAIN=TRUE, - MODEL_REGISTRY='VERTEX_AI', - DATA_SPLIT_METHOD = 'RANDOM', - DATA_SPLIT_EVAL_FRACTION = {split_fraction} - ) AS -SELECT * EXCEPT (session_id, session_starting_ts, user_id) -FROM `{project_id}.{dataset}.ecommerce_abt_table` -WHERE extract(ISOYEAR FROM session_starting_ts) = 2022 \ No newline at end of file