diff --git a/.gitignore b/.gitignore index 718277092..5b90b4eb8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,16 @@ venv/* */venv/* -**/.python-version -**/.terraform -**/terraform.tfstate* -**/terraform.tfvars -**/*.auto.tfvars -**/.terraform.tfstate.lock.info -**/.terraform.lock.hcl +.python-version +.terraform +terraform.tfstate* +terraform.tfvars +*.auto.tfvars +.terraform.tfstate.lock.info +.terraform.lock.hcl !tests/**/terraform.tfvars -**/__pycache__ -**/.pytest_cache -**/.test.lock +__pycache__ +.pytest_cache +.test.lock .idea .idx/dev.nix backend.tf @@ -20,8 +20,6 @@ key.json terraform-ls.tf bundle.zip .DS_Store -**/packer_cache -**/*.pkrvars.hcl fixture_* fast/configs fast/**/[0-9]*providers.tf @@ -29,35 +27,8 @@ fast/**/terraform.tfvars fast/**/terraform.tfvars.json fast/**/terraform-*.auto.tfvars.json fast/**/[0-9]*.auto.tfvars* -**/node_modules +node_modules fast/**/globals.auto.tfvars.json cloud_sql_proxy -examples/cloud-operations/binauthz/tenant-setup.yaml -examples/cloud-operations/binauthz/app/app.yaml env/ -examples/cloud-operations/adfs/ansible/vars/vars.yaml -examples/cloud-operations/adfs/ansible/gssh.sh -examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/vars.yaml -examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/gssh.sh -blueprints/cloud-operations/network-quota-monitoring/cloud-function.zip -blueprints/apigee/bigquery-analytics/bundle-export.zip -blueprints/apigee/bigquery-analytics/bundle-gcs2bq.zip -blueprints/apigee/bigquery-analytics/apiproxy.zip -blueprints/apigee/bigquery-analytics/create-datastore.sh -blueprints/apigee/bigquery-analytics/deploy-apiproxy.sh -blueprints/apigee/network-patterns/nb-glb-psc-neg-sb-psc-ilbl7-hybrid-neg/bundle/apiproxy/targets/default.xml -blueprints/apigee/network-patterns/nb-glb-psc-neg-sb-psc-ilbl7-hybrid-neg/bundle.zip -blueprints/apigee/network-patterns/nb-glb-psc-neg-sb-psc-ilbl7-hybrid-neg/deploy-apiproxy.sh -blueprints/apigee/hybrid-gke/apiproxy.zip -blueprints/apigee/hybrid-gke/deploy-apiproxy.sh -blueprints/apigee/hybrid-gke/ansible/gssh.sh -blueprints/apigee/hybrid-gke/ansible/vars/vars.yaml -blueprints/gke/autopilot/ansible/gssh.sh -blueprints/gke/autopilot/ansible/vars/vars.yaml -blueprints/gke/autopilot/bundle/monitoring/kustomization.yaml -blueprints/gke/autopilot/bundle/locust/kustomization.yaml -blueprints/gke/autopilot/bundle.tar.gz -blueprints/gke/patterns/batch/job-*.yaml -modules/apigee/recipe-apigee-swp/bundle.zip -modules/apigee/recipe-apigee-swp/deploy-apiproxy.sh diff --git a/README.md b/README.md index add1dee1a..510881094 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Currently available modules: - **process factories** - [project factory](./modules/project-factory/README.md) - **networking** - [DNS](./modules/dns), [DNS Response Policy](./modules/dns-response-policy/), [Cloud Endpoints](./modules/endpoints), [address reservation](./modules/net-address), [NAT](./modules/net-cloudnat), [VLAN Attachment](./modules/net-vlan-attachment/), [External Application LB](./modules/net-lb-app-ext/), [External Passthrough Network LB](./modules/net-lb-ext), [External Regional Application Load Balancer](./modules/net-lb-app-ext-regional/), [Firewall policy](./modules/net-firewall-policy), [Internal Application LB](./modules/net-lb-app-int), [Cross-region Internal Application LB](./modules/net-lb-app-int-cross-region), [Internal Passthrough Network LB](./modules/net-lb-int), [Internal Proxy Network LB](./modules/net-lb-proxy-int), [IPSec over Interconnect](./modules/net-ipsec-over-interconnect), [VPC](./modules/net-vpc), [VPC factory](./modules/net-vpc-factory/README.md), [VPC firewall](./modules/net-vpc-firewall), [VPC peering](./modules/net-vpc-peering), [VPN dynamic](./modules/net-vpn-dynamic), [HA VPN](./modules/net-vpn-ha), [VPN static](./modules/net-vpn-static), [Service Directory](./modules/service-directory), [Secure Web Proxy](./modules/net-swp) - **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [COS container](./modules/cloud-config-container/cos-generic-metadata/) (coredns, mysql, onprem, squid), [GKE cluster](./modules/gke-cluster-standard), [GKE hub](./modules/gke-hub), [GKE nodepool](./modules/gke-nodepool), [GCVE private cloud](./modules/gcve-private-cloud) -- **data** - [AlloyDB instance](./modules/alloydb), [Analytics Hub](./modules/analytics-hub), [BigQuery dataset](./modules/bigquery-dataset), [Biglake Catalog](./modules/biglake-catalog), [Bigtable instance](./modules/bigtable-instance), [Dataplex](./modules/dataplex), [Dataplex Aspect Types](./modules/dataplex-aspect-types/), [Dataplex DataScan](./modules/dataplex-datascan), [Cloud SQL instance](./modules/cloudsql-instance), [Spanner instance](./modules/spanner-instance), [Firestore](./modules/firestore), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag), [Data Catalog Tag](./modules/data-catalog-tag), [Data Catalog Tag Template](./modules/data-catalog-tag-template), [Datafusion](./modules/datafusion), [Dataproc](./modules/dataproc), [GCS](./modules/gcs), [Pub/Sub](./modules/pubsub), [Dataform Repository](./modules/dataform-repository/), [Looker Core](./modules/looker-core) +- **data** - [AlloyDB instance](./modules/alloydb), [Analytics Hub](./modules/analytics-hub), [BigQuery connection](./modules/bigquery-connection), [BigQuery dataset](./modules/bigquery-dataset), [Biglake Catalog](./modules/biglake-catalog), [Bigtable instance](./modules/bigtable-instance), [Dataplex](./modules/dataplex), [Dataplex Aspect Types](./modules/dataplex-aspect-types/), [Dataplex DataScan](./modules/dataplex-datascan), [Cloud SQL instance](./modules/cloudsql-instance), [Spanner instance](./modules/spanner-instance), [Firestore](./modules/firestore), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag), [Data Catalog Tag](./modules/data-catalog-tag), [Data Catalog Tag Template](./modules/data-catalog-tag-template), [Datafusion](./modules/datafusion), [Dataproc](./modules/dataproc), [GCS](./modules/gcs), [Pub/Sub](./modules/pubsub), [Dataform Repository](./modules/dataform-repository/), [Looker Core](./modules/looker-core) - **AI** - [AI Applications](./modules/ai-applications/README.md) - **development** - [API Gateway](./modules/api-gateway), [Apigee](./modules/apigee), [Artifact Registry](./modules/artifact-registry), [Cloud Build V2 Connection](./modules/cloud-build-v2-connection), [Container Registry](./modules/container-registry), [Cloud Source Repository](./modules/source-repository), [Cloud Deploy](./modules/cloud-deploy), [Secure Source Manager instance](./modules/secure-source-manager-instance), [Workstation cluster](./modules/workstation-cluster) - **security** - [Binauthz](./modules/binauthz/), [Certificate Authority Service (CAS)](./modules/certificate-authority-service), [KMS](./modules/kms), [SecretManager](./modules/secret-manager), [VPC Service Control](./modules/vpc-sc), [Certificate Manager](./modules/certificate-manager/) diff --git a/modules/README.md b/modules/README.md index ab43a7315..8c10b48d7 100644 --- a/modules/README.md +++ b/modules/README.md @@ -82,6 +82,7 @@ These modules are used in the examples included in this repository. If you are u - [AlloyDB](./alloydb) - [Analytics Hub](./analytics-hub) +- [BigQuery connectio](./bigquery-connection) - [BigQuery dataset](./bigquery-dataset) - [Bigtable instance](./bigtable-instance) - [Biglake catalog](./biglake-catalog) diff --git a/modules/bigquery-connection/README.md b/modules/bigquery-connection/README.md new file mode 100644 index 000000000..43e8ebdd4 --- /dev/null +++ b/modules/bigquery-connection/README.md @@ -0,0 +1,152 @@ +# BigQuery Connection + +This module allows creating a BigQuery connection. + + +- [Cloud SQL Connection](#cloud-sql-connection) +- [Cloud SQL Connection with Cloud KMS](#cloud-sql-connection-with-cloud-kms) +- [Spanner Connection](#spanner-connection) +- [Spanner Connection with Context interpolations](#spanner-connection-with-context-interpolations) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Cloud SQL Connection + +```hcl +module "bigquery-connection" { + source = "./fabric/modules/bigquery-connection" + project_id = var.project_id + location = "EU" + connection_id = "my-connection" + friendly_name = "My Cloud SQL Connection" + description = "A connection to a Cloud SQL instance." + + connection_config = { + cloud_sql = { + instance_id = "my-instance-id" + database = "my-database" + type = "POSTGRES" + credential = { + username = "my-username" + password = "my-password" + } + } + } + iam = { + "roles/bigquery.connectionUser" = ["user:my-user@example.com"] + } +} +# tftest modules=1 resources=2 inventory=cloudsql.yaml +``` + +## Cloud SQL Connection with Cloud KMS + +```hcl +module "bigquery-connection" { + source = "./fabric/modules/bigquery-connection" + project_id = var.project_id + location = "EU" + connection_id = "my-connection" + friendly_name = "My BigQuery Connection" + description = "A connection to a Cloud SQL instance." + encryption_key = "my-key" + + connection_config = { + cloud_sql = { + instance_id = "my-instance-id" + database = "my-database" + type = "POSTGRES" + credential = { + username = "my-username" + password = "my-password" + } + } + } +} +# tftest modules=1 resources=1 inventory=cloudsql_kms.yaml +``` + +## Spanner Connection + +```hcl +module "bigquery-connection" { + source = "./fabric/modules/bigquery-connection" + project_id = var.project_id + location = "EU" + connection_id = "my-connection" + friendly_name = "My BigQuery Connection" + description = "A connection to a Spanner instance." + + connection_config = { + cloud_spanner = { + database = "projects/my-project/instances/my-instance/databases/my-database" + use_parallelism = true + use_data_boost = true + max_parallelism = 2 + } + } + iam = { + "roles/bigquery.connectionUser" = ["user:my-user@example.com"] + } +} +# tftest modules=1 resources=2 inventory=spanner.yaml +``` + +## Spanner Connection with Context interpolations + +```hcl +module "bigquery-connection" { + source = "./fabric/modules/bigquery-connection" + project_id = var.project_id + location = "EU" + connection_id = "my-connection" + friendly_name = "My BigQuery Connection" + description = "A connection to a Spanner instance." + + connection_config = { + cloud_spanner = { + database = "projects/my-project/instances/my-instance/databases/my-database" + use_parallelism = true + use_data_boost = true + max_parallelism = 2 + } + } + context = { + iam_principals = { + myuser = "user:my-user@example.com" + } + } + iam = { + "roles/bigquery.connectionUser" = ["$iam_principals:myuser"] + } +} +# tftest modules=1 resources=2 inventory=spanner_context.yaml +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [connection_id](variables.tf#L59) | The ID of the connection. | string | ✓ | | +| [location](variables.tf#L132) | The geographic location where the connection should reside. | string | ✓ | | +| [project_id](variables.tf#L137) | The ID of the project in which the resource belongs. | string | ✓ | | +| [connection_config](variables.tf#L17) | Connection properties. | object({…}) | | {} | +| [context](variables.tf#L64) | Context-specific interpolations. | object({…}) | | {} | +| [description](variables.tf#L73) | A description of the connection. | string | | null | +| [encryption_key](variables.tf#L79) | The name of the KMS key used for encryption. | string | | null | +| [friendly_name](variables.tf#L85) | A descriptive name for the connection. | string | | null | +| [iam](variables.tf#L91) | IAM bindings for the connection in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_bindings](variables.tf#L97) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L111) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_by_principals](variables.tf#L125) | Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid cycle errors. Merged internally with the `iam` variable. | map(list(string)) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [connection_config](outputs.tf#L17) | The connection configuration. | | +| [connection_id](outputs.tf#L29) | The ID of the BigQuery connection. | | +| [description](outputs.tf#L34) | The description of the connection. | | +| [location](outputs.tf#L39) | The location of the connection. | | + diff --git a/modules/bigquery-connection/iam.tf b/modules/bigquery-connection/iam.tf new file mode 100644 index 000000000..e92983641 --- /dev/null +++ b/modules/bigquery-connection/iam.tf @@ -0,0 +1,77 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _iam_principal_roles = distinct(flatten(values(var.iam_by_principals))) + _iam_principals = { + for r in local._iam_principal_roles : r => [ + for k, v in var.iam_by_principals : + k if try(index(v, r), null) != null + ] + } + iam = { + for role in distinct(concat(keys(var.iam), keys(local._iam_principals))) : + role => concat( + try(var.iam[role], []), + try([for v in local._iam_principals[role] : lookup(local.ctx.iam_principals, v, v)], []) + ) + } +} + +resource "google_bigquery_connection_iam_binding" "authoritative" { + for_each = local.iam + project = google_bigquery_connection.connection.project + location = google_bigquery_connection.connection.location + connection_id = google_bigquery_connection.connection.connection_id + role = each.key + members = [ + for v in each.value : lookup(local.ctx.iam_principals, v, v) + ] +} + +resource "google_bigquery_connection_iam_binding" "bindings" { + for_each = var.iam_bindings + project = google_bigquery_connection.connection.project + location = google_bigquery_connection.connection.location + connection_id = google_bigquery_connection.connection.connection_id + role = each.value.role + members = lookup(local.ctx.iam_principals, each.value.member, each.value.member) + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_bigquery_connection_iam_member" "bindings" { + for_each = var.iam_bindings_additive + project = google_bigquery_connection.connection.project + location = google_bigquery_connection.connection.location + connection_id = google_bigquery_connection.connection.connection_id + role = each.value.role + member = lookup(local.ctx.iam_principals, each.value.member, each.value.member) + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} diff --git a/modules/bigquery-connection/main.tf b/modules/bigquery-connection/main.tf new file mode 100644 index 000000000..74474c949 --- /dev/null +++ b/modules/bigquery-connection/main.tf @@ -0,0 +1,97 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + ctx = { + for k, v in var.context : k => { + for kk, vv in v : "${local.ctx_p}${k}:${kk}" => vv + } if k != "condition_vars" + } + ctx_p = "$" +} + +resource "google_bigquery_connection" "connection" { + project = var.project_id + location = var.location + connection_id = var.connection_id + friendly_name = var.friendly_name + description = var.description + kms_key_name = var.encryption_key + + dynamic "cloud_sql" { + for_each = var.connection_config.cloud_sql == null ? [] : [var.connection_config.cloud_sql] + content { + instance_id = cloud_sql.value.instance_id + database = cloud_sql.value.database + type = cloud_sql.value.type + credential { + username = cloud_sql.value.credential.username + password = cloud_sql.value.credential.password + } + } + } + + dynamic "aws" { + for_each = var.connection_config.aws == null ? [] : [var.connection_config.aws] + content { + access_role { + iam_role_id = aws.value.access_role.iam_role_id + identity = aws.value.access_role.identity + } + } + } + + dynamic "azure" { + for_each = var.connection_config.azure == null ? [] : [var.connection_config.azure] + content { + application = azure.value.application + client_id = azure.value.client_id + object_id = azure.value.object_id + customer_tenant_id = azure.value.customer_tenant_id + federated_application_client_id = azure.value.federated_application_client_id + redirect_uri = azure.value.redirect_uri + identity = azure.value.identity + } + } + + dynamic "cloud_spanner" { + for_each = var.connection_config.cloud_spanner == null ? [] : [var.connection_config.cloud_spanner] + content { + database = cloud_spanner.value.database + use_parallelism = cloud_spanner.value.use_parallelism + use_data_boost = cloud_spanner.value.use_data_boost + max_parallelism = cloud_spanner.value.max_parallelism + database_role = cloud_spanner.value.database_role + } + } + + dynamic "cloud_resource" { + for_each = var.connection_config.cloud_resource == null ? [] : [var.connection_config.cloud_resource] + content {} + } + + dynamic "spark" { + for_each = var.connection_config.spark == null ? [] : [var.connection_config.spark] + content { + metastore_service_config { + metastore_service = spark.value.metastore_service_config.metastore_service + } + spark_history_server_config { + dataproc_cluster = spark.value.spark_history_server_config.dataproc_cluster + } + } + } +} \ No newline at end of file diff --git a/modules/bigquery-connection/outputs.tf b/modules/bigquery-connection/outputs.tf new file mode 100644 index 000000000..6bac931ff --- /dev/null +++ b/modules/bigquery-connection/outputs.tf @@ -0,0 +1,42 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "connection_config" { + description = "The connection configuration." + value = { + aws = one(google_bigquery_connection.connection.aws) + azure = one(google_bigquery_connection.connection.azure) + cloud_resource = one(google_bigquery_connection.connection.cloud_resource) + cloud_spanner = one(google_bigquery_connection.connection.cloud_spanner) + cloud_sql = one(google_bigquery_connection.connection.cloud_sql) + spark = one(google_bigquery_connection.connection.spark) + } +} + +output "connection_id" { + description = "The ID of the BigQuery connection." + value = google_bigquery_connection.connection.connection_id +} + +output "description" { + description = "The description of the connection." + value = google_bigquery_connection.connection.description +} + +output "location" { + description = "The location of the connection." + value = google_bigquery_connection.connection.location +} diff --git a/modules/bigquery-connection/variables.tf b/modules/bigquery-connection/variables.tf new file mode 100644 index 000000000..948d88a74 --- /dev/null +++ b/modules/bigquery-connection/variables.tf @@ -0,0 +1,140 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "connection_config" { + description = "Connection properties." + type = object({ + cloud_sql = optional(object({ + instance_id = string + database = string + type = string + credential = object({ + username = string + password = string + }) + })) + aws = optional(object({ + access_role = object({ + iam_role_id = string + }) + })) + azure = optional(object({ + customer_tenant_id = string + federated_application_client_id = optional(string) + })) + cloud_spanner = optional(object({ + database = string + use_parallelism = optional(bool) + use_data_boost = optional(bool) + max_parallelism = optional(number) + database_role = optional(string) + })) + cloud_resource = optional(object({ + })) + spark = optional(object({ + metastore_service_config = optional(object({ + metastore_service = string + })) + spark_history_server_config = optional(object({ + dataproc_cluster = string + })) + })) + }) + default = {} +} + +variable "connection_id" { + description = "The ID of the connection." + type = string +} + +variable "context" { + description = "Context-specific interpolations." + type = object({ + iam_principals = optional(map(string), {}) + }) + default = {} + nullable = false +} + +variable "description" { + description = "A description of the connection." + type = string + default = null +} + +variable "encryption_key" { + description = "The name of the KMS key used for encryption." + type = string + default = null +} + +variable "friendly_name" { + description = "A descriptive name for the connection." + type = string + default = null +} + +variable "iam" { + description = "IAM bindings for the connection in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} +} + +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + default = {} +} + +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + default = {} +} + +variable "iam_by_principals" { + description = "Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid cycle errors. Merged internally with the `iam` variable." + type = map(list(string)) + default = {} + nullable = false +} + +variable "location" { + description = "The geographic location where the connection should reside." + type = string +} + +variable "project_id" { + description = "The ID of the project in which the resource belongs." + type = string +} diff --git a/modules/bigquery-connection/versions.tf b/modules/bigquery-connection/versions.tf new file mode 100644 index 000000000..263be60f9 --- /dev/null +++ b/modules/bigquery-connection/versions.tf @@ -0,0 +1,35 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fabric release: v45.0.0 + +terraform { + required_version = ">= 1.12.2" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 7.0.1, < 8.0.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 7.0.1, < 8.0.0" # tftest + } + } + provider_meta "google" { + module_name = "google-pso-tool/cloud-foundation-fabric/modules/bigquery-connection:v45.0.0-tf" + } + provider_meta "google-beta" { + module_name = "google-pso-tool/cloud-foundation-fabric/modules/bigquery-connection:v45.0.0-tf" + } +} diff --git a/modules/bigquery-connection/versions.tofu b/modules/bigquery-connection/versions.tofu new file mode 100644 index 000000000..2f3b38fbe --- /dev/null +++ b/modules/bigquery-connection/versions.tofu @@ -0,0 +1,35 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fabric release: v45.0.0 + +terraform { + required_version = ">= 1.10.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 7.0.1, < 8.0.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 7.0.1, < 8.0.0" # tftest + } + } + provider_meta "google" { + module_name = "google-pso-tool/cloud-foundation-fabric/modules/bigquery-connection:v45.0.0-tofu" + } + provider_meta "google-beta" { + module_name = "google-pso-tool/cloud-foundation-fabric/modules/bigquery-connection:v45.0.0-tofu" + } +} diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index 5e4cb780a..44f25427c 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -370,10 +370,10 @@ module "bigquery-dataset" { | [location](variables.tf#L122) | Dataset location. | string | | "EU" | | [materialized_views](variables.tf#L128) | Materialized views definitions. | map(object({…})) | | {} | | [options](variables.tf#L161) | Dataset options. | object({…}) | | {} | -| [routines](variables.tf#L180) | Routine definitions. | map(object({…})) | | {} | -| [tables](variables.tf#L218) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…})) | | {} | -| [tag_bindings](variables.tf#L303) | Tag bindings for this dataset, in key => tag value id format. | map(string) | | {} | -| [views](variables.tf#L310) | View definitions. | map(object({…})) | | {} | +| [routines](variables.tf#L180) | Routine definitions. | map(object({…})) | | {} | +| [tables](variables.tf#L219) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…})) | | {} | +| [tag_bindings](variables.tf#L304) | Tag bindings for this dataset, in key => tag value id format. | map(string) | | {} | +| [views](variables.tf#L311) | View definitions. | map(object({…})) | | {} | ## Outputs diff --git a/modules/bigquery-dataset/main.tf b/modules/bigquery-dataset/main.tf index 2ed2f9927..17b676848 100644 --- a/modules/bigquery-dataset/main.tf +++ b/modules/bigquery-dataset/main.tf @@ -373,7 +373,7 @@ resource "google_bigquery_table" "default" { } resource "google_bigquery_table" "views" { - depends_on = [google_bigquery_table.default] + depends_on = [google_bigquery_table.default, google_bigquery_routine.default] for_each = var.views project = local.project_id dataset_id = google_bigquery_dataset.default.dataset_id @@ -382,7 +382,7 @@ resource "google_bigquery_table" "views" { description = each.value.description labels = each.value.labels deletion_protection = each.value.deletion_protection - schema = try(jsonencode(each.value.schema), null) + schema = each.value.schema != null ? jsonencode(each.value.schema) : null view { query = each.value.query @@ -445,6 +445,7 @@ resource "google_bigquery_routine" "default" { imported_libraries = each.value.imported_libraries determinism_level = each.value.determinism_level data_governance_type = each.value.data_governance_type + return_type = each.value.return_type return_table_type = each.value.return_table_type dynamic "arguments" { for_each = each.value.arguments diff --git a/modules/bigquery-dataset/variables.tf b/modules/bigquery-dataset/variables.tf index ff6394cb0..9e797b3f4 100644 --- a/modules/bigquery-dataset/variables.tf +++ b/modules/bigquery-dataset/variables.tf @@ -187,6 +187,7 @@ variable "routines" { imported_libraries = optional(list(string)) determinism_level = optional(string) data_governance_type = optional(string) + return_type = optional(string) return_table_type = optional(string) arguments = optional(map(object({ argument_kind = optional(string) diff --git a/modules/net-vpc/main.tf b/modules/net-vpc/main.tf index 446505cb5..5248c941a 100644 --- a/modules/net-vpc/main.tf +++ b/modules/net-vpc/main.tf @@ -37,14 +37,14 @@ locals { ? { id = format( "projects/%s/global/networks/%s", - var.project_id, + local.project_id, var.name ) name = var.name network_id = try(var.vpc_reuse.attributes.network_id, null) self_link = format( "https://www.googleapis.com/compute/v1/projects/%s/global/networks/%s", - var.project_id, + local.project_id, var.name ) } diff --git a/modules/net-vpc/outputs.tf b/modules/net-vpc/outputs.tf index 29213238a..bcb4c27da 100644 --- a/modules/net-vpc/outputs.tf +++ b/modules/net-vpc/outputs.tf @@ -94,7 +94,7 @@ output "network_id" { output "project_id" { description = "Project ID containing the network. Use this when you need to create resources *after* the VPC is fully set up (e.g. subnets created, shared VPC service projects attached, Private Service Networking configured)." - value = var.project_id + value = local.project_id depends_on = [ google_compute_subnetwork.subnetwork, google_compute_network_peering.local, diff --git a/modules/net-vpc/psa.tf b/modules/net-vpc/psa.tf index 8b9855896..1fb234f50 100644 --- a/modules/net-vpc/psa.tf +++ b/modules/net-vpc/psa.tf @@ -74,7 +74,7 @@ resource "google_service_networking_connection" "psa_connection" { resource "google_compute_network_peering_routes_config" "psa_routes" { for_each = local.psa_configs - project = var.project_id + project = local.project_id peering = ( google_service_networking_connection.psa_connection[each.key].peering ) @@ -85,7 +85,7 @@ resource "google_compute_network_peering_routes_config" "psa_routes" { resource "google_service_networking_peered_dns_domain" "name" { for_each = local.psa_peered_domains - project = var.project_id + project = local.project_id network = local.network.name name = each.key dns_suffix = each.value.dns_suffix diff --git a/tests/modules/bigquery_connection/examples/cloudsql.yaml b/tests/modules/bigquery_connection/examples/cloudsql.yaml new file mode 100644 index 000000000..ee5ce51f6 --- /dev/null +++ b/tests/modules/bigquery_connection/examples/cloudsql.yaml @@ -0,0 +1,47 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.bigquery-connection.google_bigquery_connection.connection: + aws: [] + azure: [] + cloud_resource: [] + cloud_spanner: [] + cloud_sql: + - credential: + - password: my-password + username: my-username + database: my-database + instance_id: my-instance-id + type: POSTGRES + connection_id: my-connection + description: A connection to a Cloud SQL instance. + friendly_name: My Cloud SQL Connection + kms_key_name: null + location: EU + project: project-id + spark: [] + timeouts: null + module.bigquery-connection.google_bigquery_connection_iam_binding.authoritative["roles/bigquery.connectionUser"]: + condition: [] + connection_id: my-connection + location: EU + members: + - user:my-user@example.com + project: project-id + role: roles/bigquery.connectionUser + +counts: + google_bigquery_connection: 1 + google_bigquery_connection_iam_binding: 1 \ No newline at end of file diff --git a/tests/modules/bigquery_connection/examples/cloudsql_kms.yaml b/tests/modules/bigquery_connection/examples/cloudsql_kms.yaml new file mode 100644 index 000000000..9968e4411 --- /dev/null +++ b/tests/modules/bigquery_connection/examples/cloudsql_kms.yaml @@ -0,0 +1,38 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.bigquery-connection.google_bigquery_connection.connection: + aws: [] + azure: [] + cloud_resource: [] + cloud_spanner: [] + cloud_sql: + - credential: + - password: my-password + username: my-username + database: my-database + instance_id: my-instance-id + type: POSTGRES + connection_id: my-connection + description: A connection to a Cloud SQL instance. + friendly_name: My BigQuery Connection + kms_key_name: my-key + location: EU + project: project-id + spark: [] + timeouts: null + +counts: + google_bigquery_connection: 1 diff --git a/tests/modules/bigquery_connection/examples/spanner.yaml b/tests/modules/bigquery_connection/examples/spanner.yaml new file mode 100644 index 000000000..0ebccff56 --- /dev/null +++ b/tests/modules/bigquery_connection/examples/spanner.yaml @@ -0,0 +1,47 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.bigquery-connection.google_bigquery_connection.connection: + aws: [] + azure: [] + cloud_resource: [] + cloud_spanner: + - database: projects/my-project/instances/my-instance/databases/my-database + database_role: null + max_parallelism: 2 + use_data_boost: true + use_parallelism: true + use_serverless_analytics: null + cloud_sql: [] + connection_id: my-connection + description: A connection to a Spanner instance. + friendly_name: My BigQuery Connection + kms_key_name: null + location: EU + project: project-id + spark: [] + timeouts: null + module.bigquery-connection.google_bigquery_connection_iam_binding.authoritative["roles/bigquery.connectionUser"]: + condition: [] + connection_id: my-connection + location: EU + members: + - user:my-user@example.com + project: project-id + role: roles/bigquery.connectionUser + +counts: + google_bigquery_connection: 1 + google_bigquery_connection_iam_binding: 1 \ No newline at end of file diff --git a/tests/modules/bigquery_connection/examples/spanner_context.yaml b/tests/modules/bigquery_connection/examples/spanner_context.yaml new file mode 100644 index 000000000..0ebccff56 --- /dev/null +++ b/tests/modules/bigquery_connection/examples/spanner_context.yaml @@ -0,0 +1,47 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.bigquery-connection.google_bigquery_connection.connection: + aws: [] + azure: [] + cloud_resource: [] + cloud_spanner: + - database: projects/my-project/instances/my-instance/databases/my-database + database_role: null + max_parallelism: 2 + use_data_boost: true + use_parallelism: true + use_serverless_analytics: null + cloud_sql: [] + connection_id: my-connection + description: A connection to a Spanner instance. + friendly_name: My BigQuery Connection + kms_key_name: null + location: EU + project: project-id + spark: [] + timeouts: null + module.bigquery-connection.google_bigquery_connection_iam_binding.authoritative["roles/bigquery.connectionUser"]: + condition: [] + connection_id: my-connection + location: EU + members: + - user:my-user@example.com + project: project-id + role: roles/bigquery.connectionUser + +counts: + google_bigquery_connection: 1 + google_bigquery_connection_iam_binding: 1 \ No newline at end of file diff --git a/tests/modules/bigquery_dataset/examples/routines.yaml b/tests/modules/bigquery_dataset/examples/routines.yaml index 03ae72cce..dfc1d8b52 100644 --- a/tests/modules/bigquery_dataset/examples/routines.yaml +++ b/tests/modules/bigquery_dataset/examples/routines.yaml @@ -48,7 +48,7 @@ values: project: my-project remote_function_options: [] return_table_type: null - return_type: null + return_type: '{"typeKind":"STRING"}' routine_id: custom_masking_routine routine_type: SCALAR_FUNCTION spark_options: [] @@ -58,4 +58,4 @@ counts: google_bigquery_dataset: 1 google_bigquery_routine: 1 modules: 1 - resources: 2 \ No newline at end of file + resources: 2