diff --git a/README.md b/README.md index 62be40813..432c71596 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Currently available modules: - **foundational** - [billing account](./modules/billing-account), [Cloud Identity group](./modules/cloud-identity-group/), [folder](./modules/folder), [service accounts](./modules/iam-service-account), [logging bucket](./modules/logging-bucket), [organization](./modules/organization), [project](./modules/project), [projects-data-source](./modules/projects-data-source) - **networking** - [DNS](./modules/dns), [DNS Response Policy](./modules/dns-response-policy/), [Cloud Endpoints](./modules/endpoints), [address reservation](./modules/net-address), [NAT](./modules/net-cloudnat), [VLAN Attachment](./modules/net-vlan-attachment/), [External Application LB](./modules/net-lb-app-ext/), [External Passthrough Network LB](./modules/net-lb-ext), [External Regional Application Load Balancer](./modules/net-lb-app-ext-regional/), [Firewall policy](./modules/net-firewall-policy), [Internal Application LB](./modules/net-lb-app-int), [Cross-region Internal Application LB](./modules/net-lb-app-int-cross-region), [Internal Passthrough Network LB](./modules/net-lb-int), [Internal Proxy Network LB](./modules/net-lb-proxy-int), [IPSec over Interconnect](./modules/net-ipsec-over-interconnect), [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC peering](./modules/net-vpc-peering), [VPN dynamic](./modules/net-vpn-dynamic), [HA VPN](./modules/net-vpn-ha), [VPN static](./modules/net-vpn-static), [Service Directory](./modules/service-directory), [Secure Web Proxy](./modules/net-swp) - **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [COS container](./modules/cloud-config-container/cos-generic-metadata/) (coredns, mysql, onprem, squid), [GKE cluster](./modules/gke-cluster-standard), [GKE hub](./modules/gke-hub), [GKE nodepool](./modules/gke-nodepool), [GCVE private cloud](./modules/gcve-private-cloud) -- **data** - [BigQuery dataset](./modules/bigquery-dataset), [Bigtable instance](./modules/bigtable-instance), [Dataplex](./modules/dataplex), [Dataplex DataScan](./modules/dataplex-datascan/), [Cloud SQL instance](./modules/cloudsql-instance), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag), [Datafusion](./modules/datafusion), [Dataproc](./modules/dataproc), [GCS](./modules/gcs), [Pub/Sub](./modules/pubsub) +- **data** - [BigQuery dataset](./modules/bigquery-dataset), [Bigtable instance](./modules/bigtable-instance), [Dataplex](./modules/dataplex), [Dataplex DataScan](./modules/dataplex-datascan/), [Cloud SQL instance](./modules/cloudsql-instance), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag), [Datafusion](./modules/datafusion), [Dataproc](./modules/dataproc), [GCS](./modules/gcs), [Pub/Sub](./modules/pubsub), [Dataform Repository](./modules/dataform-repository/) - **development** - [API Gateway](./modules/api-gateway), [Apigee](./modules/apigee), [Artifact Registry](./modules/artifact-registry), [Container Registry](./modules/container-registry), [Cloud Source Repository](./modules/source-repository), [Workstation cluster](./modules/workstation-cluster) - **security** - [Binauthz](./modules/binauthz/), [KMS](./modules/kms), [SecretManager](./modules/secret-manager), [VPC Service Control](./modules/vpc-sc) - **serverless** - [Cloud Function v1](./modules/cloud-function-v1), [Cloud Function v2](./modules/cloud-function-v2), [Cloud Run](./modules/cloud-run), [Cloud Run v2](./modules/cloud-run-v2) diff --git a/modules/README.md b/modules/README.md index 2641f1669..cf5750959 100644 --- a/modules/README.md +++ b/modules/README.md @@ -85,6 +85,7 @@ These modules are used in the examples included in this repository. If you are u - [Dataproc](./dataproc) - [GCS](./gcs) - [Pub/Sub](./pubsub) +- [Dataform Repository](./dataform-repository/) ## Development diff --git a/modules/dataform-repository/README.md b/modules/dataform-repository/README.md new file mode 100644 index 000000000..daf60a273 --- /dev/null +++ b/modules/dataform-repository/README.md @@ -0,0 +1,72 @@ +# Google Cloud Dataform Repository module + +This module allows managing a dataform repository, allows adding IAM permissions. Also enables attaching a remote repository. + +## TODO +[] Add validation rules to variable. + +## Examples + +### Simple dataform repository with access configration + +Simple dataform repository and specifying repository access via the IAM variable. + +```hcl +module "dataform" { + source = "./fabric/modules/dataform-repository" + project_id = "my-project" + name = "my-repository" + region = "europe-west1" + iam = { + "roles/dataform.editor" = ["user:user1@example.org"] + } +} +# tftest modules=1 resources=2 +``` + +### Repository with an attached remote repository + +This creates a dataform repository with a remote repository attached to it. In order to enable dataform to communicate with a 3P GIT provider, an access token must be generated and stored as a secret on GCP. For that, we utilize the existing [secret-manager module](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/modules/secret-manager). + +```hcl +module "secret" { + source = "./fabric/modules/secret-manager" + project_id = "fast-bi-fabric" + secrets = { + my-secret = { + } + } + versions = { + my-secret = { + v1 = { enabled = true, data = "MYTOKEN" } + } + } +} + +module "dataform" { + source = "./fabric/modules/dataform-repository" + project_id = "fast-bi-fabric" + name = "my-repository" + region = "europe-west1" + remote_repository_settings = { + url = "my-url" + secret_name = "my-secret" + token = module.secret.version_ids["my-secret:v1"] + } +} +# tftest modules=2 resources=3 +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L54) | Name of the dataform repository. | string | ✓ | | +| [project_id](variables.tf#L59) | Id of the project where resources will be created. | string | ✓ | | +| [region](variables.tf#L64) | The repository's region. | string | ✓ | | +| [iam](variables.tf#L17) | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | map(list(string)) | | {} | +| [iam_bindings](variables.tf#L24) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L39) | Keyring individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [remote_repository_settings](variables.tf#L69) | Remote settings required to attach the repository to a remote repository. | object({…}) | | null | +| [service_account](variables.tf#L81) | Service account used to execute the dataform workflow. | string | | "" | + diff --git a/modules/dataform-repository/iam.tf b/modules/dataform-repository/iam.tf new file mode 100644 index 000000000..5763e3c6c --- /dev/null +++ b/modules/dataform-repository/iam.tf @@ -0,0 +1,57 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_dataform_repository_iam_binding" "authoritative" { + provider = google-beta + for_each = var.iam + role = each.key + members = each.value + repository = google_dataform_repository.default.name +} + +resource "google_dataform_repository_iam_binding" "bindings" { + provider = google-beta + for_each = var.iam_bindings + role = each.value.role + members = each.value.members + repository = google_dataform_repository.default.name + + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_dataform_repository_iam_member" "bindings" { + provider = google-beta + for_each = var.iam_bindings_additive + role = each.value.role + member = each.value.member + repository = google_dataform_repository.default.name + + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} diff --git a/modules/dataform-repository/main.tf b/modules/dataform-repository/main.tf new file mode 100644 index 000000000..d37155876 --- /dev/null +++ b/modules/dataform-repository/main.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_dataform_repository" "default" { + provider = google-beta + project = var.project_id + name = var.name + region = var.region + service_account = var.service_account + + dynamic "git_remote_settings" { + for_each = var.remote_repository_settings != null ? [1] : [] + content { + url = var.remote_repository_settings.url + default_branch = var.remote_repository_settings.branch + authentication_token_secret_version = var.remote_repository_settings.secret_version + } + } +} diff --git a/modules/dataform-repository/variables.tf b/modules/dataform-repository/variables.tf new file mode 100644 index 000000000..649507b2f --- /dev/null +++ b/modules/dataform-repository/variables.tf @@ -0,0 +1,85 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "iam" { + description = "IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + default = {} + nullable = false +} + +variable "iam_bindings_additive" { + description = "Keyring individual additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + default = {} + nullable = false +} + +variable "name" { + description = "Name of the dataform repository." + type = string +} + +variable "project_id" { + description = "Id of the project where resources will be created." + type = string +} + +variable "region" { + description = "The repository's region." + type = string +} + +variable "remote_repository_settings" { + description = "Remote settings required to attach the repository to a remote repository." + type = object({ + url = optional(string) + branch = optional(string, "main") + secret_name = optional(string) + secret_version = optional(string, "v1") + token = optional(string) + }) + default = null +} + +variable "service_account" { + description = "Service account used to execute the dataform workflow." + type = string + default = "" +} diff --git a/modules/dataform-repository/versions.tf b/modules/dataform-repository/versions.tf new file mode 100644 index 000000000..84b66699c --- /dev/null +++ b/modules/dataform-repository/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.5.1" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 5.11.0, < 6.0.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 5.11.0, < 6.0.0" # tftest + } + } +}