diff --git a/.ci/cloudbuild.lint.yaml b/.ci/cloudbuild.lint.yaml index d4e88d5b2..002536cd6 100644 --- a/.ci/cloudbuild.lint.yaml +++ b/.ci/cloudbuild.lint.yaml @@ -19,13 +19,25 @@ steps: args: - -c - | - python -m pip install --user --no-warn-script-location -r /workspace/tools/REQUIREMENTS.txt + python -m pip install --user --no-warn-script-location -r /workspace/tools/REQUIREMENTS.txt && + wget https://releases.hashicorp.com/terraform/${_TERRAFORM_VERSION}/terraform_${_TERRAFORM_VERSION}_linux_amd64.zip && + unzip terraform_${_TERRAFORM_VERSION}_linux_amd64.zip -d /builder/home/.local/bin && + rm terraform_${_TERRAFORM_VERSION}_linux_amd64.zip && + chmod 755 /builder/home/.local/bin/terraform && + mkdir -p /workspace/.terraform.d/plugin-cache - name: python:3-alpine id: boilerplate args: ["/workspace/tools/check_boilerplate.py", "/workspace"] - - name: wata727/tflint - id: lint - args: ["/workspace"] + - name: python:3-alpine + id: terraform-fmt-check + entrypoint: sh + args: + - -c + - | + terraform fmt -recursive -check /workspace/ + env: + - PATH=/usr/local/bin:/usr/bin:/bin:/builder/home/.local/bin + - TF_CLI_CONFIG_FILE=/workspace/.ci/.terraformrc - name: python:3-alpine id: documentation args: @@ -37,6 +49,10 @@ steps: "foundations", "networking", ] + +substitutions: + _TERRAFORM_VERSION: 1.0.4 + tags: - ci - lint diff --git a/.ci/cloudbuild.test.environments.yaml b/.ci/cloudbuild.test.environments.yaml index 4b2bdd6a6..9c5b2a6ff 100644 --- a/.ci/cloudbuild.test.environments.yaml +++ b/.ci/cloudbuild.test.environments.yaml @@ -39,7 +39,7 @@ steps: - TF_CLI_CONFIG_FILE=/workspace/.ci/.terraformrc substitutions: - _TERRAFORM_VERSION: 0.15.4 + _TERRAFORM_VERSION: 1.0.4 tags: - "ci" diff --git a/.ci/cloudbuild.test.examples.yaml b/.ci/cloudbuild.test.examples.yaml index 4b7186e51..83c7cb946 100644 --- a/.ci/cloudbuild.test.examples.yaml +++ b/.ci/cloudbuild.test.examples.yaml @@ -40,7 +40,7 @@ options: machineType: "N1_HIGHCPU_8" substitutions: - _TERRAFORM_VERSION: 0.15.4 + _TERRAFORM_VERSION: 1.0.4 tags: - "ci" diff --git a/.ci/cloudbuild.test.modules.yaml b/.ci/cloudbuild.test.modules.yaml index ed1e6cc23..1222f2a53 100644 --- a/.ci/cloudbuild.test.modules.yaml +++ b/.ci/cloudbuild.test.modules.yaml @@ -39,7 +39,7 @@ options: machineType: "N1_HIGHCPU_8" substitutions: - _TERRAFORM_VERSION: 0.15.4 + _TERRAFORM_VERSION: 1.0.4 tags: - "ci" diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a0118241..c8bbd4e78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,12 +3,23 @@ All notable changes to this project will be documented in this file. ## [Unreleased] + +- new `apigee-organization` and `apigee-x-instance` +- generate `email` and `iam_email` statically in the `iam-service-account` module +- new `billing-budget` module +- fix `scheduled-asset-inventory-export-bq` module +- output custom role information from the `organization` module + +## [5.1.0] - 2021-08-30 + +- add support for `lifecycle_rule` in gcs module - create `pubsub` service identity if service is enabled - support for creation of GKE Autopilot clusters - add support for CMEK keys in Data Foundation end to end example - add support for VPC-SC perimeters in Data Foundation end to end example - fix `vpc-sc` module -- new networking example showing how to use [Private Service Connect to call a Cloud Function from on-premises](networking/private-cloud-function-from-onprem/) +- new networking example showing how to use [Private Service Connect to call a Cloud Function from on-premises](./networking/private-cloud-function-from-onprem/) +- new networking example showing how to organize [decentralized firewall](./networking/decentralized-firewall/) management on GCP ## [5.0.0] - 2021-06-17 @@ -330,7 +341,8 @@ All notable changes to this project will be documented in this file. - merge development branch with suite of new modules and end-to-end examples -[Unreleased]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v5.0.0...HEAD +[Unreleased]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v5.1.0...HEAD +[5.1.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v5.0.0...v5.1.0 [5.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v4.9.0...v5.0.0 [4.9.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v4.8.0...v4.9.0 [4.8.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v4.7.0...v4.8.0 diff --git a/README.md b/README.md index df419fcae..3cc9315fa 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ The examples in this repository are split in several main sections: **foundation Currently available examples: - **foundations** - [single level hierarchy](./foundations/environments/) (environments), [multiple level hierarchy](./foundations/business-units/) (business units + environments) -- **networking** - [hub and spoke via peering](./networking/hub-and-spoke-peering/), [hub and spoke via VPN](./networking/hub-and-spoke-vpn/), [DNS and Google Private Access for on-premises](./networking/onprem-google-access-dns/), [Shared VPC with GKE support](./networking/shared-vpc-gke/), [ILB as next hop](./networking/ilb-next-hop) +- **networking** - [hub and spoke via peering](./networking/hub-and-spoke-peering/), [hub and spoke via VPN](./networking/hub-and-spoke-vpn/), [DNS and Google Private Access for on-premises](./networking/onprem-google-access-dns/), [Shared VPC with GKE support](./networking/shared-vpc-gke/), [ILB as next hop](./networking/ilb-next-hop), [PSC for on-premises Cloud Function invocation](./networking/private-cloud-function-from-onprem/), [decentralized firewall](./networking/decentralized-firewall) - **data solutions** - [GCE/GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms/), [Cloud Storage to Bigquery with Cloud Dataflow](./data-solutions/gcs-to-bq-with-dataflow/) - **cloud operations** - [Resource tracking and remediation via Cloud Asset feeds](.//cloud-operations/asset-inventory-feed-remediation), [Granular Cloud DNS IAM via Service Directory](./cloud-operations/dns-fine-grained-iam), [Granular Cloud DNS IAM for Shared VPC](./cloud-operations/dns-shared-vpc), [Compute Engine quota monitoring](./cloud-operations/quota-monitoring), [Scheduled Cloud Asset Inventory Export to Bigquery](./cloud-operations/scheduled-asset-inventory-export-bq) - **third party solutions** - [OpenShift cluster on Shared VPC](./third-party-solutions/openshift) @@ -34,12 +34,12 @@ The current list of modules supports most of the core foundational and networkin Currently available modules: -- **foundational** - [folder](./modules/folder), [organization](./modules/organization), [project](./modules/project), [service accounts](./modules/iam-service-account) +- **foundational** - [folder](./modules/folder), [organization](./modules/organization), [project](./modules/project), [service accounts](./modules/iam-service-account), [logging bucket](./modules/logging-bucket) - **networking** - [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC peering](./modules/net-vpc-peering), [VPN static](./modules/net-vpn-static), [VPN dynamic](./modules/net-vpn-dynamic), [VPN HA](./modules/net-vpn-ha), [NAT](./modules/net-cloudnat), [address reservation](./modules/net-address), [DNS](./modules/dns), [L4 ILB](./modules/net-ilb), [Service Directory](./modules/service-directory), [Cloud Endpoints](./modules/cloudenpoints) - **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [GKE cluster](./modules/gke-cluster), [GKE nodepool](./modules/gke-nodepool), [COS container](./modules/cos-container) (coredns, mysql, onprem, squid) - **data** - [GCS](./modules/gcs), [BigQuery dataset](./modules/bigquery-dataset), [Pub/Sub](./modules/pubsub), [Datafusion](./modules/datafusion), [Bigtable instance](./modules/bigtable-instance) -- **development** - [Cloud Source Repository](./modules/source-repository), [Container Registry](./modules/container-registry), [Artifact Registry](./modules/artifact-registry) +- **development** - [Cloud Source Repository](./modules/source-repository), [Container Registry](./modules/container-registry), [Artifact Registry](./modules/artifact-registry), [Apigee Organization](./modules/apigee-organization), [Apigee X Instance](./modules/apigee-x-instance) - **security** - [KMS](./modules/kms), [SecretManager](./modules/secret-manager), [VPC Service Control](./modules/vpc-sc) -- **serverless** - [Cloud Functions](./modules/cloud-function) +- **serverless** - [Cloud Function](./modules/cloud-function) For more information and usage examples see each module's README file. diff --git a/cloud-operations/scheduled-asset-inventory-export-bq/README.md b/cloud-operations/scheduled-asset-inventory-export-bq/README.md index 44c6ecbb7..1abecdd94 100644 --- a/cloud-operations/scheduled-asset-inventory-export-bq/README.md +++ b/cloud-operations/scheduled-asset-inventory-export-bq/README.md @@ -43,9 +43,9 @@ You can also create a dashboard connecting [Datalab](https://datastudio.google.c | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| billing_account | Billing account id used as default for new projects. | string | ✓ | | | cai_config | Cloud Asset inventory export config. | object({...}) | ✓ | | | project_id | Project id that references existing project. | string | ✓ | | +| *billing_account* | Billing account id used as default for new projects. | string | | null | | *bundle_path* | Path used to write the intermediate Cloud Function code bundle. | string | | ./bundle.zip | | *location* | Appe Engine location used in the example. | string | | europe-west | | *name* | Arbitrary string used to name created resources. | string | | asset-inventory | diff --git a/cloud-operations/scheduled-asset-inventory-export-bq/main.tf b/cloud-operations/scheduled-asset-inventory-export-bq/main.tf index c11b2c1b4..005c0fe38 100644 --- a/cloud-operations/scheduled-asset-inventory-export-bq/main.tf +++ b/cloud-operations/scheduled-asset-inventory-export-bq/main.tf @@ -22,7 +22,7 @@ module "project" { source = "../../modules/project" name = var.project_id parent = var.root_node - billing_account = var.billing_account + billing_account = try(var.billing_account, null) project_create = var.project_create services = [ "bigquery.googleapis.com", @@ -33,6 +33,11 @@ module "project" { "cloudscheduler.googleapis.com", "pubsub.googleapis.com" ] + iam = { + "roles/resourcemanager.projectIamAdmin" = ["serviceAccount:${module.project.service_accounts.robots.cloudasset}"] + "roles/bigquery.dataEditor" = ["serviceAccount:${module.project.service_accounts.robots.cloudasset}"] + "roles/bigquery.user" = ["serviceAccount:${module.project.service_accounts.robots.cloudasset}"] + } } module "service-account" { @@ -40,7 +45,9 @@ module "service-account" { project_id = module.project.project_id name = "${var.name}-cf" iam_project_roles = { - (var.project_id) = ["roles/cloudasset.viewer"] + (var.project_id) = [ + "roles/cloudasset.owner", + ] } } diff --git a/cloud-operations/scheduled-asset-inventory-export-bq/variables.tf b/cloud-operations/scheduled-asset-inventory-export-bq/variables.tf index ab89f77e0..6f8217d33 100644 --- a/cloud-operations/scheduled-asset-inventory-export-bq/variables.tf +++ b/cloud-operations/scheduled-asset-inventory-export-bq/variables.tf @@ -17,6 +17,7 @@ variable "billing_account" { description = "Billing account id used as default for new projects." type = string + default = null } variable "bundle_path" { diff --git a/data-solutions/data-platform-foundations/01-environment/variables.tf b/data-solutions/data-platform-foundations/01-environment/variables.tf index ec945a881..92ba230d7 100644 --- a/data-solutions/data-platform-foundations/01-environment/variables.tf +++ b/data-solutions/data-platform-foundations/01-environment/variables.tf @@ -71,6 +71,6 @@ variable "service_encryption_key_ids" { variable "service_perimeter_standard" { description = "VPC Service control standard perimeter name in the form of 'accessPolicies/ACCESS_POLICY_NAME/servicePerimeters/PERIMETER_NAME'. All projects will be added to the perimeter in enforced mode." - type = string - default = null + type = string + default = null } diff --git a/foundations/environments/locals.tf b/foundations/environments/locals.tf index 841fe152f..e18fc2acc 100644 --- a/foundations/environments/locals.tf +++ b/foundations/environments/locals.tf @@ -21,7 +21,7 @@ locals { var.iam_billing_config.target_org ? [] : ["roles/billing.user"] ) sa_billing_org_role = ( - ! var.iam_billing_config.target_org ? [] : ["roles/billing.user"] + !var.iam_billing_config.target_org ? [] : ["roles/billing.user"] ) sa_xpn_folder_role = ( local.sa_xpn_target_org ? [] : ["roles/compute.xpnAdmin"] diff --git a/foundations/environments/main.tf b/foundations/environments/main.tf index 4352f2f22..7174b2f5b 100644 --- a/foundations/environments/main.tf +++ b/foundations/environments/main.tf @@ -24,7 +24,7 @@ module "tf-project" { parent = var.root_node prefix = var.prefix billing_account = var.billing_account_id - iam_additive = { + iam_additive = { "roles/owner" = var.iam_terraform_owners } services = var.project_services @@ -158,7 +158,7 @@ module "sharedsvc-project" { parent = var.root_node prefix = var.prefix billing_account = var.billing_account_id - iam_additive = { + iam_additive = { "roles/owner" = var.iam_shared_owners } services = var.project_services diff --git a/modules/README.md b/modules/README.md index a86196f51..2e12ef3ae 100644 --- a/modules/README.md +++ b/modules/README.md @@ -14,6 +14,7 @@ Specific modules also offer support for non-authoritative bindings (e.g. `google - [organization](./organization) - [project](./project) - [service account](./iam-service-account) +- [logging bucket](./logging-bucket) ## Networking modules @@ -52,6 +53,8 @@ Specific modules also offer support for non-authoritative bindings (e.g. `google - [Artifact Registry](./artifact-registry) - [Container Registry](./container-registry) - [Source Repository](./source-repository) +- [Apigee Organization](./apigee-organization) +- [Apigee X Instance](./apigee-x-instance) ## Security diff --git a/modules/apigee-organization/README.md b/modules/apigee-organization/README.md new file mode 100644 index 000000000..b725243f4 --- /dev/null +++ b/modules/apigee-organization/README.md @@ -0,0 +1,125 @@ +# Google Apigee Organization Module + +This module allows managing a single Apigee organization and its environments and environmentgroups. + +## Examples + +### Apigee X Evaluation Organization + +```hcl +module "apigee-organization" { + source = "./modules/apigee-organization" + project_id = "my-project" + analytics_region = "us-central1" + runtime_type = "CLOUD" + authorized_network = "my-vpc" + apigee_environments = [ + "eval1", + "eval2" + ] + apigee_envgroups = { + eval = { + environments = [ + "eval1", + "eval2" + ] + hostnames = [ + "eval.api.example.com" + ] + } + } +} +# tftest:modules=1:resources=6 +``` + +### Apigee X Paid Organization + +```hcl +module "apigee-organization" { + source = "./modules/apigee-organization" + project_id = "my-project" + analytics_region = "us-central1" + runtime_type = "CLOUD" + authorized_network = "my-vpc" + database_encryption_key = "my-data-key" + apigee_environments = [ + "dev1", + "dev2", + "test1", + "test2" + ] + apigee_envgroups = { + dev = { + environments = [ + "dev1", + "dev2" + ] + hostnames = [ + "dev.api.example.com" + ] + } + test = { + environments = [ + "test1", + "test2" + ] + hostnames = [ + "test.api.example.com" + ] + } + } +} +# tftest:modules=1:resources=11 +``` + +### Apigee hybrid Organization + +```hcl +module "apigee-organization" { + source = "./modules/apigee-organization" + project_id = "my-project" + analytics_region = "us-central1" + runtime_type = "HYBRID" + apigee_environments = [ + "eval1", + "eval2" + ] + apigee_envgroups = { + eval = { + environments = [ + "eval1", + "eval2" + ] + hostnames = [ + "eval.api.example.com" + ] + } + } +} +# tftest:modules=1:resources=6 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| analytics_region | Analytics Region for the Apigee Organization (immutable). See https://cloud.google.com/apigee/docs/api-platform/get-started/install-cli. | string | ✓ | | +| project_id | Project ID to host this Apigee organization (will also become the Apigee Org name). | string | ✓ | | +| runtime_type | None | string | ✓ | | +| *apigee_envgroups* | Apigee Environment Groups. | map(object({...})) | | {} | +| *apigee_environments* | Apigee Environment Names. | list(string) | | [] | +| *authorized_network* | VPC network self link (requires service network peering enabled (Used in Apigee X only). | string | | null | +| *database_encryption_key* | Cloud KMS key self link (e.g. `projects/foo/locations/us/keyRings/bar/cryptoKeys/baz`) used for encrypting the data that is stored and replicated across runtime instances (immutable, used in Apigee X only). | string | | null | +| *description* | Description of the Apigee Organization. | string | | Apigee Organization created by tf module | +| *display_name* | Display Name of the Apigee Organization. | string | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| org | Apigee Organization. | | +| org_ca_certificate | Apigee organization CA certificate. | | +| org_id | Apigee Organization ID. | | +| subscription_type | Apigee subscription type. | | + diff --git a/modules/apigee-organization/main.tf b/modules/apigee-organization/main.tf new file mode 100644 index 000000000..b1c134814 --- /dev/null +++ b/modules/apigee-organization/main.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + env_envgroup_pairs = flatten([ + for eg_name, eg in var.apigee_envgroups : [ + for e in eg.environments : { + envgroup = eg_name + env = e + } + ] + ]) +} + +resource "google_apigee_organization" "apigee_org" { + project_id = var.project_id + analytics_region = var.analytics_region + display_name = var.display_name + description = var.description + runtime_type = var.runtime_type + authorized_network = var.authorized_network + runtime_database_encryption_key_name = var.database_encryption_key +} + +resource "google_apigee_environment" "apigee_env" { + for_each = toset(var.apigee_environments) + org_id = google_apigee_organization.apigee_org.id + name = each.key +} + +resource "google_apigee_envgroup" "apigee_envgroup" { + for_each = var.apigee_envgroups + org_id = google_apigee_organization.apigee_org.id + name = each.key + hostnames = each.value.hostnames +} + +resource "google_apigee_envgroup_attachment" "env_to_envgroup_attachment" { + for_each = { for pair in local.env_envgroup_pairs : "${pair.envgroup}-${pair.env}" => pair } + envgroup_id = google_apigee_envgroup.apigee_envgroup[each.value.envgroup].id + environment = google_apigee_environment.apigee_env[each.value.env].name +} \ No newline at end of file diff --git a/modules/apigee-organization/outputs.tf b/modules/apigee-organization/outputs.tf new file mode 100644 index 000000000..6ff012500 --- /dev/null +++ b/modules/apigee-organization/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "org" { + description = "Apigee Organization." + value = google_apigee_organization.apigee_org +} + +output "org_ca_certificate" { + description = "Apigee organization CA certificate." + value = google_apigee_organization.apigee_org.ca_certificate +} + +output "org_id" { + description = "Apigee Organization ID." + value = google_apigee_organization.apigee_org.id +} + +output "subscription_type" { + description = "Apigee subscription type." + value = google_apigee_organization.apigee_org.subscription_type +} diff --git a/modules/apigee-organization/variables.tf b/modules/apigee-organization/variables.tf new file mode 100644 index 000000000..8978542e7 --- /dev/null +++ b/modules/apigee-organization/variables.tf @@ -0,0 +1,75 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "authorized_network" { + description = "VPC network self link (requires service network peering enabled (Used in Apigee X only)." + type = string + default = null +} + +variable "analytics_region" { + description = "Analytics Region for the Apigee Organization (immutable). See https://cloud.google.com/apigee/docs/api-platform/get-started/install-cli." + type = string +} + +variable "apigee_envgroups" { + description = "Apigee Environment Groups." + type = map(object({ + environments = list(string) + hostnames = list(string) + })) + default = {} +} + +variable "apigee_environments" { + description = "Apigee Environment Names." + type = list(string) + default = [] +} + +variable "database_encryption_key" { + description = "Cloud KMS key self link (e.g. `projects/foo/locations/us/keyRings/bar/cryptoKeys/baz`) used for encrypting the data that is stored and replicated across runtime instances (immutable, used in Apigee X only)." + type = string + default = null +} + +variable "description" { + description = "Description of the Apigee Organization." + type = string + default = "Apigee Organization created by tf module" +} + +variable "display_name" { + description = "Display Name of the Apigee Organization." + type = string + default = null +} + +variable "project_id" { + description = "Project ID to host this Apigee organization (will also become the Apigee Org name)." + type = string +} + +variable "runtime_type" { + type = string + + validation { + condition = contains(["CLOUD", "HYBRID"], var.runtime_type) + error_message = "Allowed values for runtime_type \"CLOUD\" or \"HYBRID\"." + } +} + + diff --git a/modules/apigee-x-instance/README.md b/modules/apigee-x-instance/README.md new file mode 100644 index 000000000..371f8f0b7 --- /dev/null +++ b/modules/apigee-x-instance/README.md @@ -0,0 +1,67 @@ +# Google Apigee X Instance Module + +This module allows managing a single Apigee X instance and its environment attachments. + +## Examples + +### Apigee X Evaluation Instance + +```hcl +module "apigee-x-instance" { + source = "./modules/apigee-x-instance" + name = "my-us-instance" + region = "us-central1" + cidr_mask = 22 + + apigee_org_id = "my-project" + apigee_environments = [ + "eval1", + "eval2" + ] +} +# tftest:modules=1:resources=3 +``` + +### Apigee X Paid Instance + +```hcl +module "apigee-x-instance" { + source = "./modules/apigee-x-instance" + name = "my-us-instance" + region = "us-central1" + cidr_mask = 16 + disk_encryption_key = "my-disk-key" + + apigee_org_id = "my-project" + apigee_environments = [ + "dev1", + "dev2", + "test1", + "test2" + ] +} +# tftest:modules=1:resources=5 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| apigee_org_id | Apigee Organization ID | string | ✓ | | +| cidr_mask | CIDR mask for the Apigee instance | number | ✓ | | +| name | Apigee instance name. | string | ✓ | | +| region | Compute region. | string | ✓ | | +| *apigee_envgroups* | Apigee Environment Groups. | map(object({...})) | | {} | +| *apigee_environments* | Apigee Environment Names. | list(string) | | [] | +| *disk_encryption_key* | Customer Managed Encryption Key (CMEK) self link (e.g. `projects/foo/locations/us/keyRings/bar/cryptoKeys/baz`) used for disk and volume encryption (required for PAID Apigee Orgs only). | string | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| endpoint | Internal endpoint of the Apigee instance. | | +| id | Apigee instance ID. | | +| instance | Apigee instance. | | +| port | Port number of the internal endpoint of the Apigee instance. | | + diff --git a/modules/apigee-x-instance/main.tf b/modules/apigee-x-instance/main.tf new file mode 100644 index 000000000..552d6d085 --- /dev/null +++ b/modules/apigee-x-instance/main.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_apigee_instance" "apigee_instance" { + org_id = var.apigee_org_id + name = var.name + location = var.region + peering_cidr_range = "SLASH_${var.cidr_mask}" + disk_encryption_key_name = var.disk_encryption_key +} + +resource "google_apigee_instance_attachment" "apigee_instance_attchment" { + for_each = toset(var.apigee_environments) + instance_id = google_apigee_instance.apigee_instance.id + environment = each.key +} diff --git a/modules/apigee-x-instance/outputs.tf b/modules/apigee-x-instance/outputs.tf new file mode 100644 index 000000000..0f2d5d6bb --- /dev/null +++ b/modules/apigee-x-instance/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * limitations under the License. + * See the License for the specific language governing permissions and + */ + +output "endpoint" { + description = "Internal endpoint of the Apigee instance." + value = google_apigee_instance.apigee_instance.host +} + +output "id" { + description = "Apigee instance ID." + value = google_apigee_instance.apigee_instance.id +} + +output "instance" { + description = "Apigee instance." + value = google_apigee_instance.apigee_instance +} + +output "port" { + description = "Port number of the internal endpoint of the Apigee instance." + value = google_apigee_instance.apigee_instance.port +} diff --git a/modules/apigee-x-instance/variables.tf b/modules/apigee-x-instance/variables.tf new file mode 100644 index 000000000..219ee7d6a --- /dev/null +++ b/modules/apigee-x-instance/variables.tf @@ -0,0 +1,60 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "apigee_envgroups" { + description = "Apigee Environment Groups." + type = map(object({ + environments = list(string) + hostnames = list(string) + })) + default = {} +} + +variable "apigee_environments" { + description = "Apigee Environment Names." + type = list(string) + default = [] +} + +variable "apigee_org_id" { + description = "Apigee Organization ID" + type = string +} + +variable "cidr_mask" { + description = "CIDR mask for the Apigee instance" + type = number + validation { + condition = contains([16, 20, 22], var.cidr_mask) + error_message = "Invalid CIDR mask; Allowed values for cidr_mask: [16, 20, 22]." + } +} + +variable "disk_encryption_key" { + description = "Customer Managed Encryption Key (CMEK) self link (e.g. `projects/foo/locations/us/keyRings/bar/cryptoKeys/baz`) used for disk and volume encryption (required for PAID Apigee Orgs only)." + type = string + default = null +} + +variable "name" { + description = "Apigee instance name." + type = string +} + +variable "region" { + description = "Compute region." + type = string +} diff --git a/modules/bigtable-instance/main.tf b/modules/bigtable-instance/main.tf index 33c4806e3..f3081fca3 100644 --- a/modules/bigtable-instance/main.tf +++ b/modules/bigtable-instance/main.tf @@ -49,7 +49,7 @@ resource "google_bigtable_table" "default" { name = each.key split_keys = each.value.split_keys - dynamic column_family { + dynamic "column_family" { for_each = each.value.column_family != null ? [""] : [] content { diff --git a/modules/billing-budget/README.md b/modules/billing-budget/README.md new file mode 100644 index 000000000..2b1eaced9 --- /dev/null +++ b/modules/billing-budget/README.md @@ -0,0 +1,88 @@ +# Google Cloud Billing Budget Module + +This module allows creating a Cloud Billing budget for a set of services and projects. + +To create billing budgets you need one of the following IAM roles on the target billing account: + +* Billing Account Administrator +* Billing Account Costs Manager + +## Examples + +### Simple email notification + +Send a notification to an email when a set of projects reach $100 of spend. + +```hcl +module "budget" { + source = "./modules/billing-budget" + billing_account = var.billing_account_id + name = "$100 budget" + amount = 100 + thresholds = { + current = [0.5, 0.75, 1.0] + forecasted = [1.0] + } + projects = [ + "projects/123456789000", + "projects/123456789111" + ] + email_recipients = { + project_id = "my-project" + emails = ["user@example.com"] + } +} +# tftest:modules=1:resources=2 +``` + +### Pubsub notification + +Send a notification to a PubSub topic the total spend of a billing account reaches the previous month's spend. + + +```hcl +module "budget" { + source = "./modules/billing-budget" + billing_account = var.billing_account_id + name = "previous period budget" + amount = 0 + thresholds = { + current = [1.0] + forecasted = [] + } + pubsub_topic = module.pubsub.id +} + +module "pubsub" { + source = "./modules/pubsub" + project_id = var.project_id + name = "budget-topic" +} + +# tftest:modules=2:resources=2 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| billing_account | Billing account id. | string | ✓ | | +| name | Budget name. | string | ✓ | | +| thresholds | None | object({...}) | ✓ | | +| *amount* | Amount in the billing account's currency for the budget. Use 0 to set budget to 100% of last period's spend. | number | | 0 | +| *credit_treatment* | How credits should be treated when determining spend for threshold calculations. Only INCLUDE_ALL_CREDITS or EXCLUDE_ALL_CREDITS are supported | string | | ... | +| *email_recipients* | Emails where budget notifications will be sent. Setting this will create a notification channel for each email in the specified project. | object({...}) | | null | +| *notification_channels* | Monitoring notification channels where to send updates. | list(string) | | null | +| *notify_default_recipients* | Notify Billing Account Administrators and Billing Account Users IAM roles for the target account. | bool | | false | +| *projects* | List of projects of the form projects/{project_number}, specifying that usage from only this set of projects should be included in the budget. Set to null to include all projects linked to the billing account. | list(string) | | null | +| *pubsub_topic* | The ID of the Cloud Pub/Sub topic where budget related messages will be published. | string | | null | +| *services* | List of services of the form services/{service_id}, specifying that usage from only this set of services should be included in the budget. Set to null to include usage for all services. | list(string) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| budget | Budget resource. | | +| id | Budget ID. | | + diff --git a/modules/billing-budget/main.tf b/modules/billing-budget/main.tf new file mode 100644 index 000000000..739dcedd4 --- /dev/null +++ b/modules/billing-budget/main.tf @@ -0,0 +1,95 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + spend_basis = { + current = "CURRENT_SPEND" + forecasted = "FORECASTED_SPEND" + } + threshold_pairs = flatten([ + for type, values in var.thresholds : [ + for value in values : { + spend_basis = local.spend_basis[type] + threshold_percent = value + } + ] + ]) + + notification_channels = concat( + [for channel in google_monitoring_notification_channel.email_channels : channel.id], + coalesce(var.notification_channels, []) + ) +} + +resource "google_monitoring_notification_channel" "email_channels" { + for_each = toset(try(var.email_recipients.emails, [])) + display_name = "${var.name} budget email notification (${each.value})" + type = "email" + project = var.email_recipients.project_id + labels = { + email_address = each.value + } + user_labels = {} +} + + +resource "google_billing_budget" "budget" { + billing_account = var.billing_account + display_name = var.name + + budget_filter { + projects = var.projects + credit_types_treatment = var.credit_treatment + services = var.services + } + + dynamic "amount" { + for_each = var.amount == 0 ? [1] : [] + content { + last_period_amount = true + } + } + + dynamic "amount" { + for_each = var.amount != 0 ? [1] : [] + content { + dynamic "specified_amount" { + for_each = var.amount != 0 ? [1] : [] + content { + units = var.amount + } + } + } + } + + dynamic "threshold_rules" { + for_each = local.threshold_pairs + iterator = threshold + content { + threshold_percent = threshold.value.threshold_percent + spend_basis = threshold.value.spend_basis + } + } + + all_updates_rule { + monitoring_notification_channels = local.notification_channels + pubsub_topic = var.pubsub_topic + # disable_default_iam_recipients can only be set if + # monitoring_notification_channels is nonempty + disable_default_iam_recipients = try(length(var.notification_channels), 0) > 0 && !var.notify_default_recipients + schema_version = "1.0" + } +} diff --git a/modules/billing-budget/outputs.tf b/modules/billing-budget/outputs.tf new file mode 100644 index 000000000..9f2dd4ffb --- /dev/null +++ b/modules/billing-budget/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "budget" { + description = "Budget resource." + value = google_billing_budget.budget +} + +output "id" { + description = "Budget ID." + value = google_billing_budget.budget.id +} diff --git a/modules/billing-budget/variables.tf b/modules/billing-budget/variables.tf new file mode 100644 index 000000000..3125d37d3 --- /dev/null +++ b/modules/billing-budget/variables.tf @@ -0,0 +1,94 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "amount" { + description = "Amount in the billing account's currency for the budget. Use 0 to set budget to 100% of last period's spend." + type = number + default = 0 +} + +variable "billing_account" { + description = "Billing account id." + type = string +} + +variable "credit_treatment" { + description = "How credits should be treated when determining spend for threshold calculations. Only INCLUDE_ALL_CREDITS or EXCLUDE_ALL_CREDITS are supported" + type = string + default = "INCLUDE_ALL_CREDITS" + validation { + condition = ( + var.credit_treatment == "INCLUDE_ALL_CREDITS" || + var.credit_treatment == "EXCLUDE_ALL_CREDITS" + ) + error_message = "Argument credit_treatment must be INCLUDE_ALL_CREDITS or EXCLUDE_ALL_CREDITS." + } +} + +variable "email_recipients" { + description = "Emails where budget notifications will be sent. Setting this will create a notification channel for each email in the specified project." + type = object({ + project_id = string + emails = list(string) + }) + default = null +} + +variable "name" { + description = "Budget name." + type = string +} + +variable "notification_channels" { + description = "Monitoring notification channels where to send updates." + type = list(string) + default = null +} + +variable "notify_default_recipients" { + description = "Notify Billing Account Administrators and Billing Account Users IAM roles for the target account." + type = bool + default = false +} + +variable "projects" { + description = "List of projects of the form projects/{project_number}, specifying that usage from only this set of projects should be included in the budget. Set to null to include all projects linked to the billing account." + type = list(string) + default = null +} + +variable "pubsub_topic" { + description = "The ID of the Cloud Pub/Sub topic where budget related messages will be published." + type = string + default = null +} + +variable "services" { + description = "List of services of the form services/{service_id}, specifying that usage from only this set of services should be included in the budget. Set to null to include usage for all services." + type = list(string) + default = null +} + +variable "thresholds" { + type = object({ + current = list(number) + forecasted = list(number) + }) + validation { + condition = length(var.thresholds.current) > 0 || length(var.thresholds.forecasted) > 0 + error_message = "Must specify at least one budget threshold." + } +} diff --git a/modules/billing-budget/versions.tf b/modules/billing-budget/versions.tf new file mode 100644 index 000000000..968f411ef --- /dev/null +++ b/modules/billing-budget/versions.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13.0" + required_providers { + google = ">= 3.79.0" + google-beta = ">= 3.79.0" + } +} diff --git a/modules/cloud-config-container/instance.tf b/modules/cloud-config-container/instance.tf index a947c4f67..447becf51 100644 --- a/modules/cloud-config-container/instance.tf +++ b/modules/cloud-config-container/instance.tf @@ -59,7 +59,7 @@ resource "google_compute_instance" "default" { user-data = local.cloud_config }) - dynamic attached_disk { + dynamic "attached_disk" { for_each = var.test_instance_defaults.disks iterator = disk content { @@ -84,7 +84,7 @@ resource "google_compute_instance" "default" { network_interface { network = var.test_instance.network subnetwork = var.test_instance.subnetwork - dynamic access_config { + dynamic "access_config" { for_each = var.test_instance_defaults.nat ? [""] : [] iterator = config content { diff --git a/modules/cloud-function/main.tf b/modules/cloud-function/main.tf index 52904093c..cbef1b477 100644 --- a/modules/cloud-function/main.tf +++ b/modules/cloud-function/main.tf @@ -78,12 +78,12 @@ resource "google_cloudfunctions_function" "function" { var.vpc_connector_config.egress_settings, null ) - dynamic event_trigger { + dynamic "event_trigger" { for_each = var.trigger_config == null ? [] : [""] content { event_type = var.trigger_config.event resource = var.trigger_config.resource - dynamic failure_policy { + dynamic "failure_policy" { for_each = var.trigger_config.retry == null ? [] : [""] content { retry = var.trigger_config.retry @@ -114,7 +114,7 @@ resource "google_storage_bucket" "bucket" { ) labels = var.labels - dynamic lifecycle_rule { + dynamic "lifecycle_rule" { for_each = var.bucket_config.lifecycle_delete_age == null ? [] : [""] content { action { type = "Delete" } diff --git a/modules/cloud-function/outputs.tf b/modules/cloud-function/outputs.tf index 593a607fd..0b625b12c 100644 --- a/modules/cloud-function/outputs.tf +++ b/modules/cloud-function/outputs.tf @@ -16,7 +16,7 @@ output "bucket" { description = "Bucket resource (only if auto-created)." - value = try( + value = try( var.bucket_config == null ? null : google_storage_bucket.bucket.0, null ) } @@ -38,7 +38,7 @@ output "function_name" { output "service_account" { description = "Service account resource." - value = try(google_service_account.service_account[0], null) + value = try(google_service_account.service_account[0], null) } output "service_account_email" { diff --git a/modules/compute-mig/main.tf b/modules/compute-mig/main.tf index 968d41ed8..75ab2d3dc 100644 --- a/modules/compute-mig/main.tf +++ b/modules/compute-mig/main.tf @@ -28,7 +28,7 @@ resource "google_compute_autoscaler" "default" { min_replicas = var.autoscaler_config.min_replicas cooldown_period = var.autoscaler_config.cooldown_period - dynamic cpu_utilization { + dynamic "cpu_utilization" { for_each = ( var.autoscaler_config.cpu_utilization_target == null ? [] : [""] ) @@ -37,7 +37,7 @@ resource "google_compute_autoscaler" "default" { } } - dynamic load_balancing_utilization { + dynamic "load_balancing_utilization" { for_each = ( var.autoscaler_config.load_balancing_utilization_target == null ? [] : [""] ) @@ -46,7 +46,7 @@ resource "google_compute_autoscaler" "default" { } } - dynamic metric { + dynamic "metric" { for_each = ( var.autoscaler_config.metric == null ? [] @@ -76,7 +76,7 @@ resource "google_compute_instance_group_manager" "default" { target_size = var.target_size target_pools = var.target_pools wait_for_instances = var.wait_for_instances - dynamic auto_healing_policies { + dynamic "auto_healing_policies" { for_each = var.auto_healing_policies == null ? [] : [var.auto_healing_policies] iterator = config content { @@ -84,7 +84,7 @@ resource "google_compute_instance_group_manager" "default" { initial_delay_sec = config.value.initial_delay_sec } } - dynamic update_policy { + dynamic "update_policy" { for_each = var.update_policy == null ? [] : [var.update_policy] iterator = config content { @@ -105,7 +105,7 @@ resource "google_compute_instance_group_manager" "default" { ) } } - dynamic named_port { + dynamic "named_port" { for_each = var.named_ports == null ? {} : var.named_ports iterator = config content { @@ -117,7 +117,7 @@ resource "google_compute_instance_group_manager" "default" { instance_template = var.default_version.instance_template name = var.default_version.name } - dynamic version { + dynamic "version" { for_each = var.versions == null ? {} : var.versions iterator = version content { @@ -150,7 +150,7 @@ resource "google_compute_region_autoscaler" "default" { min_replicas = var.autoscaler_config.min_replicas cooldown_period = var.autoscaler_config.cooldown_period - dynamic cpu_utilization { + dynamic "cpu_utilization" { for_each = ( var.autoscaler_config.cpu_utilization_target == null ? [] : [""] ) @@ -159,7 +159,7 @@ resource "google_compute_region_autoscaler" "default" { } } - dynamic load_balancing_utilization { + dynamic "load_balancing_utilization" { for_each = ( var.autoscaler_config.load_balancing_utilization_target == null ? [] : [""] ) @@ -168,7 +168,7 @@ resource "google_compute_region_autoscaler" "default" { } } - dynamic metric { + dynamic "metric" { for_each = ( var.autoscaler_config.metric == null ? [] @@ -198,7 +198,7 @@ resource "google_compute_region_instance_group_manager" "default" { target_size = var.target_size target_pools = var.target_pools wait_for_instances = var.wait_for_instances - dynamic auto_healing_policies { + dynamic "auto_healing_policies" { for_each = var.auto_healing_policies == null ? [] : [var.auto_healing_policies] iterator = config content { @@ -206,7 +206,7 @@ resource "google_compute_region_instance_group_manager" "default" { initial_delay_sec = config.value.initial_delay_sec } } - dynamic update_policy { + dynamic "update_policy" { for_each = var.update_policy == null ? [] : [var.update_policy] iterator = config content { @@ -227,7 +227,7 @@ resource "google_compute_region_instance_group_manager" "default" { ) } } - dynamic named_port { + dynamic "named_port" { for_each = var.named_ports == null ? {} : var.named_ports iterator = config content { @@ -239,7 +239,7 @@ resource "google_compute_region_instance_group_manager" "default" { instance_template = var.default_version.instance_template name = var.default_version.name } - dynamic version { + dynamic "version" { for_each = var.versions == null ? {} : var.versions iterator = version content { @@ -279,7 +279,7 @@ resource "google_compute_health_check" "http" { response = try(var.health_check_config.check.response, null) } - dynamic log_config { + dynamic "log_config" { for_each = try(var.health_check_config.logging, false) ? [""] : [] content { enable = true @@ -309,7 +309,7 @@ resource "google_compute_health_check" "https" { response = try(var.health_check_config.check.response, null) } - dynamic log_config { + dynamic "log_config" { for_each = try(var.health_check_config.logging, false) ? [""] : [] content { enable = true @@ -338,7 +338,7 @@ resource "google_compute_health_check" "tcp" { response = try(var.health_check_config.check.response, null) } - dynamic log_config { + dynamic "log_config" { for_each = try(var.health_check_config.logging, false) ? [""] : [] content { enable = true @@ -367,7 +367,7 @@ resource "google_compute_health_check" "ssl" { response = try(var.health_check_config.check.response, null) } - dynamic log_config { + dynamic "log_config" { for_each = try(var.health_check_config.logging, false) ? [""] : [] content { enable = true @@ -397,7 +397,7 @@ resource "google_compute_health_check" "http2" { response = try(var.health_check_config.check.response, null) } - dynamic log_config { + dynamic "log_config" { for_each = try(var.health_check_config.logging, false) ? [""] : [] content { enable = true diff --git a/modules/datafusion/outputs.tf b/modules/datafusion/outputs.tf index 06023210d..9c626a72c 100644 --- a/modules/datafusion/outputs.tf +++ b/modules/datafusion/outputs.tf @@ -21,7 +21,7 @@ output "id" { output "ip_allocation" { description = "IP range reserved for Data Fusion instance in case of a private instance." - value = "${local.ip_allocation}" + value = local.ip_allocation } output "resource" { diff --git a/modules/dns/main.tf b/modules/dns/main.tf index 8957b970f..738794c23 100644 --- a/modules/dns/main.tf +++ b/modules/dns/main.tf @@ -115,7 +115,7 @@ resource "google_dns_managed_zone" "public" { visibility = "public" dynamic "dnssec_config" { - for_each = var.dnssec_config == {} ? [] : list(var.dnssec_config) + for_each = var.dnssec_config == {} ? [] : tolist([var.dnssec_config]) iterator = config content { kind = lookup(config.value, "kind", "dns#managedZoneDnsSecConfig") diff --git a/modules/endpoints/outputs.tf b/modules/endpoints/outputs.tf index 181e15fed..7878d026a 100644 --- a/modules/endpoints/outputs.tf +++ b/modules/endpoints/outputs.tf @@ -26,5 +26,5 @@ output "endpoints_service" { output "endpoints" { description = "A list of Endpoint objects." - value = google_endpoints_service.default.endpoints + value = google_endpoints_service.default.endpoints } diff --git a/modules/folders-unit/locals.tf b/modules/folders-unit/locals.tf index c7f4e2239..a275da1cc 100644 --- a/modules/folders-unit/locals.tf +++ b/modules/folders-unit/locals.tf @@ -40,7 +40,7 @@ locals { var.iam_billing_config.target_org ? [] : ["roles/billing.user"] ) sa_billing_org_roles = ( - ! var.iam_billing_config.target_org ? [] : ["roles/billing.user"] + !var.iam_billing_config.target_org ? [] : ["roles/billing.user"] ) sa_xpn_folder_roles = ( local.sa_xpn_target_org ? [] : ["roles/compute.xpnAdmin"] diff --git a/modules/gcs/README.md b/modules/gcs/README.md index ff27a5b63..095bb1059 100644 --- a/modules/gcs/README.md +++ b/modules/gcs/README.md @@ -60,6 +60,40 @@ module "bucket" { # tftest:modules=1:resources=2 ``` +### Example with lifecycle rule + +```hcl +module "bucket" { + source = "./modules/gcs" + project_id = "myproject" + prefix = "test" + name = "my-bucket" + + iam = { + "roles/storage.admin" = ["group:storage@example.com"] + } + + lifecycle_rule = { + action = { + type = "SetStorageClass" + storage_class = "STANDARD" + } + condition = { + age = 30 + created_before = null + with_state = null + matches_storage_class = null + num_newer_versions = null + custom_time_before = null + days_since_custom_time = null + days_since_noncurrent_time = null + noncurrent_time_before = null + } + } +} +# tftest:modules=1:resources=2 +``` + ## Variables @@ -72,6 +106,7 @@ module "bucket" { | *force_destroy* | Optional map to set force destroy keyed by name, defaults to false. | bool | | false | | *iam* | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | *labels* | Labels to be attached to all buckets. | map(string) | | {} | +| *lifecycle_rule* | Bucket lifecycle rule | object({...}) | | null | | *location* | Bucket location. | string | | EU | | *logging_config* | Bucket logging configuration. | object({...}) | | null | | *prefix* | Prefix used to generate the bucket name. | string | | null | diff --git a/modules/gcs/main.tf b/modules/gcs/main.tf index 0b86f770b..26fb66277 100644 --- a/modules/gcs/main.tf +++ b/modules/gcs/main.tf @@ -38,7 +38,7 @@ resource "google_storage_bucket" "bucket" { storage_class = lower(var.storage_class) }) - dynamic encryption { + dynamic "encryption" { for_each = var.encryption_key == null ? [] : [""] content { @@ -46,7 +46,7 @@ resource "google_storage_bucket" "bucket" { } } - dynamic retention_policy { + dynamic "retention_policy" { for_each = var.retention_policy == null ? [] : [""] content { retention_period = var.retention_policy.retention_period @@ -54,7 +54,7 @@ resource "google_storage_bucket" "bucket" { } } - dynamic logging { + dynamic "logging" { for_each = var.logging_config == null ? [] : [""] content { log_bucket = var.logging_config.log_bucket @@ -62,7 +62,7 @@ resource "google_storage_bucket" "bucket" { } } - dynamic cors { + dynamic "cors" { for_each = var.cors == null ? [] : [""] content { origin = var.cors.origin @@ -71,6 +71,27 @@ resource "google_storage_bucket" "bucket" { max_age_seconds = max(3600, var.cors.max_age_seconds) } } + + dynamic "lifecycle_rule" { + for_each = var.lifecycle_rule == null ? [] : [""] + content { + action { + type = var.lifecycle_rule.action["type"] + storage_class = var.lifecycle_rule.action["storage_class"] + } + condition { + age = var.lifecycle_rule.condition["age"] + created_before = var.lifecycle_rule.condition["created_before"] + with_state = var.lifecycle_rule.condition["with_state"] + matches_storage_class = var.lifecycle_rule.condition["matches_storage_class"] + num_newer_versions = var.lifecycle_rule.condition["num_newer_versions"] + custom_time_before = var.lifecycle_rule.condition["custom_time_before"] + days_since_custom_time = var.lifecycle_rule.condition["days_since_custom_time"] + days_since_noncurrent_time = var.lifecycle_rule.condition["days_since_noncurrent_time"] + noncurrent_time_before = var.lifecycle_rule.condition["noncurrent_time_before"] + } + } + } } resource "google_storage_bucket_iam_binding" "bindings" { diff --git a/modules/gcs/variables.tf b/modules/gcs/variables.tf index 268acfb35..cfb5e573f 100644 --- a/modules/gcs/variables.tf +++ b/modules/gcs/variables.tf @@ -110,3 +110,25 @@ variable "cors" { }) default = null } + +variable "lifecycle_rule" { + description = "Bucket lifecycle rule" + type = object({ + action = object({ + type = string + storage_class = string + }) + condition = object({ + age = number + created_before = string + with_state = string + matches_storage_class = list(string) + num_newer_versions = string + custom_time_before = string + days_since_custom_time = string + days_since_noncurrent_time = string + noncurrent_time_before = string + }) + }) + default = null +} diff --git a/modules/gke-nodepool/main.tf b/modules/gke-nodepool/main.tf index 750d606c6..ad6591dd5 100644 --- a/modules/gke-nodepool/main.tf +++ b/modules/gke-nodepool/main.tf @@ -91,7 +91,7 @@ resource "google_container_node_pool" "nodepool" { tags = var.node_tags boot_disk_kms_key = var.node_boot_disk_kms_key - dynamic guest_accelerator { + dynamic "guest_accelerator" { for_each = var.node_guest_accelerator iterator = config content { @@ -100,7 +100,7 @@ resource "google_container_node_pool" "nodepool" { } } - dynamic sandbox_config { + dynamic "sandbox_config" { for_each = ( var.node_sandbox_config != null ? [var.node_sandbox_config] @@ -112,7 +112,7 @@ resource "google_container_node_pool" "nodepool" { } } - dynamic shielded_instance_config { + dynamic "shielded_instance_config" { for_each = ( var.node_shielded_instance_config != null ? [var.node_shielded_instance_config] @@ -131,7 +131,7 @@ resource "google_container_node_pool" "nodepool" { } - dynamic autoscaling { + dynamic "autoscaling" { for_each = var.autoscaling_config != null ? [var.autoscaling_config] : [] iterator = config content { @@ -140,7 +140,7 @@ resource "google_container_node_pool" "nodepool" { } } - dynamic management { + dynamic "management" { for_each = var.management_config != null ? [var.management_config] : [] iterator = config content { @@ -149,7 +149,7 @@ resource "google_container_node_pool" "nodepool" { } } - dynamic upgrade_settings { + dynamic "upgrade_settings" { for_each = var.upgrade_config != null ? [var.upgrade_config] : [] iterator = config content { diff --git a/modules/iam-service-account/main.tf b/modules/iam-service-account/main.tf index 2095ec5a1..e5f4bb74c 100644 --- a/modules/iam-service-account/main.tf +++ b/modules/iam-service-account/main.tf @@ -56,8 +56,10 @@ locals { ? google_service_account_key.key["1"] : map("", null) , {}) - prefix = var.prefix != null ? "${var.prefix}-" : "" - resource_iam_email = "serviceAccount:${local.service_account.email}" + prefix = var.prefix != null ? "${var.prefix}-" : "" + resource_email_static = "${local.prefix}${var.name}@${var.project_id}.iam.gserviceaccount.com" + resource_iam_email_static = "serviceAccount:${local.resource_email_static}" + resource_iam_email = "serviceAccount:${local.service_account.email}" service_account = ( var.service_account_create ? try(google_service_account.service_account.0, null) diff --git a/modules/iam-service-account/outputs.tf b/modules/iam-service-account/outputs.tf index 642cbb89a..9b8f1ff55 100644 --- a/modules/iam-service-account/outputs.tf +++ b/modules/iam-service-account/outputs.tf @@ -21,12 +21,18 @@ output "service_account" { output "email" { description = "Service account email." - value = local.service_account.email + value = local.resource_email_static + depends_on = [ + local.service_account + ] } output "iam_email" { description = "IAM-format service account email." - value = local.resource_iam_email + value = local.resource_iam_email_static + depends_on = [ + local.service_account + ] } output "key" { diff --git a/modules/kms/main.tf b/modules/kms/main.tf index 5c773556f..62ed33c53 100644 --- a/modules/kms/main.tf +++ b/modules/kms/main.tf @@ -64,7 +64,7 @@ resource "google_kms_crypto_key" "default" { rotation_period = try(each.value.rotation_period, null) labels = try(each.value.labels, null) purpose = try(local.key_purpose[each.key].purpose, null) - dynamic version_template { + dynamic "version_template" { for_each = local.key_purpose[each.key].version_template == null ? [] : [""] content { algorithm = local.key_purpose[each.key].version_template.algorithm diff --git a/modules/net-ilb/main.tf b/modules/net-ilb/main.tf index f1382ab02..329a56983 100644 --- a/modules/net-ilb/main.tf +++ b/modules/net-ilb/main.tf @@ -67,7 +67,7 @@ resource "google_compute_region_backend_service" "default" { timeout_sec = try(var.backend_config.timeout_sec, null) connection_draining_timeout_sec = try(var.backend_config.connection_draining_timeout_sec, null) - dynamic backend { + dynamic "backend" { for_each = { for b in var.backends : b.group => b } iterator = backend content { @@ -78,7 +78,7 @@ resource "google_compute_region_backend_service" "default" { } } - dynamic failover_policy { + dynamic "failover_policy" { for_each = var.failover_config == null ? [] : [var.failover_config] iterator = config content { @@ -97,7 +97,7 @@ resource "google_compute_instance_group" "unmanaged" { name = each.key description = "Terraform-managed." instances = each.value.instances - dynamic named_port { + dynamic "named_port" { for_each = each.value.named_ports != null ? each.value.named_ports : {} iterator = config content { @@ -131,7 +131,7 @@ resource "google_compute_health_check" "http" { response = try(var.health_check_config.check.response, null) } - dynamic log_config { + dynamic "log_config" { for_each = try(var.health_check_config.logging, false) ? [""] : [] content { enable = true @@ -163,7 +163,7 @@ resource "google_compute_health_check" "https" { response = try(var.health_check_config.check.response, null) } - dynamic log_config { + dynamic "log_config" { for_each = try(var.health_check_config.logging, false) ? [""] : [] content { enable = true @@ -194,7 +194,7 @@ resource "google_compute_health_check" "tcp" { response = try(var.health_check_config.check.response, null) } - dynamic log_config { + dynamic "log_config" { for_each = try(var.health_check_config.logging, false) ? [""] : [] content { enable = true @@ -225,7 +225,7 @@ resource "google_compute_health_check" "ssl" { response = try(var.health_check_config.check.response, null) } - dynamic log_config { + dynamic "log_config" { for_each = try(var.health_check_config.logging, false) ? [""] : [] content { enable = true @@ -257,7 +257,7 @@ resource "google_compute_health_check" "http2" { response = try(var.health_check_config.check.response, null) } - dynamic log_config { + dynamic "log_config" { for_each = try(var.health_check_config.logging, false) ? [""] : [] content { enable = true diff --git a/modules/net-vpc-firewall-yaml/README.md b/modules/net-vpc-firewall-yaml/README.md index 955080922..2d4ab90e7 100644 --- a/modules/net-vpc-firewall-yaml/README.md +++ b/modules/net-vpc-firewall-yaml/README.md @@ -4,7 +4,7 @@ This module allows creation and management of different types of firewall rules Yaml abstraction for FW rules can simplify users onboarding and also makes rules definition simpler and clearer comparing to HCL. -Nested folder structure for yaml configurations is supported, which allows better and structured code management. +Nested folder structure for yaml configurations is supported, which allows better and structured code management for multiple teams and environments. ## Example @@ -12,20 +12,29 @@ Nested folder structure for yaml configurations is supported, which allows bette ```hcl module "prod-firewall" { - source = "./modules/net-vpc-firewall-yaml" - project_id = "my-prod-project" - network = "my-prod-network" - config_path = "./prod" + source = "./modules/net-vpc-firewall-yaml" + + project_id = "my-prod-project" + network = "my-prod-network" + config_directories = [ + "./prod", + "./common" + ] + log_config = { metadata = "INCLUDE_ALL_METADATA" } } module "dev-firewall" { - source = "./modules/net-vpc-firewall-yaml" - project_id = "my-dev-project" - network = "my-dev-network" - config_path = "./dev" + source = "./modules/net-vpc-firewall-yaml" + + project_id = "my-dev-project" + network = "my-dev-network" + config_directories = [ + "./dev", + "./common" + ] } # tftest:skip ``` @@ -33,9 +42,11 @@ module "dev-firewall" { ### Configuration Structure ```bash +├── common +│ ├── default-egress.yaml +│   ├── lb-rules.yaml +│   └── iap-ingress.yaml ├── dev -│   ├── core -│   │   └── common-rules.yaml │   ├── team-a │   │   ├── databases.yaml │   │   └── webb-app-a.yaml @@ -43,8 +54,6 @@ module "dev-firewall" { │   ├── backend.yaml │   └── frontend.yaml └── prod - ├── core - │   └── common-rules.yaml ├── team-a │   ├── databases.yaml │   └── webb-app-a.yaml @@ -63,7 +72,7 @@ rule-name: # descriptive name, naming convention is adjusted by the module - ports: ['443', '80'] # ports for a specific protocol, keep empty list `[]` for all ports protocol: tcp # protocol, put `all` for any protocol direction: EGRESS # EGRESS or INGRESS - disabled: false # `false` or `true`, FW rule is disabled when `true`, default value is `true` + disabled: false # `false` or `true`, FW rule is disabled when `true`, default value is `false` priority: 1000 # rule priority value, default value is 1000 source_ranges: # list of source ranges, should be specified only for `INGRESS` rule - 0.0.0.0/0 @@ -131,7 +140,7 @@ web-app-a-ingress: | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| config_path | Path to a folder where firewall configs are stored in yaml format. Folder may include subfolders with configuration files. Files suffix must be `.yaml` | string | ✓ | | +| config_directories | List of paths to folders where firewall configs are stored in yaml format. Folder may include subfolders with configuration files. Files suffix must be `.yaml` | list(string) | ✓ | | | network | Name of the network this set of firewall rules applies to. | string | ✓ | | | project_id | Project Id. | string | ✓ | | | *log_config* | Log configuration. Possible values for `metadata` are `EXCLUDE_ALL_METADATA` and `INCLUDE_ALL_METADATA`. Set to `null` for disabling firewall logging. | object({...}) | | null | diff --git a/modules/net-vpc-firewall-yaml/main.tf b/modules/net-vpc-firewall-yaml/main.tf index e401f3b96..ab19b23a7 100644 --- a/modules/net-vpc-firewall-yaml/main.tf +++ b/modules/net-vpc-firewall-yaml/main.tf @@ -15,10 +15,23 @@ */ locals { + firewall_rule_files = flatten( + [ + for config_path in var.config_directories : + concat( + [ + for config_file in fileset("${path.root}/${config_path}", "**/*.yaml") : + "${path.root}/${config_path}/${config_file}" + ] + ) + + ] + ) + firewall_rules = merge( [ - for config_file in fileset("${path.root}/${var.config_path}", "**/*.yaml") : - try(yamldecode(file("${path.root}/${var.config_path}/${config_file}")), {}) + for config_file in local.firewall_rule_files : + try(yamldecode(file(config_file)), {}) ]... ) } diff --git a/modules/net-vpc-firewall-yaml/outputs.tf b/modules/net-vpc-firewall-yaml/outputs.tf index 63c3d0c85..d964b5a99 100644 --- a/modules/net-vpc-firewall-yaml/outputs.tf +++ b/modules/net-vpc-firewall-yaml/outputs.tf @@ -18,7 +18,7 @@ output "ingress_allow_rules" { description = "Ingress rules with allow blocks." value = [ for rule in google_compute_firewall.rules : - rule.name if rule.direction == "INGRESS" && length(rule.allow) > 0 + rule if rule.direction == "INGRESS" && length(rule.allow) > 0 ] } @@ -26,7 +26,7 @@ output "ingress_deny_rules" { description = "Ingress rules with deny blocks." value = [ for rule in google_compute_firewall.rules : - rule.name if rule.direction == "INGRESS" && length(rule.deny) > 0 + rule if rule.direction == "INGRESS" && length(rule.deny) > 0 ] } @@ -34,7 +34,7 @@ output "egress_allow_rules" { description = "Egress rules with allow blocks." value = [ for rule in google_compute_firewall.rules : - rule.name if rule.direction == "EGRESS" && length(rule.allow) > 0 + rule if rule.direction == "EGRESS" && length(rule.allow) > 0 ] } @@ -42,6 +42,6 @@ output "egress_deny_rules" { description = "Egress rules with allow blocks." value = [ for rule in google_compute_firewall.rules : - rule.name if rule.direction == "EGRESS" && length(rule.deny) > 0 + rule if rule.direction == "EGRESS" && length(rule.deny) > 0 ] } diff --git a/modules/net-vpc-firewall-yaml/variables.tf b/modules/net-vpc-firewall-yaml/variables.tf index 0d5d4da31..d54d5a356 100644 --- a/modules/net-vpc-firewall-yaml/variables.tf +++ b/modules/net-vpc-firewall-yaml/variables.tf @@ -24,9 +24,9 @@ variable "project_id" { type = string } -variable "config_path" { - description = "Path to a folder where firewall configs are stored in yaml format. Folder may include subfolders with configuration files. Files suffix must be `.yaml`" - type = string +variable "config_directories" { + description = "List of paths to folders where firewall configs are stored in yaml format. Folder may include subfolders with configuration files. Files suffix must be `.yaml`" + type = list(string) } variable "log_config" { diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md index fb6a6a5d4..6be63a8fc 100644 --- a/modules/net-vpc/README.md +++ b/modules/net-vpc/README.md @@ -74,6 +74,8 @@ module "vpc-spoke-1" { ### Shared VPC +[Shared VPC](https://cloud.google.com/vpc/docs/shared-vpc) is a project-level functionality which enables a project to share its VPCs with other projects. The `shared_vpc_host` variable is here to help with rapid prototyping, we recommend leveraging the project module for production usage. + ```hcl locals { service_project_1 = { diff --git a/modules/net-vpc/versions.tf b/modules/net-vpc/versions.tf index 20c35afe3..04caecd4b 100644 --- a/modules/net-vpc/versions.tf +++ b/modules/net-vpc/versions.tf @@ -16,7 +16,7 @@ terraform { required_version = ">= 0.13.0" - required_providers { - google = ">= 3.45" + required_providers { + google = ">= 3.45" } } diff --git a/modules/net-vpn-ha/main.tf b/modules/net-vpn-ha/main.tf index 9b0016195..cb2ba6877 100644 --- a/modules/net-vpn-ha/main.tf +++ b/modules/net-vpn-ha/main.tf @@ -79,7 +79,7 @@ resource "google_compute_router" "router" { : var.router_advertise_config.groups ) ) - dynamic advertised_ip_ranges { + dynamic "advertised_ip_ranges" { for_each = ( var.router_advertise_config == null ? {} : ( var.router_advertise_config.mode != "CUSTOM" @@ -122,7 +122,7 @@ resource "google_compute_router_peer" "bgp_peer" { : each.value.bgp_peer_options.advertise_groups ) ) - dynamic advertised_ip_ranges { + dynamic "advertised_ip_ranges" { for_each = ( each.value.bgp_peer_options == null ? {} : ( each.value.bgp_peer_options.advertise_mode != "CUSTOM" diff --git a/modules/organization/README.md b/modules/organization/README.md index c912c517f..94e9ff25a 100644 --- a/modules/organization/README.md +++ b/modules/organization/README.md @@ -151,6 +151,23 @@ module "org" { # tftest:modules=5:resources=11 ``` +## Custom Roles +```hcl +module "org" { + source = "./modules/organization" + organization_id = var.organization_id + custom_roles = { + "myRole" = [ + "compute.instances.list", + ] + } + iam = { + (module.org.custom_role_id.myRole) = ["user:me@example.com"] + } +} +# tftest:modules=1:resources=2 +``` + ## Variables @@ -177,6 +194,8 @@ module "org" { | name | description | sensitive | |---|---|:---:| +| custom_role_id | Map of custom role IDs created in the organization. | | +| custom_roles | Map of custom roles resources created in the organization. | | | firewall_policies | Map of firewall policy resources created in the organization. | | | firewall_policy_id | Map of firewall policy ids created in the organization. | | | organization_id | Organization id dependent on module resources. | | diff --git a/modules/organization/outputs.tf b/modules/organization/outputs.tf index 1dd51ee8b..456befb66 100644 --- a/modules/organization/outputs.tf +++ b/modules/organization/outputs.tf @@ -50,3 +50,21 @@ output "sink_writer_identities" { for name, sink in google_logging_organization_sink.sink : name => sink.writer_identity } } + +output "custom_roles" { + description = "Map of custom roles resources created in the organization." + value = google_organization_iam_custom_role.roles +} + +output "custom_role_id" { + description = "Map of custom role IDs created in the organization." + value = { + for role_id, role in google_organization_iam_custom_role.roles : + # build the string manually so that role IDs can be used as map + # keys (useful for folder/organization/project-level iam bindings) + (role_id) => "${var.organization_id}/roles/${role_id}" + } + depends_on = [ + google_organization_iam_custom_role.roles + ] +} diff --git a/modules/project/README.md b/modules/project/README.md index 697f9d25a..03ec1ef95 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -183,6 +183,7 @@ module "project" { | *billing_account* | Billing account id. | string | | null | | *contacts* | List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES | map(list(string)) | | {} | | *custom_roles* | Map of role name => list of permissions to create in this project. | map(list(string)) | | {} | +| *descriptive_name* | Name of the project name. Used for project name instead of `name` variable | string | | null | | *group_iam* | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | | *iam* | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | *iam_additive* | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | diff --git a/modules/project/main.tf b/modules/project/main.tf index e17f6f0a9..d52c8e87e 100644 --- a/modules/project/main.tf +++ b/modules/project/main.tf @@ -15,7 +15,8 @@ */ locals { - group_iam_roles = distinct(flatten(values(var.group_iam))) + descriptive_name = var.descriptive_name != null ? var.descriptive_name : "${local.prefix}${var.name}" + group_iam_roles = distinct(flatten(values(var.group_iam))) group_iam = { for r in local.group_iam_roles : r => [ for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null @@ -75,6 +76,7 @@ locals { ]) } + data "google_project" "project" { count = var.project_create ? 0 : 1 project_id = "${local.prefix}${var.name}" @@ -85,7 +87,7 @@ resource "google_project" "project" { org_id = local.parent_type == "organizations" ? local.parent_id : null folder_id = local.parent_type == "folders" ? local.parent_id : null project_id = "${local.prefix}${var.name}" - name = "${local.prefix}${var.name}" + name = local.descriptive_name billing_account = var.billing_account auto_create_network = var.auto_create_network labels = var.labels diff --git a/modules/project/variables.tf b/modules/project/variables.tf index d4f917b33..a72e4d108 100644 --- a/modules/project/variables.tf +++ b/modules/project/variables.tf @@ -215,3 +215,9 @@ variable "service_perimeter_bridges" { type = list(string) default = null } + +variable "descriptive_name" { + description = "Name of the project name. Used for project name instead of `name` variable" + type = string + default = null +} diff --git a/modules/pubsub/README.md b/modules/pubsub/README.md index 938f27166..0146b53e8 100644 --- a/modules/pubsub/README.md +++ b/modules/pubsub/README.md @@ -95,7 +95,7 @@ module "pubsub" { |---|---|:---: |:---:|:---:| | name | PubSub topic name. | string | ✓ | | | project_id | Project used for resources. | string | ✓ | | -| *dead_letter_configs* | Per-subscription dead letter policy configuration. | map(object({...})) | | {} | +| *dead_letter_configs* | Per-subscription dead letter policy configuration. | map(object({...})) | | {} | | *defaults* | Subscription defaults for options. | object({...}) | | ... | | *iam* | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | *kms_key* | KMS customer managed encryption key. | string | | null | diff --git a/modules/pubsub/main.tf b/modules/pubsub/main.tf index d65756365..f66c73245 100644 --- a/modules/pubsub/main.tf +++ b/modules/pubsub/main.tf @@ -41,7 +41,7 @@ resource "google_pubsub_topic" "default" { kms_key_name = var.kms_key labels = var.labels - dynamic message_storage_policy { + dynamic "message_storage_policy" { for_each = length(var.regions) > 0 ? [var.regions] : [] content { allowed_persistence_regions = var.regions @@ -67,14 +67,14 @@ resource "google_pubsub_subscription" "default" { message_retention_duration = each.value.options.message_retention_duration retain_acked_messages = each.value.options.retain_acked_messages - dynamic expiration_policy { + dynamic "expiration_policy" { for_each = each.value.options.expiration_policy_ttl == null ? [] : [""] content { ttl = each.value.options.expiration_policy_ttl } } - dynamic dead_letter_policy { + dynamic "dead_letter_policy" { for_each = try(var.dead_letter_configs[each.key], null) == null ? [] : [""] content { dead_letter_topic = var.dead_letter_configs[each.key].topic @@ -82,12 +82,12 @@ resource "google_pubsub_subscription" "default" { } } - dynamic push_config { + dynamic "push_config" { for_each = try(var.push_configs[each.key], null) == null ? [] : [""] content { push_endpoint = var.push_configs[each.key].endpoint attributes = var.push_configs[each.key].attributes - dynamic oidc_token { + dynamic "oidc_token" { for_each = ( local.oidc_config[each.key] == null ? [] : [""] ) diff --git a/modules/pubsub/variables.tf b/modules/pubsub/variables.tf index 6657d4354..5dabafa21 100644 --- a/modules/pubsub/variables.tf +++ b/modules/pubsub/variables.tf @@ -17,8 +17,8 @@ variable "dead_letter_configs" { description = "Per-subscription dead letter policy configuration." type = map(object({ - topic = string - max_delivery_attemps = number + topic = string + max_delivery_attempts = number })) default = {} } diff --git a/modules/secret-manager/main.tf b/modules/secret-manager/main.tf index 696331a81..6b6154e6e 100644 --- a/modules/secret-manager/main.tf +++ b/modules/secret-manager/main.tf @@ -42,19 +42,19 @@ resource "google_secret_manager_secret" "default" { secret_id = each.key labels = lookup(var.labels, each.key, null) - dynamic replication { + dynamic "replication" { for_each = each.value == null ? [""] : [] content { automatic = true } } - dynamic replication { + dynamic "replication" { for_each = each.value == null ? [] : [each.value] iterator = locations content { user_managed { - dynamic replicas { + dynamic "replicas" { for_each = locations.value iterator = location content { diff --git a/networking/README.md b/networking/README.md index 5a1489da2..00611913b 100644 --- a/networking/README.md +++ b/networking/README.md @@ -37,7 +37,14 @@ It is meant to be used as a starting point for most Shared VPC configurations, a ### ILB as next hop This [example](./ilb-next-hop/) allows testing [ILB as next hop](https://cloud.google.com/load-balancing/docs/internal/ilb-next-hop-overview) using simple Linux gateway VMS between two VPCs, to emulate virtual appliances. An optional additional ILB can be enabled to test multiple load balancer configurations and hashing. +
### Calling a private Cloud Function from On-premises - This [example](./private-cloud-function-from-onprem/) shows how to invoke a [private Google Cloud Function](https://cloud.google.com/functions/docs/networking/network-settings) from the on-prem environment via a [Private Service Connect endpoint](https://cloud.google.com/vpc/docs/private-service-connect#benefits-apis). \ No newline at end of file + This [example](./private-cloud-function-from-onprem/) shows how to invoke a [private Google Cloud Function](https://cloud.google.com/functions/docs/networking/network-settings) from the on-prem environment via a [Private Service Connect endpoint](https://cloud.google.com/vpc/docs/private-service-connect#benefits-apis). +
+ +### Decentralized firewall management + + This [example](./decentralized-firewall/) shows how a decentralized firewall management can be organized using [firewall-yaml](../modules/net-vpc-firewall-yaml) module. +
diff --git a/networking/decentralized-firewall/README.md b/networking/decentralized-firewall/README.md new file mode 100644 index 000000000..8a4c0066e --- /dev/null +++ b/networking/decentralized-firewall/README.md @@ -0,0 +1,33 @@ +# Decentralized firewall management + +This sample shows how a decentralized firewall management can be organized using the [firewall-yaml](../../modules/net-vpc-firewall-yaml) module. + +This approach is a good fit when Shared VPCs are used across multiple application/infrastructure teams. A central repository keeps environment/team specific folders with firewall definitions in `yaml` format. + +In the current example multiple teams can define their [VPC Firewall Rules](https://cloud.google.com/vpc/docs/firewalls) for [dev](./firewall/dev) and [prod](./firewall/prod) environments using team specific subfolders. Rules defined in the [common](./firewall/common) folder are applied to both dev and prod environments. +> **_NOTE:_** Common rules are meant to be used for situations where [hierarchical rules](https://cloud.google.com/vpc/docs/firewall-policies) do not map precisely to requirements (e.g. SA, etc.) + +This is the high level diagram: + +![High-level diagram](diagram.png "High-level diagram") + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| billing_account_id | Billing account id used as default for new projects. | string | ✓ | | +| prefix | Prefix used for resources that need unique names. | string | ✓ | | +| root_node | Hierarchy node where projects will be created, 'organizations/org_id' or 'folders/folder_id'. | string | ✓ | | +| *ip_ranges* | Subnet IP CIDR ranges. | map(string) | | ... | +| *project_services* | Service APIs enabled by default in new projects. | list(string) | | ... | +| *region* | Region used. | string | | europe-west1 | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| fw_rules | Firewall rules. | | +| projects | Project ids. | | +| vpc | Shared VPCs. | | + diff --git a/networking/decentralized-firewall/backend.tf.sample b/networking/decentralized-firewall/backend.tf.sample new file mode 100644 index 000000000..99f84b17c --- /dev/null +++ b/networking/decentralized-firewall/backend.tf.sample @@ -0,0 +1,20 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +terraform { + backend "gcs" { + bucket = "" + } +} diff --git a/networking/decentralized-firewall/diagram.png b/networking/decentralized-firewall/diagram.png new file mode 100644 index 000000000..bf655309e Binary files /dev/null and b/networking/decentralized-firewall/diagram.png differ diff --git a/networking/decentralized-firewall/firewall/common/common-egress.yaml b/networking/decentralized-firewall/firewall/common/common-egress.yaml new file mode 100644 index 000000000..716c1498b --- /dev/null +++ b/networking/decentralized-firewall/firewall/common/common-egress.yaml @@ -0,0 +1,43 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Deny all egress (egress traffic is allowed by default) +deny-all: + deny: + - ports: [] + protocol: all + direction: EGRESS + priority: 65535 + destination_ranges: + - 0.0.0.0/0 + +# Allow access to GCP APIs via Private Google Access +# https://cloud.google.com/vpc/docs/access-apis-external-ip#config +gcp-pga-apis: + allow: + - ports: [443] + protocol: tcp + direction: EGRESS + priority: 500 + destination_ranges: + - 199.36.153.8/30 + +# Allow egress to internal networks +internal-egress: + allow: + - ports: [] + protocol: tcp + direction: EGRESS + destination_ranges: + - 10.0.0.0/16 diff --git a/networking/decentralized-firewall/firewall/common/iap-access.yaml b/networking/decentralized-firewall/firewall/common/iap-access.yaml new file mode 100644 index 000000000..931a180e4 --- /dev/null +++ b/networking/decentralized-firewall/firewall/common/iap-access.yaml @@ -0,0 +1,24 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Access via SSH from IAP to all instancess https://cloud.google.com/iap/docs/using-tcp-forwarding#create-firewall-rule +iap-ssh-access: + allow: + - ports: [22] + protocol: tcp + direction: INGRESS + priority: 1001 + source_ranges: + - 35.235.240.0/20 + \ No newline at end of file diff --git a/networking/decentralized-firewall/firewall/common/lb-access.yaml b/networking/decentralized-firewall/firewall/common/lb-access.yaml new file mode 100644 index 000000000..975d3ca05 --- /dev/null +++ b/networking/decentralized-firewall/firewall/common/lb-access.yaml @@ -0,0 +1,24 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Access from GCP LBs https://cloud.google.com/load-balancing/docs/https/#firewall_rules +lb-health-checks: + allow: + - ports: [] + protocol: tcp + direction: INGRESS + priority: 1001 + source_ranges: + - 35.191.0.0/16 + - 130.211.0.0/22 diff --git a/networking/decentralized-firewall/firewall/dev/app-1/app1-rules.yaml b/networking/decentralized-firewall/firewall/dev/app-1/app1-rules.yaml new file mode 100644 index 000000000..9a26650be --- /dev/null +++ b/networking/decentralized-firewall/firewall/dev/app-1/app1-rules.yaml @@ -0,0 +1,31 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Allow traffic from the frontend VMs +app1-backend: + allow: + - ports: ['443', '80'] + protocol: tcp + direction: INGRESS + source_tags: ['app1-frontend'] + target_tags: ['app1-backend'] + +# Allow traffic to MySQL Servers from App1 backend +app1-db: + allow: + - ports: ['3306'] + protocol: tcp + direction: INGRESS + source_tags: ['app1-backend'] + target_tags: ['mysql-server'] diff --git a/networking/decentralized-firewall/firewall/dev/app-2/app2-rules.yaml b/networking/decentralized-firewall/firewall/dev/app-2/app2-rules.yaml new file mode 100644 index 000000000..d7b79b636 --- /dev/null +++ b/networking/decentralized-firewall/firewall/dev/app-2/app2-rules.yaml @@ -0,0 +1,31 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Allow traffic from app1 frontend +app2-backend: + allow: + - ports: ['443', '80'] + protocol: tcp + direction: INGRESS + source_tags: ['app1-frontend'] + target_tags: ['app2-backend'] + +# Allow traffic to MySQL servers from App2 backend +app2-db: + allow: + - ports: ['3306'] + protocol: tcp + direction: INGRESS + source_tags: ['app2-backend'] + target_tags: ['mysql-server'] diff --git a/networking/decentralized-firewall/firewall/prod/app-1/app1-rules.yaml b/networking/decentralized-firewall/firewall/prod/app-1/app1-rules.yaml new file mode 100644 index 000000000..9a26650be --- /dev/null +++ b/networking/decentralized-firewall/firewall/prod/app-1/app1-rules.yaml @@ -0,0 +1,31 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Allow traffic from the frontend VMs +app1-backend: + allow: + - ports: ['443', '80'] + protocol: tcp + direction: INGRESS + source_tags: ['app1-frontend'] + target_tags: ['app1-backend'] + +# Allow traffic to MySQL Servers from App1 backend +app1-db: + allow: + - ports: ['3306'] + protocol: tcp + direction: INGRESS + source_tags: ['app1-backend'] + target_tags: ['mysql-server'] diff --git a/networking/decentralized-firewall/main.tf b/networking/decentralized-firewall/main.tf new file mode 100644 index 000000000..2502d41f6 --- /dev/null +++ b/networking/decentralized-firewall/main.tf @@ -0,0 +1,136 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############################################################################### +# Shared VPC Host projects # +############################################################################### + +module "project-host-prod" { + source = "../../modules/project" + parent = var.root_node + billing_account = var.billing_account_id + prefix = var.prefix + name = "prod-host" + services = var.project_services + + shared_vpc_host_config = { + enabled = true + service_projects = [] + } +} + +module "project-host-dev" { + source = "../../modules/project" + parent = var.root_node + billing_account = var.billing_account_id + prefix = var.prefix + name = "dev-host" + services = var.project_services + + shared_vpc_host_config = { + enabled = true + service_projects = [] + } +} + +################################################################################ +# Networking # +################################################################################ + +module "vpc-prod" { + source = "../../modules/net-vpc" + project_id = module.project-host-prod.project_id + name = "prod-vpc" + subnets = [ + { + ip_cidr_range = var.ip_ranges.prod + name = "prod" + region = var.region + secondary_ip_range = {} + } + ] +} + +module "vpc-dev" { + source = "../../modules/net-vpc" + project_id = module.project-host-dev.project_id + name = "dev-vpc" + subnets = [ + { + ip_cidr_range = var.ip_ranges.dev + name = "dev" + region = var.region + secondary_ip_range = {} + } + ] +} + +############################################################################### +# Private Google Access DNS # +############################################################################### + +module "dns-api-prod" { + source = "../../modules/dns" + project_id = module.project-host-prod.project_id + type = "private" + name = "googleapis" + domain = "googleapis.com." + client_networks = [module.vpc-prod.self_link] + recordsets = [ + { name = "*", type = "CNAME", ttl = 300, records = ["private.googleapis.com."] }, + ] +} + +module "dns-api-dev" { + source = "../../modules/dns" + project_id = module.project-host-dev.project_id + type = "private" + name = "googleapis" + domain = "googleapis.com." + client_networks = [module.vpc-dev.self_link] + recordsets = [ + { name = "*", type = "CNAME", ttl = 300, records = ["private.googleapis.com."] }, + ] +} + +############################################################################### +# Distributed Firewall # +############################################################################### + +module "vpc-firewall-prod" { + source = "../../modules/net-vpc-firewall-yaml" + + project_id = module.project-host-prod.project_id + network = module.vpc-prod.name + config_directories = [ + "${path.module}/firewall/common", + "${path.module}/firewall/prod" + ] + + # Enable Firewall Logging for the production fwl rules + log_config = { + metadata = "INCLUDE_ALL_METADATA" + } +} + +module "vpc-firewall-dev" { + source = "../../modules/net-vpc-firewall-yaml" + + project_id = module.project-host-dev.project_id + network = module.vpc-dev.name + config_directories = [ + "${path.module}/firewall/common", + "${path.module}/firewall/dev" + ] +} diff --git a/networking/decentralized-firewall/outputs.tf b/networking/decentralized-firewall/outputs.tf new file mode 100644 index 000000000..f744821f5 --- /dev/null +++ b/networking/decentralized-firewall/outputs.tf @@ -0,0 +1,53 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "projects" { + description = "Project ids." + value = { + prod-host = module.project-host-prod.project_id + dev-host = module.project-host-dev.project_id + } +} + +output "vpc" { + description = "Shared VPCs." + value = { + prod = { + name = module.vpc-prod.name + subnets = module.vpc-prod.subnet_ips + } + dev = { + name = module.vpc-dev.name + subnets = module.vpc-dev.subnet_ips + } + } +} + +output "fw_rules" { + description = "Firewall rules." + value = { + prod = { + ingress_allow_rules = module.vpc-firewall-prod.ingress_allow_rules + ingress_deny_rules = module.vpc-firewall-prod.ingress_deny_rules + egress_allow_rules = module.vpc-firewall-prod.egress_allow_rules + egress_deny_rules = module.vpc-firewall-prod.egress_deny_rules + } + dev = { + ingress_allow_rules = module.vpc-firewall-dev.ingress_allow_rules + ingress_deny_rules = module.vpc-firewall-dev.ingress_deny_rules + egress_allow_rules = module.vpc-firewall-dev.egress_allow_rules + egress_deny_rules = module.vpc-firewall-dev.egress_deny_rules + } + } +} diff --git a/networking/decentralized-firewall/validator/Dockerfile b/networking/decentralized-firewall/validator/Dockerfile new file mode 100644 index 000000000..be4b22b47 --- /dev/null +++ b/networking/decentralized-firewall/validator/Dockerfile @@ -0,0 +1,29 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +FROM python:3.9-slim + +RUN mkdir /validator +COPY requirements.txt /validator/requirements.txt +RUN pip install -r /validator/requirements.txt +COPY validator.py /validator/validator.py + +RUN mkdir /schemas +COPY firewallSchema.yaml /schemas/firewallSchema.yaml +COPY firewallSchemaAutoApprove.yaml /schemas/firewallAutoApprove.yaml +COPY firewallSchemaSettings.yaml /schemas/firewallSchemaSettings.yaml + +RUN mkdir /rules + +CMD ["/rules/**/*.yaml"] +ENTRYPOINT ["python3", "/validator/validator.py"] \ No newline at end of file diff --git a/networking/decentralized-firewall/validator/README.md b/networking/decentralized-firewall/validator/README.md new file mode 100644 index 000000000..fd5880370 --- /dev/null +++ b/networking/decentralized-firewall/validator/README.md @@ -0,0 +1,80 @@ +# Decentralized firewall validator + +The decentralized firewall validator is a Python scripts that utilizes [Yamale](https://github.com/23andMe/Yamale) schema +validation library to validate the configured firewall rules. + +## Configuring schemas + +There are three configuration files: +- [firewallSchema.yaml](firewallSchema.yaml), where the basic validation schema is configured +- [firewallSchemaAutoApprove.yaml](firewallSchemaAutoApprove.yaml), where the a different schema for auto-approval + can be configured (in case more validation is required than what is available in the schema settings) +- [firewallSchemaSettings.yaml](firewallSchemaSettings.yaml), configures list of allowed and approved + source and destination ranges, ports, network tags and service accounts. + +## Building the container + +You can build the container like this: + +```sh +docker build -t eu.gcr.io/YOUR-PROJECT/firewall-validator:latest . +docker push eu.gcr.io/YOUR-PROJECT/firewall-validator:latest +``` + +## Running the validator + +Example: + +```sh +docker run -v $(pwd)/firewall:/rules/ -t eu.gcr.io/YOUR-PROJECT/firewall-validator:latest +``` + +Output is JSON with keys `ok` and `errors` (if any were found). + +## Using as a GitHub action + +An `action.yml` is provided for this validator to be used as a GitHub action. + +Example of being used in a pipeline: + +```yaml + - uses: actions/checkout@v2 + + - name: Get changed files + if: ${{ github.event_name == 'pull_request' }} + id: changed-files + uses: tj-actions/changed-files@v1.1.2 + + - uses: ./.github/actions/validate-firewall + if: ${{ github.event_name == 'pull_request' }} + id: validation + with: + files: ${{ steps.changed-files.outputs.all_modified_files }} + + - uses: actions/github-script@v3 + if: ${{ github.event_name == 'pull_request' && steps.validation.outputs.ok != 'true' }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + var comments = []; + var errors = JSON.parse(process.env.ERRORS); + for (const filename in errors) { + var fn = filename.replace('/github/workspace/', ''); + comments.push({ + path: fn, + body: "```\n" + errors[filename].join("\n") + "\n```\n", + position: 1, + }); + } + github.pulls.createReview({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + event: "REQUEST_CHANGES", + body: "Firewall rule validation failed.", + comments: comments, + }); + core.setFailed("Firewall validation failed"); + env: + ERRORS: '${{ steps.validation.outputs.errors }}' +``` diff --git a/networking/decentralized-firewall/validator/action.yml b/networking/decentralized-firewall/validator/action.yml new file mode 100644 index 000000000..d6e6177c5 --- /dev/null +++ b/networking/decentralized-firewall/validator/action.yml @@ -0,0 +1,44 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: 'Validate firewall rules' +description: 'Validate firewall rule YAML files' +inputs: + files: + description: 'Files to scan (supports wildcards)' + required: false + default: '/github/workspace/firewall/**/*.yaml' + mode: + description: 'Mode (validate or approve)' + required: false + default: 'validate' + schema: + description: 'Schema' + required: false + default: '/schemas/firewallSchema.yaml' +outputs: + ok: + description: 'Validation successful' + errors: + description: 'Validation results' +runs: + using: 'docker' + image: 'Dockerfile' + args: + - ${{ inputs.files }} + - "--mode" + - ${{ inputs.mode }} + - "--schema" + - ${{ inputs.schema }} + - "--github" \ No newline at end of file diff --git a/networking/decentralized-firewall/validator/firewallSchema.yaml b/networking/decentralized-firewall/validator/firewallSchema.yaml new file mode 100644 index 000000000..697d982a4 --- /dev/null +++ b/networking/decentralized-firewall/validator/firewallSchema.yaml @@ -0,0 +1,32 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +map(include('rule'), key=str(min=3, max=30)) +--- +rule: + disabled: bool(required=False) + deny: list(include('trafficSpec'), required=False) + allow: list(include('trafficSpec'), required=False) + direction: enum('ingress', 'INGRESS', 'egress', 'EGRESS') + priority: int(min=1, max=65535, required=False) + destination_ranges: list(netmask(type='destination'), max=256, required=False) + source_ranges: list(netmask(type='source'), max=256, required=False) + source_tags: list(networktag(), max=30, required=False) + target_tags: list(networktag(), max=70, required=False) + source_service_accounts: list(serviceaccount(), max=10, required=False) + target_service_account: list(serviceaccount(), max=10, required=False) +--- +trafficSpec: + ports: list(networkports()) + protocol: enum('all', 'tcp', 'udp', 'icmp', 'esp', 'ah', 'ipip', 'sctp') diff --git a/networking/decentralized-firewall/validator/firewallSchemaAutoApprove.yaml b/networking/decentralized-firewall/validator/firewallSchemaAutoApprove.yaml new file mode 100644 index 000000000..a5a425f3b --- /dev/null +++ b/networking/decentralized-firewall/validator/firewallSchemaAutoApprove.yaml @@ -0,0 +1,42 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +map(include('ingress'), include('egress'), key=str(min=3, max=30)) +--- +ingress: + disabled: bool(required=False) + deny: list(include('trafficSpec'), required=False) + allow: list(include('trafficSpec'), required=False) + direction: enum('ingress', 'INGRESS') + priority: int(min=1, max=65535, required=False) + source_ranges: list(netmask(type='source'), max=256, required=False) + source_tags: list(networktag(), max=30, required=False) + target_tags: list(networktag(), max=70, required=False) + source_service_accounts: list(serviceaccount(), max=10, required=False) + target_service_account: list(serviceaccount(), max=10, required=False) +--- +egress: + disabled: bool(required=False) + deny: list(include('trafficSpec'), required=False) + allow: list(include('trafficSpec'), required=False) + direction: enum('egress', 'EGRESS') + priority: int(min=1, max=65535, required=False) + destination_ranges: list(netmask(type='destination'), max=256, required=False) + source_tags: list(networktag(), max=30, required=False) + target_tags: list(networktag(), max=70, required=False) + source_service_accounts: list(serviceaccount(), max=10, required=False) + target_service_account: list(serviceaccount(), max=10, required=False) +--- +trafficSpec: + ports: list() + protocol: enum('all', 'tcp', 'udp', 'icmp', 'esp', 'ah', 'ipip', 'sctp') diff --git a/networking/decentralized-firewall/validator/firewallSchemaSettings.yaml b/networking/decentralized-firewall/validator/firewallSchemaSettings.yaml new file mode 100644 index 000000000..77c5ec65f --- /dev/null +++ b/networking/decentralized-firewall/validator/firewallSchemaSettings.yaml @@ -0,0 +1,49 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +allowedPorts: +- ports: 22 # SSH + approved: false +- ports: 80 # HTTP + approved: true +- ports: 443 # HTTPS + approved: true +- ports: 3306 # MySQL + approved: false +- ports: 8000-8999 + approved: true + +allowedSourceRanges: +- cidr: 10.0.0.0/8 # Example on-premise range + approved: true +- cidr: 35.191.0.0/16 # Load balancing & health checks + approved: true +- cidr: 130.211.0.0/22 # Load balancing & health checks + approved: false +- cidr: 35.235.240.0/20 # IAP source range + approved: true + +allowedDestinationRanges: +- cidr: 10.0.0.0/8 + approved: true +- cidr: 0.0.0.0/0 + approved: false + +allowedNetworkTags: +- tag: '*' + approved: true + +allowedServiceAccounts: +- serviceAccount: '*' + approved: true \ No newline at end of file diff --git a/networking/decentralized-firewall/validator/requirements.txt b/networking/decentralized-firewall/validator/requirements.txt new file mode 100644 index 000000000..05fa91c40 --- /dev/null +++ b/networking/decentralized-firewall/validator/requirements.txt @@ -0,0 +1,16 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +yamale~=3.0.0 +PyYAML~=5.4.0 +click~=7.1.0 \ No newline at end of file diff --git a/networking/decentralized-firewall/validator/validator.py b/networking/decentralized-firewall/validator/validator.py new file mode 100644 index 000000000..ca625882f --- /dev/null +++ b/networking/decentralized-firewall/validator/validator.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import ipaddress +import json +import sys + +import click +import yaml +import yamale + +from fnmatch import fnmatch +from types import SimpleNamespace +from yamale.validators import DefaultValidators, Validator + + +class Netmask(Validator): + """ Custom netmask validator """ + tag = 'netmask' + settings = {} + mode = None + _type = None + + def __init__(self, *args, **kwargs): + self._type = kwargs.pop('type', 'source-or-dest') + super().__init__(*args, **kwargs) + + def fail(self, value): + dir_str = 'source or destination' + mode_str = 'allowed' + if self._type == 'source': + dir_str = 'source' + elif self._type == 'destination': + dir_str = 'destination' + if self.mode == 'approve': + mode_str = 'automatically approved' + return '\'%s\' is not an %s %s network.' % (value, mode_str, dir_str) + + def _is_valid(self, value): + is_ok = False + network = ipaddress.ip_network(value) + if self._type == 'source' or self._type == 'source-or-dest': + for ip_range in self.settings['allowedSourceRanges']: + allowed_network = ipaddress.ip_network(ip_range['cidr']) + if network.subnet_of(allowed_network): + if self.mode != 'approve' or ip_range['approved']: + is_ok = True + break + if self._type == 'destination' or self._type == 'source-or-dest': + for ip_range in self.settings['allowedDestinationRanges']: + allowed_network = ipaddress.ip_network(ip_range['cidr']) + if network.subnet_of(allowed_network): + if self.mode != 'approve' or ip_range['approved']: + is_ok = True + break + + return is_ok + + +class NetworkTag(Validator): + """ Custom network tag validator """ + tag = 'networktag' + settings = {} + mode = None + _type = None + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def fail(self, value): + mode_str = 'allowed' + if self.mode == 'approve': + mode_str = 'automatically approved' + return '\'%s\' is not an %s network tag.' % (value, mode_str) + + def _is_valid(self, value): + is_ok = False + for tag in self.settings['allowedNetworkTags']: + if fnmatch(value, tag['tag']): + if self.mode != 'approve' or tag['approved']: + is_ok = True + break + return is_ok + + +class ServiceAccount(Validator): + """ Custom service account validator """ + tag = 'serviceaccount' + settings = {} + mode = None + _type = None + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def fail(self, value): + mode_str = 'allowed' + if self.mode == 'approve': + mode_str = 'automatically approved' + return '\'%s\' is not an %s service account.' % (value, mode_str) + + def _is_valid(self, value): + is_ok = False + for sa in self.settings['allowedServiceAccounts']: + if fnmatch(value, sa['serviceAccount']): + if self.mode != 'approve' or sa['approved']: + is_ok = True + break + return is_ok + + +class NetworkPorts(Validator): + """ Custom ports validator """ + tag = 'networkports' + settings = {} + mode = None + _type = None + allowed_port_map = [] + approved_port_map = [] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + for port in self.settings['allowedPorts']: + ports = self._process_port_definition(port['ports']) + self.allowed_port_map.extend(ports) + if port['approved']: + self.approved_port_map.extend(ports) + + def _process_port_definition(self, port_definition): + ports = [] + if not isinstance(port_definition, int) and '-' in port_definition: + start, end = port_definition.split('-', 2) + for port in range(int(start), int(end) + 1): + ports.append(int(port)) + else: + ports.append(int(port_definition)) + return ports + + def fail(self, value): + mode_str = 'allowed' + if self.mode == 'approve': + mode_str = 'automatically approved' + return '\'%s\' is not an %s IP port.' % (value, mode_str) + + def _is_valid(self, value): + ports = self._process_port_definition(value) + is_ok = True + for port in ports: + if self.mode == 'approve' and port not in self.approved_port_map: + is_ok = False + break + elif port not in self.allowed_port_map: + is_ok = False + break + + return is_ok + + +class FirewallValidator: + schema = None + settings = None + validators = None + + def __init__(self, settings, mode): + self.settings = settings + + self.validators = DefaultValidators.copy() + Netmask.settings = self.settings + Netmask.mode = mode + self.validators[Netmask.tag] = Netmask + + NetworkTag.settings = self.settings + NetworkTag.mode = mode + self.validators[NetworkTag.tag] = NetworkTag + + ServiceAccount.settings = self.settings + ServiceAccount.mode = mode + self.validators[ServiceAccount.tag] = ServiceAccount + + NetworkPorts.settings = self.settings + NetworkPorts.mode = mode + self.validators[NetworkPorts.tag] = NetworkPorts + + def set_schema_from_file(self, schema): + self.schema = yamale.make_schema(path=schema, validators=self.validators) + + def set_schema_from_string(self, schema): + self.schema = yamale.make_schema(content=schema, validators=self.validators) + + def validate_file(self, file): + print('Validating %s...' % (file), file=sys.stderr) + data = yamale.make_data(file) + yamale.validate(self.schema, data) + + +@click.command() +@click.argument('files') +@click.option('--schema', + default='/schemas/firewallSchema.yaml', + help='YAML schema file') +@click.option('--settings', + default='/schemas/firewallSchemaSettings.yaml', + help='schema configuration file') +@click.option('--mode', + default='validate', + help='select mode (validate or approve)') +@click.option('--github', + is_flag=True, + default=False, + help='output GitHub action compatible variables') +def main(**kwargs): + args = SimpleNamespace(**kwargs) + files = [args.files] + if '*' in args.files: + files = glob.glob(args.files, recursive=True) + + print('Arguments: %s' % (str(sys.argv)), file=sys.stderr) + + f = open(args.settings) + settings = yaml.load(f, Loader=yaml.SafeLoader) + + firewall_validator = FirewallValidator(settings, args.mode) + firewall_validator.set_schema_from_file(args.schema) + output = {'ok': True, 'errors': {}} + for file in files: + try: + firewall_validator.validate_file(file) + except yamale.yamale_error.YamaleError as e: + if file not in output['errors']: + output['errors'][file] = [] + output['ok'] = False + for result in e.results: + for err in result.errors: + output['errors'][file].append(err) + + if args.github: + print('::set-output name=ok::%s' % ('true' if output['ok'] else 'false')) + print('::set-output name=errors::%s' % (json.dumps(output['errors']))) + print(json.dumps(output), file=sys.stderr) + else: + print(json.dumps(output)) + if not output['ok'] and not args.github: + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/networking/decentralized-firewall/variables.tf b/networking/decentralized-firewall/variables.tf new file mode 100644 index 000000000..6e71fbc37 --- /dev/null +++ b/networking/decentralized-firewall/variables.tf @@ -0,0 +1,53 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "billing_account_id" { + description = "Billing account id used as default for new projects." + type = string +} + +variable "prefix" { + description = "Prefix used for resources that need unique names." + type = string +} + +variable "region" { + description = "Region used." + type = string + default = "europe-west1" +} + +variable "root_node" { + description = "Hierarchy node where projects will be created, 'organizations/org_id' or 'folders/folder_id'." + type = string +} + +variable "ip_ranges" { + description = "Subnet IP CIDR ranges." + type = map(string) + default = { + prod = "10.0.16.0/24" + dev = "10.0.32.0/24" + } +} + +variable "project_services" { + description = "Service APIs enabled by default in new projects." + type = list(string) + default = [ + "container.googleapis.com", + "dns.googleapis.com", + "stackdriver.googleapis.com", + ] +} diff --git a/networking/onprem-google-access-dns/README.md b/networking/onprem-google-access-dns/README.md index 3e8c19bd3..df679fae7 100644 --- a/networking/onprem-google-access-dns/README.md +++ b/networking/onprem-google-access-dns/README.md @@ -33,7 +33,7 @@ The Cloud DNS inbound policy reserves an IP address in the VPC, which is used by Run this gcloud command to (find out the address assigned to the inbound forwarder)[https://cloud.google.com/dns/docs/policies#list-in-entrypoints]: ```bash -gcloud compute addresses list -project [your project id] +gcloud compute addresses list --project [your project id] ``` In the list of addresses, look for the address with purpose `DNS_RESOLVER` in the subnet `to-onprem-default`. If its IP address is `10.0.0.2` it matches the default value in the Terraform `forwarder_address` variable, which means you're all set. If it's different, proceed to the next step. diff --git a/networking/private-cloud-function-from-onprem/outputs.tf b/networking/private-cloud-function-from-onprem/outputs.tf index 2c52e5809..76a2fc680 100644 --- a/networking/private-cloud-function-from-onprem/outputs.tf +++ b/networking/private-cloud-function-from-onprem/outputs.tf @@ -16,5 +16,5 @@ output "function_url" { description = "URL of the Cloud Function." - value = module.function-hello.function.https_trigger_url + value = module.function-hello.function.https_trigger_url } \ No newline at end of file diff --git a/tests/cloud_operations/scheduled_asset_inventory_export_bq/test_plan.py b/tests/cloud_operations/scheduled_asset_inventory_export_bq/test_plan.py index a80a3ac8d..de94c82d5 100644 --- a/tests/cloud_operations/scheduled_asset_inventory_export_bq/test_plan.py +++ b/tests/cloud_operations/scheduled_asset_inventory_export_bq/test_plan.py @@ -24,4 +24,4 @@ def test_resources(e2e_plan_runner): "Test that plan works and the numbers of resources is as expected." modules, resources = e2e_plan_runner(FIXTURES_DIR) assert len(modules) == 5 - assert len(resources) == 20 + assert len(resources) == 23 diff --git a/tests/modules/net-interconnect-attachment-direct/__init__.py b/tests/modules/apigee_organization/__init__.py similarity index 100% rename from tests/modules/net-interconnect-attachment-direct/__init__.py rename to tests/modules/apigee_organization/__init__.py diff --git a/tests/modules/apigee_organization/fixture/main.tf b/tests/modules/apigee_organization/fixture/main.tf new file mode 100644 index 000000000..e3b60b6d9 --- /dev/null +++ b/tests/modules/apigee_organization/fixture/main.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "test" { + source = "../../../../modules/apigee-organization" + project_id = "my-project" + analytics_region = var.analytics_region + runtime_type = "CLOUD" + authorized_network = var.network + apigee_environments = [ + "eval1", + "eval2" + ] + apigee_envgroups = { + eval = { + environments = [ + "eval1", + "eval2" + ] + hostnames = [ + "eval.api.example.com" + ] + } + } +} diff --git a/tests/modules/apigee_organization/fixture/variables.tf b/tests/modules/apigee_organization/fixture/variables.tf new file mode 100644 index 000000000..3e9109347 --- /dev/null +++ b/tests/modules/apigee_organization/fixture/variables.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "analytics_region" { + type = string + default = "europe-west1" +} + +variable "network" { + type = string + default = "apigee-vpc" +} \ No newline at end of file diff --git a/tests/modules/apigee_organization/test_plan.py b/tests/modules/apigee_organization/test_plan.py new file mode 100644 index 000000000..680d3cab3 --- /dev/null +++ b/tests/modules/apigee_organization/test_plan.py @@ -0,0 +1,49 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import pytest + + +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') + + +@pytest.fixture +def resources(plan_runner): + _, resources = plan_runner(FIXTURES_DIR) + return resources + + +def test_resource_count(resources): + "Test number of resources created." + assert len(resources) == 6 + + +def test_envgroup_attachment(resources): + "Test Apigee Envgroup Attachments." + attachments = [r['values'] for r in resources if r['type'] + == 'google_apigee_envgroup_attachment'] + assert len(attachments) == 2 + assert set(a['environment'] for a in attachments) == set(['eval1', 'eval2']) + + +def test_envgroup(resources): + "Test env group." + envgroups = [r['values'] for r in resources if r['type'] + == 'google_apigee_envgroup'] + assert len(envgroups) == 1 + assert envgroups[0]['name'] == 'eval' + assert len(envgroups[0]['hostnames']) == 1 + assert envgroups[0]['hostnames'][0] == 'eval.api.example.com' diff --git a/tests/modules/apigee_x_instance/__init__.py b/tests/modules/apigee_x_instance/__init__.py new file mode 100644 index 000000000..d46dbae5e --- /dev/null +++ b/tests/modules/apigee_x_instance/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/modules/apigee_x_instance/fixture/main.tf b/tests/modules/apigee_x_instance/fixture/main.tf new file mode 100644 index 000000000..9915ef207 --- /dev/null +++ b/tests/modules/apigee_x_instance/fixture/main.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "apigee-x-instance" { + source = "../../../../modules/apigee-x-instance" + name = var.name + region = var.region + cidr_mask = 22 + + apigee_org_id = "my-project" + apigee_environments = [ + "eval1", + "eval2" + ] +} \ No newline at end of file diff --git a/tests/modules/apigee_x_instance/fixture/variables.tf b/tests/modules/apigee_x_instance/fixture/variables.tf new file mode 100644 index 000000000..603ec5085 --- /dev/null +++ b/tests/modules/apigee_x_instance/fixture/variables.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "name" { + type = string + default = "my-test-instance" +} + +variable "region" { + type = string + default = "europe-west1" +} \ No newline at end of file diff --git a/tests/modules/apigee_x_instance/test_plan.py b/tests/modules/apigee_x_instance/test_plan.py new file mode 100644 index 000000000..4b3a9256d --- /dev/null +++ b/tests/modules/apigee_x_instance/test_plan.py @@ -0,0 +1,50 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import pytest + + +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') + + +@pytest.fixture +def resources(plan_runner): + _, resources = plan_runner(FIXTURES_DIR) + return resources + + +def test_resource_count(resources): + "Test number of resources created." + assert len(resources) == 3 + + +def test_instance_attachment(resources): + "Test Apigee Instance Attachments." + attachments = [r['values'] for r in resources if r['type'] + == 'google_apigee_instance_attachment'] + assert len(attachments) == 2 + assert set(a['environment'] for a in attachments) == set(['eval1', 'eval2']) + + +def test_instance(resources): + "Test Instance." + instances = [r['values'] for r in resources if r['type'] + == 'google_apigee_instance'] + assert len(instances) == 1 + assert instances[0]['peering_cidr_range'] == 'SLASH_22' + assert instances[0]['name'] == 'my-test-instance' + assert instances[0]['location'] == 'europe-west1' + diff --git a/tests/modules/billing_budget/__init__.py b/tests/modules/billing_budget/__init__.py new file mode 100644 index 000000000..d46dbae5e --- /dev/null +++ b/tests/modules/billing_budget/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/modules/billing_budget/fixture/main.tf b/tests/modules/billing_budget/fixture/main.tf new file mode 100644 index 000000000..91c05e4a6 --- /dev/null +++ b/tests/modules/billing_budget/fixture/main.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "budget" { + source = "../../../../modules/billing-budget" + billing_account = "123456-123456-123456" + name = "my budget" + projects = var.projects + services = var.services + notify_default_recipients = var.notify_default_recipients + amount = var.amount + credit_treatment = var.credit_treatment + pubsub_topic = var.pubsub_topic + notification_channels = var.notification_channels + thresholds = var.thresholds + email_recipients = var.email_recipients +} diff --git a/tests/modules/billing_budget/fixture/variables.tf b/tests/modules/billing_budget/fixture/variables.tf new file mode 100644 index 000000000..6eb8e4e39 --- /dev/null +++ b/tests/modules/billing_budget/fixture/variables.tf @@ -0,0 +1,69 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "amount" { + type = number + default = 0 +} + +variable "credit_treatment" { + type = string + default = "INCLUDE_ALL_CREDITS" +} + +variable "email_recipients" { + type = object({ + project_id = string + emails = list(string) + }) + default = null +} + +variable "notification_channels" { + type = list(string) + default = null +} + +variable "notify_default_recipients" { + type = bool + default = false +} + +variable "projects" { + type = list(string) + default = null +} + +variable "pubsub_topic" { + type = string + default = null +} + +variable "services" { + type = list(string) + default = null +} + +variable "thresholds" { + type = object({ + current = list(number) + forecasted = list(number) + }) + default = { + current = [0.5, 1.0] + forecasted = [1.0] + } +} diff --git a/tests/modules/billing_budget/test_plan.py b/tests/modules/billing_budget/test_plan.py new file mode 100644 index 000000000..5692bf0a5 --- /dev/null +++ b/tests/modules/billing_budget/test_plan.py @@ -0,0 +1,69 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import pytest + + +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') + + +def test_pubsub(plan_runner): + "Test number of resources created." + _, resources = plan_runner(FIXTURES_DIR, pubsub_topic='topic') + assert len(resources) == 1 + resource = resources[0] + assert resource['values']['all_updates_rule'] == [ + {'disable_default_iam_recipients': False, + 'monitoring_notification_channels': [], + 'pubsub_topic': 'topic', + 'schema_version': '1.0'} + ] + +def test_channel(plan_runner): + _, resources = plan_runner(FIXTURES_DIR, notification_channels='["channel"]') + assert len(resources) == 1 + resource = resources[0] + assert resource['values']['all_updates_rule'] == [ + {'disable_default_iam_recipients': True, + 'monitoring_notification_channels': ['channel'], + 'pubsub_topic': None, + 'schema_version': '1.0'} + ] + +def test_emails(plan_runner): + email_recipients = '{project_id = "project", emails = ["a@b.com", "c@d.com"]}' + _, resources = plan_runner(FIXTURES_DIR, email_recipients=email_recipients) + assert len(resources) == 3 + + +def test_absolute_amount(plan_runner): + "Test absolute amount budget." + _, resources = plan_runner(FIXTURES_DIR, pubsub_topic='topic', amount="100") + assert len(resources) == 1 + resource = resources[0] + + amount = resource['values']['amount'][0] + assert amount['last_period_amount'] is None + assert amount['specified_amount'] == [{'nanos': None, 'units': '100'}] + + assert resource['values']['threshold_rules'] == [ + {'spend_basis': 'CURRENT_SPEND', + 'threshold_percent': 0.5}, + {'spend_basis': 'CURRENT_SPEND', + 'threshold_percent': 1}, + {'spend_basis': 'FORECASTED_SPEND', + 'threshold_percent': 1} + ] diff --git a/tests/modules/net_interconnect_attachment_direct/__init__.py b/tests/modules/net_interconnect_attachment_direct/__init__.py new file mode 100644 index 000000000..d46dbae5e --- /dev/null +++ b/tests/modules/net_interconnect_attachment_direct/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/modules/net-interconnect-attachment-direct/fixture/main.tf b/tests/modules/net_interconnect_attachment_direct/fixture/main.tf similarity index 100% rename from tests/modules/net-interconnect-attachment-direct/fixture/main.tf rename to tests/modules/net_interconnect_attachment_direct/fixture/main.tf diff --git a/tests/modules/net-interconnect-attachment-direct/fixture/variables.tf b/tests/modules/net_interconnect_attachment_direct/fixture/variables.tf similarity index 96% rename from tests/modules/net-interconnect-attachment-direct/fixture/variables.tf rename to tests/modules/net_interconnect_attachment_direct/fixture/variables.tf index 16353cbbb..08126b344 100644 --- a/tests/modules/net-interconnect-attachment-direct/fixture/variables.tf +++ b/tests/modules/net_interconnect_attachment_direct/fixture/variables.tf @@ -81,9 +81,9 @@ variable "router_config" { }) default = { - description = null - asn = 64514 - advertise_config = null + description = null + asn = 64514 + advertise_config = null } } diff --git a/tests/modules/net-interconnect-attachment-direct/test_plan.py b/tests/modules/net_interconnect_attachment_direct/test_plan.py similarity index 100% rename from tests/modules/net-interconnect-attachment-direct/test_plan.py rename to tests/modules/net_interconnect_attachment_direct/test_plan.py diff --git a/tests/modules/net_vpc_firewall_yaml/fixture/main.tf b/tests/modules/net_vpc_firewall_yaml/fixture/main.tf index 7db37c77c..4dcc9b7c2 100644 --- a/tests/modules/net_vpc_firewall_yaml/fixture/main.tf +++ b/tests/modules/net_vpc_firewall_yaml/fixture/main.tf @@ -15,9 +15,11 @@ */ module "firewall" { - source = "../../../../modules/net-vpc-firewall-yaml" - project_id = "my-project" - network = "my-network" - config_path = "./rules" - log_config = var.log_config + source = "../../../../modules/net-vpc-firewall-yaml" + project_id = "my-project" + network = "my-network" + config_directories = [ + "./rules" + ] + log_config = var.log_config } diff --git a/tests/networking/decentralized_firewall/__init__.py b/tests/networking/decentralized_firewall/__init__.py new file mode 100644 index 000000000..d46dbae5e --- /dev/null +++ b/tests/networking/decentralized_firewall/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/networking/decentralized_firewall/fixture/main.tf b/tests/networking/decentralized_firewall/fixture/main.tf new file mode 100644 index 000000000..9bef2d737 --- /dev/null +++ b/tests/networking/decentralized_firewall/fixture/main.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "test" { + source = "../../../../networking/decentralized-firewall" + billing_account_id = var.billing_account_id + prefix = var.prefix + root_node = var.root_node +} diff --git a/tests/networking/decentralized_firewall/fixture/variables.tf b/tests/networking/decentralized_firewall/fixture/variables.tf new file mode 100644 index 000000000..9646fe1b7 --- /dev/null +++ b/tests/networking/decentralized_firewall/fixture/variables.tf @@ -0,0 +1,28 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "billing_account_id" { + type = string + default = "ABCDE-12345-ABCDE" +} + +variable "prefix" { + type = string + default = "test" +} + +variable "root_node" { + type = string + default = "organizations/0123456789" +} diff --git a/tests/networking/decentralized_firewall/test_plan.py b/tests/networking/decentralized_firewall/test_plan.py new file mode 100644 index 000000000..cb1764a9d --- /dev/null +++ b/tests/networking/decentralized_firewall/test_plan.py @@ -0,0 +1,27 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import pytest + + +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixture") + + +def test_resources(e2e_plan_runner): + "Test that plan works and the numbers of resources is as expected." + modules, resources = e2e_plan_runner(FIXTURES_DIR) + assert len(modules) == 8 + assert len(resources) == 50 diff --git a/tests/networking/hub_and_spoke_peering/fixture/main.tf b/tests/networking/hub_and_spoke_peering/fixture/main.tf index 5df89997e..899a622e1 100644 --- a/tests/networking/hub_and_spoke_peering/fixture/main.tf +++ b/tests/networking/hub_and_spoke_peering/fixture/main.tf @@ -15,7 +15,7 @@ */ module "test" { - source = "../../../../networking/hub-and-spoke-peering" + source = "../../../../networking/hub-and-spoke-peering" project_create = { billing_account = "123456-123456-123456" oslogin = true diff --git a/third-party-solutions/openshift/tf/providers.tf b/third-party-solutions/openshift/tf/providers.tf index 11735b9f7..52f428818 100644 --- a/third-party-solutions/openshift/tf/providers.tf +++ b/third-party-solutions/openshift/tf/providers.tf @@ -15,7 +15,7 @@ */ # pinning to avoid some weird issues we had with the following version - + terraform { required_providers { google = { diff --git a/tools/check_documentation.py b/tools/check_documentation.py index 5f03abf9f..667b2dd5b 100755 --- a/tools/check_documentation.py +++ b/tools/check_documentation.py @@ -31,7 +31,7 @@ class DocState(enum.Enum): UNKNOWN = 4 def __str__(self): - return {1: '✗', 2: '✓', 3: '✗', 4: '?'}[self.value] + return {1: '✗', 2: '✓', 3: '!', 4: '?'}[self.value] def check_path(pathname):