diff --git a/modules/cloud-config-container/cos-generic-metadata/README.md b/modules/cloud-config-container/cos-generic-metadata/README.md new file mode 100644 index 000000000..8e85e4219 --- /dev/null +++ b/modules/cloud-config-container/cos-generic-metadata/README.md @@ -0,0 +1,87 @@ +# Generic cloud-init generator for Container Optimized OS + +This helper module manages a `cloud-config` configuration that can start a container on [Container Optimized OS](https://cloud.google.com/container-optimized-os/docs) (COS). Either a complete `cloud-config` template can be provided via the `cloud_config` variable with optional template variables via the `config_variables`, or a generic `cloud-config` can be generated based on typical parameters needed to start a container. + +Logging can be enabled via the [Google Cloud Logging docker driver](https://docs.docker.com/config/containers/logging/gcplogs/) using the `gcp_logging` variable. This is enabled by default, but requires that the service account running the COS instance have the `roles/logging.logWriter` IAM role or equivalent permissions on the project. If it doesn't, the container will fail to start unless this is disabled. + +The module renders the generated cloud config in the `cloud_config` output, which can be directly used in instances or instance templates via the `user-data` metadata attribute. + +## Examples + +### Default configuration + +This example will create a `cloud-config` that starts [Envoy Proxy](https://www.envoyproxy.io) and expose it on port 80. For a complete example, look at the sibling [`envoy-traffic-director`](../envoy-traffic-director/README.md) module that uses this module to start Envoy Proxy and connect it to [Traffic Director](https://cloud.google.com/traffic-director). + +```hcl +module "cos-envoy" { + source = "./modules/cos-generic-metadata" + + container_image = "envoyproxy/envoy:v1.14.1" + container_name = "envoy" + container_args = "-c /etc/envoy/envoy.yaml --log-level info --allow-unknown-static-fields" + + container_volumes = [ + { host = "/etc/envoy/envoy.yaml", + container = "/etc/envoy/envoy.yaml" + } + ] + + docker_args = "--network host --pid host" + + files = { + "/var/run/envoy/customize.sh" = { + content = file("customize.sh") + owner = "root" + permissions = "0744" + } + "/etc/envoy/envoy.yaml" = { + content = file("envoy.yaml") + owner = "root" + permissions = "0644" + } + } + + run_commands = [ + "iptables -t nat -N ENVOY_IN_REDIRECT", + "iptables -t nat -A ENVOY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15001", + "iptables -t nat -A PREROUTING -p tcp -m tcp --dport 80 -j ENVOY_IN_REDIRECT", + "iptables -t filter -A INPUT -p tcp -m tcp --dport 15001 -m state --state NEW,ESTABLISHED -j ACCEPT", + "/var/run/envoy/customize.sh", + "systemctl daemon-reload", + "systemctl start envoy", + ] + + users = [ + { + username = "envoy", + uid = 1337 + } + ] +} +``` + + +## Variables + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| container\_image | Container image. | `string` | n/a | yes | +| boot\_commands | List of cloud-init `bootcmd`s | `list(string)` | `[]` | no | +| cloud\_config | Cloud config template path. If provided, takes precedence over all other arguments. | `string` | `null` | no | +| config\_variables | Additional variables used to render the template passed via `cloud_config` | `map(any)` | `{}` | no | +| container\_args | Arguments for container | `string` | `""` | no | +| container\_name | Name of the container to be run | `string` | `"container"` | no | +| container\_volumes | List of volumes | 
list(object({
    host      = string,
    container = string
  }))
| `[]` | no | +| docker\_args | Extra arguments to be passed for docker | `string` | `null` | no | +| file\_defaults | Default owner and permissions for files. | 
object({
    owner       = string
    permissions = string
  })
| 
{
  "owner": "root",
  "permissions": "0644"
}
| no | +| files | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | 
map(object({
    content     = string
    owner       = string
    permissions = string
  }))
| `{}` | no | +| gcp\_logging | Should container logs be sent to Google Cloud Logging | `bool` | `true` | no | +| run\_commands | List of cloud-init `runcmd`s | `list(string)` | `[]` | no | +| users | List of usernames to be created. If provided, first user will be used to run the container. | 
list(object({
    username = string,
    uid      = number,
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| cloud\_config | Rendered cloud-config file to be passed as user-data instance metadata. | + diff --git a/modules/cloud-config-container/cos-generic-metadata/cloud-config.yaml b/modules/cloud-config-container/cos-generic-metadata/cloud-config.yaml new file mode 100644 index 000000000..9f8e38fa6 --- /dev/null +++ b/modules/cloud-config-container/cos-generic-metadata/cloud-config.yaml @@ -0,0 +1,82 @@ +#cloud-config + +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +%{ if length(users) > 0 ~} +users: +%{ for user in users ~} + - name: ${user.username} + uid: ${user.uid} +%{ endfor ~} +%{ endif ~} + +write_files: + - path: /var/lib/docker/daemon.json + permissions: 0644 + owner: root + content: | + { + "live-restore": true, + "storage-driver": "overlay2", + "log-opts": { + "max-size": "1024m" + } + } + # ${container_name} container service + - path: /etc/systemd/system/${container_name}.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Start ${container_name} container + After=gcr-online.target docker.socket + Wants=gcr-online.target docker.socket docker-events-collector.service + [Service] + ExecStart=/usr/bin/docker run --rm --name=${container_name} \ +%{ if length(users) > 0 ~} + --user=${users[0].uid} \ +%{ endif ~} +%{ if gcp_logging == true ~} + --log-driver=gcplogs \ +%{ endif ~} +%{ if docker_args != null ~} + ${docker_args} \ +%{ endif ~} +%{ for volume in container_volumes ~} + -v ${volume.host}:${volume.container} \ +%{ endfor ~} + ${container_image} ${container_args} + ExecStop=/usr/bin/docker stop ${container_name} +%{ for path, data in files ~} + - path: ${path} + owner: ${lookup(data, "owner", "root")} + permissions: ${lookup(data, "permissions", "0644")} + content: | + ${indent(6, data.content)} +%{ endfor ~} + +%{ if length(boot_commands) > 0 ~} +bootcmd: +%{ for command in boot_commands ~} + - ${command} +%{ endfor ~} +%{ endif ~} + +%{ if length(run_commands) > 0 ~} +runcmd: +%{ for command in run_commands ~} + - ${command} +%{ endfor ~} +%{ endif ~} diff --git a/modules/cloud-config-container/cos-generic-metadata/main.tf b/modules/cloud-config-container/cos-generic-metadata/main.tf new file mode 100644 index 000000000..ee4c2ae0c --- /dev/null +++ b/modules/cloud-config-container/cos-generic-metadata/main.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cloud_config = templatefile(local.template, merge(var.config_variables, { + boot_commands = var.boot_commands + container_args = var.container_args + container_image = var.container_image + container_name = var.container_name + container_volumes = var.container_volumes + docker_args = var.docker_args + files = local.files + gcp_logging = var.gcp_logging + run_commands = var.run_commands + users = var.users + })) + files = { + for path, attrs in var.files : path => { + content = attrs.content, + owner = attrs.owner == null ? var.file_defaults.owner : attrs.owner, + permissions = ( + attrs.permissions == null + ? var.file_defaults.permissions + : attrs.permissions + ) + } + } + template = ( + var.cloud_config == null + ? "${path.module}/cloud-config.yaml" + : var.cloud_config + ) +} diff --git a/modules/cloud-config-container/cos-generic-metadata/outputs.tf b/modules/cloud-config-container/cos-generic-metadata/outputs.tf new file mode 100644 index 000000000..ec0d3a868 --- /dev/null +++ b/modules/cloud-config-container/cos-generic-metadata/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloud_config" { + description = "Rendered cloud-config file to be passed as user-data instance metadata." + value = local.cloud_config +} diff --git a/modules/cloud-config-container/cos-generic-metadata/variables.tf b/modules/cloud-config-container/cos-generic-metadata/variables.tf new file mode 100644 index 000000000..1d54de00e --- /dev/null +++ b/modules/cloud-config-container/cos-generic-metadata/variables.tf @@ -0,0 +1,110 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "boot_commands" { + description = "List of cloud-init `bootcmd`s" + type = list(string) + default = [] +} + +variable "cloud_config" { + description = "Cloud config template path. If provided, takes precedence over all other arguments." + type = string + default = null +} + +variable "config_variables" { + description = "Additional variables used to render the template passed via `cloud_config`" + type = map(any) + default = {} +} + +variable "container_args" { + description = "Arguments for container" + type = string + default = "" +} + + +variable "container_image" { + description = "Container image." + type = string +} + +variable "container_name" { + description = "Name of the container to be run" + type = string + default = "container" +} + +variable "container_volumes" { + description = "List of volumes" + type = list(object({ + host = string, + container = string + })) + default = [] +} + +variable "docker_args" { + description = "Extra arguments to be passed for docker" + type = string + default = null +} + +variable "file_defaults" { + description = "Default owner and permissions for files." + type = object({ + owner = string + permissions = string + }) + default = { + owner = "root" + permissions = "0644" + } +} + +variable "files" { + description = "Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null." + type = map(object({ + content = string + owner = string + permissions = string + })) + default = {} +} + +variable "gcp_logging" { + description = "Should container logs be sent to Google Cloud Logging" + type = bool + default = true +} + +variable "run_commands" { + description = "List of cloud-init `runcmd`s" + type = list(string) + default = [] +} + +variable "users" { + description = "List of usernames to be created. If provided, first user will be used to run the container." + type = list(object({ + username = string, + uid = number, + })) + default = [ + ] +} diff --git a/modules/cloud-config-container/envoy-traffic-director/README.md b/modules/cloud-config-container/envoy-traffic-director/README.md new file mode 100644 index 000000000..8cb395637 --- /dev/null +++ b/modules/cloud-config-container/envoy-traffic-director/README.md @@ -0,0 +1,59 @@ +# Containerized Envoy Proxy with Traffic Director on Container Optimized OS + +This module manages a `cloud-config` configuration that starts a containerized Envoy Proxy on Container Optimized OS connected to Traffic Director. The default configuration creates a reverse proxy exposed on the node's port 80. Traffic routing policies and management should be managed by other means via Traffic Director. + +## Examples + +### Default configuration + +```hcl +# Envoy TD config +module "cos-envoy-td" { + source = "./modules/cloud-config-container/envoy-traffic-director" +} + +# COS VM +module "vm-cos" { + source = "./modules/compute-vm" + project_id = local.project_id + region = local.region + zone = local.zone + name = "cos-envoy-td" + network_interfaces = [{ + network = local.vpc.self_link, + subnetwork = local.vpc.subnet_self_link, + nat = false, + addresses = null + }] + instance_count = 1 + tags = ["ssh", "http"] + + metadata = { + user-data = module.cos-envoy-td.cloud_config + } + + boot_disk = { + image = "projects/cos-cloud/global/images/family/cos-stable" + type = "pd-ssd" + size = 10 + } + + service_account_scopes = ["https://www.googleapis.com/auth/cloud-platform"] +} +``` + + + +## Variables + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| envoy\_image | Envoy Proxy container image to use. | `string` | `"envoyproxy/envoy:v1.14.1"` | no | +| gcp\_logging | Should container logs be sent to Google Cloud Logging | `bool` | `true` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| cloud\_config | Rendered cloud-config file to be passed as user-data instance metadata. | + diff --git a/modules/cloud-config-container/envoy-traffic-director/files/customize.sh b/modules/cloud-config-container/envoy-traffic-director/files/customize.sh new file mode 100644 index 000000000..f809685e3 --- /dev/null +++ b/modules/cloud-config-container/envoy-traffic-director/files/customize.sh @@ -0,0 +1,9 @@ +#!/bin/bash +ENVOY_NODE_ID=$(uuidgen)~$(curl -s -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/network-interfaces/0/ip) +ENVOY_ZONE=$(curl -s -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/zone | cut -f 4 -d '/') +CONFIG_PROJECT_NUMBER=$(curl -s -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/network-interfaces/0/network | cut -f 2 -d '/') +VPC_NETWORK_NAME=$(curl -s -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/network-interfaces/0/network | cut -f 4 -d '/') +sed -i "s/_ENVOY_NODE_ID_/${ENVOY_NODE_ID}/" /etc/envoy/envoy.yaml +sed -i "s/_ENVOY_ZONE_/${ENVOY_ZONE}/" /etc/envoy/envoy.yaml +sed -i "s/_CONFIG_PROJECT_NUMBER_/${CONFIG_PROJECT_NUMBER}/" /etc/envoy/envoy.yaml +sed -i "s/_VPC_NETWORK_NAME_/${VPC_NETWORK_NAME}/" /etc/envoy/envoy.yaml diff --git a/modules/cloud-config-container/envoy-traffic-director/files/envoy.yaml b/modules/cloud-config-container/envoy-traffic-director/files/envoy.yaml new file mode 100644 index 000000000..49cb7ac9d --- /dev/null +++ b/modules/cloud-config-container/envoy-traffic-director/files/envoy.yaml @@ -0,0 +1,140 @@ +node: + id: "_ENVOY_NODE_ID_" + cluster: cluster # unused + locality: + zone: "_ENVOY_ZONE_" + metadata: + TRAFFICDIRECTOR_INTERCEPTION_PORT: "15001" + TRAFFICDIRECTOR_NETWORK_NAME: "_VPC_NETWORK_NAME_" + TRAFFICDIRECTOR_GCP_PROJECT_NUMBER: "_CONFIG_PROJECT_NUMBER_" + TRAFFICDIRECTOR_ENABLE_TRACING: "false" + TRAFFICDIRECTOR_ACCESS_LOG_PATH: "" + TRAFFICDIRECTOR_INBOUND_BACKEND_PORTS: "" + +dynamic_resources: + lds_config: {ads: {}} + cds_config: {ads: {}} + ads_config: + api_type: GRPC + grpc_services: + - google_grpc: + target_uri: trafficdirector.googleapis.com:443 + stat_prefix: trafficdirector + channel_credentials: + ssl_credentials: + root_certs: + filename: /etc/ssl/certs/ca-certificates.crt + call_credentials: + google_compute_engine: {} + +cluster_manager: + load_stats_config: + api_type: GRPC + grpc_services: + - google_grpc: + target_uri: trafficdirector.googleapis.com:443 + stat_prefix: trafficdirector + channel_credentials: + ssl_credentials: + root_certs: + filename: /etc/ssl/certs/ca-certificates.crt + call_credentials: + google_compute_engine: {} + +admin: + access_log_path: /dev/stdout + address: + socket_address: + address: 127.0.0.1 # Admin page is only accessible locally. + port_value: 15000 + +tracing: + http: + name: envoy.tracers.opencensus + typed_config: + "@type": type.googleapis.com/envoy.config.trace.v2.OpenCensusConfig + stackdriver_exporter_enabled: "false" + stackdriver_project_id: "" + +layered_runtime: + layers: + - name: rtds_layer + rtds_layer: + name: traffic_director_runtime + rtds_config: {ads: {}} + - name: static_layer + static_layer: + envoy: + deprecated_features: + cluster: + proto:ORIGINAL_DST_LB: "true" + proto:extension_protocol_options: "true" + proto:tls_context: "true" + health_check: + proto:use_http2: "true" + http_connection_manager: + proto:operation_name: "true" + listener: + proto:tls_context: "true" + listener_components: + proto:config: "true" + route_components: + proto:allow_origin: "true" + proto:method: "true" + proto:pattern: "true" + proto:regex: "true" + proto:regex_match: "true" + proto:value: "true" + string: + proto:regex: "true" + trace: + proto:HTTP_JSON_V1: "true" + deprecated_features:envoy: + api: + v2: + Cluster: + LbPolicy: + ORIGINAL_DST_LB: "true" + extension_protocol_options: "true" + tls_context: "true" + Listener: + tls_context: "true" + core: + HealthCheck: + HttpHealthCheck: + use_http2: "true" + listener: + Filter: + config: "true" + ListenerFilter: + config: "true" + route: + CorsPolicy: + allow_origin: "true" + HeaderMatcher: + regex_match: "true" + QueryParameterMatcher: + regex: "true" + value: "true" + RouteMatch: + regex: "true" + VirtualCluster: + method: "true" + pattern: "true" + config: + filter: + network: + http_connection_manager: + v2: + HttpConnectionManager: + Tracing: + operation_name: "true" + trace: + v2: + ZipkinConfig: + CollectorEndpointVersion: + HTTP_JSON_V1: "true" + type: + matcher: + StringMatcher: + regex: "true" diff --git a/modules/cloud-config-container/envoy-traffic-director/main.tf b/modules/cloud-config-container/envoy-traffic-director/main.tf new file mode 100644 index 000000000..768ed5af4 --- /dev/null +++ b/modules/cloud-config-container/envoy-traffic-director/main.tf @@ -0,0 +1,67 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "cos-envoy-td" { + source = "./modules/cos-generic-metadata" + + boot_commands = [ + "systemctl start node-problem-detector", + ] + + container_image = var.envoy_image + container_name = "envoy" + container_args = "-c /etc/envoy/envoy.yaml --log-level info --allow-unknown-static-fields" + + container_volumes = [ + { host = "/etc/envoy/envoy.yaml", + container = "/etc/envoy/envoy.yaml" + } + ] + + docker_args = "--network host --pid host" + + files = { + "/var/run/envoy/customize.sh" = { + content = file("${path.module}/files/customize.sh") + owner = "root" + permissions = "0744" + } + "/etc/envoy/envoy.yaml" = { + content = file("${path.module}/files/envoy.yaml") + owner = "root" + permissions = "0644" + } + } + + gcp_logging = var.gcp_logging + + run_commands = [ + "iptables -t nat -N ENVOY_IN_REDIRECT", + "iptables -t nat -A ENVOY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15001", + "iptables -t nat -A PREROUTING -p tcp -m tcp --dport 80 -j ENVOY_IN_REDIRECT", + "iptables -t filter -A INPUT -p tcp -m tcp --dport 15001 -m state --state NEW,ESTABLISHED -j ACCEPT", + "/var/run/envoy/customize.sh", + "systemctl daemon-reload", + "systemctl start envoy", + ] + + users = [ + { + username = "envoy", + uid = 1337 + } + ] +} diff --git a/modules/cloud-config-container/envoy-traffic-director/modules/cos-generic-metadata b/modules/cloud-config-container/envoy-traffic-director/modules/cos-generic-metadata new file mode 120000 index 000000000..66c564eff --- /dev/null +++ b/modules/cloud-config-container/envoy-traffic-director/modules/cos-generic-metadata @@ -0,0 +1 @@ +../../cos-generic-metadata \ No newline at end of file diff --git a/modules/cloud-config-container/envoy-traffic-director/outputs.tf b/modules/cloud-config-container/envoy-traffic-director/outputs.tf new file mode 100644 index 000000000..4783b7f3a --- /dev/null +++ b/modules/cloud-config-container/envoy-traffic-director/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloud_config" { + description = "Rendered cloud-config file to be passed as user-data instance metadata." + value = module.cos-envoy-td.cloud_config +} diff --git a/modules/cloud-config-container/envoy-traffic-director/variables.tf b/modules/cloud-config-container/envoy-traffic-director/variables.tf new file mode 100644 index 000000000..f38708dcc --- /dev/null +++ b/modules/cloud-config-container/envoy-traffic-director/variables.tf @@ -0,0 +1,11 @@ +variable "envoy_image" { + description = "Envoy Proxy container image to use." + type = string + default = "envoyproxy/envoy:v1.14.1" +} + +variable "gcp_logging" { + description = "Should container logs be sent to Google Cloud Logging" + type = bool + default = true +} \ No newline at end of file