diff --git a/fast/stages/03-gke-multitenant/config/cluster/README.md b/fast/stages/03-gke-multitenant/config/cluster/README.md new file mode 100644 index 000000000..fe3a85281 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/cluster/README.md @@ -0,0 +1,3 @@ +The cluster/ directory contains configs that apply to entire clusters, rather than to namespaces. By default, any config in the cluster/ directory applies to every cluster enrolled in Config Sync. You can limit which clusters a config can affect by using a ClusterSelector. + +Doc ref: https://cloud.google.com/anthos-config-management/docs/concepts/hierarchical-repo#cluster \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/cluster/clusterrole-ns-viewer.yml b/fast/stages/03-gke-multitenant/config/cluster/clusterrole-ns-viewer.yml new file mode 100644 index 000000000..3bf4cf773 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/cluster/clusterrole-ns-viewer.yml @@ -0,0 +1,28 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: dev-namespace-viewer +rules: +- apiGroups: ["", "metrics.k8s.io", "extensions", "apps"] + resources: + - "pods" + - "pods/log" + - "events" + - "deployments" + - "replicasets" + verbs: ["get", "watch", "list"] \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/cluster/ns-must-have-geo-constraint_v1beta_gatekeeper.yml b/fast/stages/03-gke-multitenant/config/cluster/ns-must-have-geo-constraint_v1beta_gatekeeper.yml new file mode 100644 index 000000000..a89b549f7 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/cluster/ns-must-have-geo-constraint_v1beta_gatekeeper.yml @@ -0,0 +1,43 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredLabels +metadata: + name: ns-must-have-geo + annotations: + configsync.gke.io/cluster-name-selector: gke-1 +spec: + enforcementAction: dryrun + match: + excludedNamespaces: + - "kube-system" + - "kube-public" + - "kube-node-lease" + - "resource-group-system" + - "config-management-monitoring" + - "config-management-system" + - "cos-auditd" + - "gatekeeper-system" + - "kube-node-lease" + - "cnrm-system" + - "gke-mcs" + - "configconnector-operator-system" + kinds: + - apiGroups: [""] + kinds: ["Namespace"] + parameters: + labels: + - key: "geo" \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/cluster/pod_priority_classes.yml b/fast/stages/03-gke-multitenant/config/cluster/pod_priority_classes.yml new file mode 100644 index 000000000..070e1e9cf --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/cluster/pod_priority_classes.yml @@ -0,0 +1,47 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: high-priority-system +value: 1000000 +globalDefault: false +description: "This priority class will cause other pods to be preempted." +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: high-priority +value: 1000 +globalDefault: false +description: "This priority class will cause other pods to be preempted." +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: mid-priority +value: 100 +globalDefault: false +# preemptionPolicy: Never +description: "This priority class will cause other pods to be preempted." +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: low-priority +value: 10 +globalDefault: true +# preemptionPolicy: Never +description: "This priority class will cause other pods to be preempted." \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/README.md b/fast/stages/03-gke-multitenant/config/namespaces/README.md new file mode 100644 index 000000000..99382474b --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/README.md @@ -0,0 +1,7 @@ + + +The namespaces/ directory contains configs for namespaces and namespace-scoped objects. + +The structure within namespaces/ is the mechanism that drives namespace inheritance. You can limit which namespaces can inherit a config, by using a NamespaceSelector. + +Doc ref: https://cloud.google.com/anthos-config-management/docs/concepts/hierarchical-repo#namespaces \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_configmap_cos-auditd.yml b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_configmap_cos-auditd.yml new file mode 100644 index 000000000..0c3a25c62 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_configmap_cos-auditd.yml @@ -0,0 +1,54 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +kind: ConfigMap +apiVersion: v1 +metadata: + name: fluentd-gcp-config-cos-auditd + namespace: cos-auditd + annotations: + kubernetes.io/description: 'ConfigMap for Linux auditd logging daemonset on COS nodes.' +data: + google-fluentd.conf: |- + + @type systemd + filters [{ "SYSLOG_IDENTIFIER": "audit" }] + pos_file /var/log/gcp-journald-audit.pos + read_from_head true + tag linux-auditd + + + # Do not collect fluentd's own logs to avoid infinite loops. + + @type null + + + + @type google_cloud + + enable_monitoring false + split_logs_by_tag false + detect_subservice false + buffer_type file + buffer_path /var/log/fluentd-buffers/system.audit.buffer + buffer_queue_full_action block + buffer_chunk_limit 512k + buffer_queue_limit 2 + flush_interval 5s + max_retry_wait 30 + disable_retry_limit + num_threads 2 + use_grpc true + \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_daemonset_cos-auditd.yml b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_daemonset_cos-auditd.yml new file mode 100644 index 000000000..d9980318c --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_daemonset_cos-auditd.yml @@ -0,0 +1,128 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cos-auditd-logging + namespace: cos-auditd + annotations: + kubernetes.io/description: 'DaemonSet that enables Linux auditd logging on COS nodes.' +spec: + selector: + matchLabels: + name: cos-auditd-logging + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + name: cos-auditd-logging + spec: + hostNetwork: true + hostPID: true + nodeSelector: + cloud.google.com/gke-os-distribution: cos + volumes: + - hostPath: + path: / + name: host + - hostPath: + path: /var/log + name: varlog + - hostPath: + path: /usr/lib64 + name: libsystemddir + - configMap: + defaultMode: 420 + name: fluentd-gcp-config-cos-auditd + name: config-volume + initContainers: + - name: cos-auditd-setup + image: ubuntu + command: ["chroot", "/host", "systemctl", "start", "cloud-audit-setup"] + securityContext: + privileged: true + volumeMounts: + - name: host + mountPath: /host + resources: + requests: + memory: "10Mi" + cpu: "10m" + priorityClassName: high-priority-system + containers: + - name: fluentd-gcp-cos-auditd + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/stackdriver-agents/stackdriver-logging-agent:0.6-1.6.0-1 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - /bin/sh + - -c + - | + LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300}; STUCK_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-900}; if [[ ! -e /var/log/fluentd-buffers ]]; then + exit 1; + fi; touch -d "${STUCK_THRESHOLD_SECONDS} seconds ago" /tmp/marker-stuck; if [[ -z "$(find /var/log/fluentd-buffers -type f -newer /tmp/marker-stuck -print -quit)" ]]; then + rm -rf /var/log/fluentd-buffers; + exit 1; + fi; touch -d "${LIVENESS_THRESHOLD_SECONDS} seconds ago" /tmp/marker-liveness; if [[ -z "$(find /var/log/fluentd-buffers -type f -newer /tmp/marker-liveness -print -quit)" ]]; then + exit 1; + fi; + failureThreshold: 3 + initialDelaySeconds: 600 + periodSeconds: 60 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: "1" + memory: 500Mi + requests: + cpu: 100m + memory: 200Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/log + name: varlog + - mountPath: /host/lib + name: libsystemddir + readOnly: true + - mountPath: /etc/google-fluentd/google-fluentd.conf + subPath: google-fluentd.conf + name: config-volume + dnsPolicy: Default + restartPolicy: Always + terminationGracePeriodSeconds: 30 + tolerations: + - effect: NoSchedule + key: node.alpha.kubernetes.io/ismaster + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: sandbox.gke.io/runtime + operator: Equal + value: gvisor + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/namespace.yml b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/namespace.yml new file mode 100644 index 000000000..fc78be56c --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/namespace.yml @@ -0,0 +1,19 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +apiVersion: v1 +kind: Namespace +metadata: + name: cos-auditd \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/limit-range.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/limit-range.yml new file mode 100644 index 000000000..d50f33c3b --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/limit-range.yml @@ -0,0 +1,34 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# LimitRange Reference: https://kubernetes.io/docs/concepts/policy/limit-range/ +apiVersion: v1 +kind: LimitRange +metadata: + name: default-limits +spec: + limits: + - type: Container + default: + cpu: "1" + memory: 512Mi + defaultRequest: + memory: 256Mi + cpu: "0.5" + max: + cpu: "2" + memory: 1Gi + - type: PersistentVolumeClaim + max: + storage: 2Gi \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/network-policy-default-deny-egress.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/network-policy-default-deny-egress.yml new file mode 100644 index 000000000..aabbe4e14 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/network-policy-default-deny-egress.yml @@ -0,0 +1,22 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# apiVersion: networking.k8s.io/v1 +# kind: NetworkPolicy +# metadata: +# name: default-deny-egress +# spec: +# podSelector: {} +# policyTypes: +# - Egress diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/namespace.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/namespace.yml new file mode 100644 index 000000000..4d8d1c48f --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/namespace.yml @@ -0,0 +1,18 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: team-a diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/repo-sync.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/repo-sync.yml new file mode 100644 index 000000000..f3615b1d5 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/repo-sync.yml @@ -0,0 +1,30 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This Namespace is configured with RepoSync +# Doc ref: https://cloud.google.com/anthos-config-management/docs/how-to/namespace-repositories#namespace-root + +apiVersion: configsync.gke.io/v1beta1 +kind: RepoSync +metadata: + name: repo-sync + namespace: team-a +spec: + sourceFormat: unstructured + git: + repo: "https://github.com/danielmarzini/configsync-application-example" + branch: main + revision: HEAD + dir: "configsync/team-a" + auth: none diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/resourcequotas.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/resourcequotas.yml new file mode 100644 index 000000000..b47cdaa2d --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/resourcequotas.yml @@ -0,0 +1,35 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ResourceQuota Reference: https://kubernetes.io/docs/concepts/policy/resource-quotas/ +kind: ResourceQuota +apiVersion: v1 +metadata: + namespace: team-a + name: pvc +spec: + hard: + persistentvolumeclaims: "3" +--- +apiVersion: v1 +kind: ResourceQuota +metadata: + namespace: team-a + name: mem-cpu-store +spec: + hard: + requests.cpu: "1" + requests.memory: 1Gi + limits.cpu: "2" + limits.memory: 2Gi \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/rolebinding-ns-viewer.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/rolebinding-ns-viewer.yml new file mode 100644 index 000000000..72377417d --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/rolebinding-ns-viewer.yml @@ -0,0 +1,28 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: namespace-viewer +roleRef: + kind: ClusterRole + name: dev-namespace-viewer + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: Group + name: team-a@marzi.gcp-pso-italy.net + apiGroup: rbac.authorization.k8s.io + namespace: team-a \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/sync-rolebinding.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/sync-rolebinding.yml new file mode 100644 index 000000000..ec5de2503 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/sync-rolebinding.yml @@ -0,0 +1,28 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ROOT_REPO/namespaces/NAMESPACE/sync-rolebinding.yaml + kind: RoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: syncs-repo + namespace: team-a + subjects: + - kind: ServiceAccount + name: ns-reconciler-team-a + namespace: config-management-system + roleRef: + kind: ClusterRole + name: edit + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/apps_v1_deployment_v1.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/apps_v1_deployment_v1.yml new file mode 100644 index 000000000..3f2c61dac --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/apps_v1_deployment_v1.yml @@ -0,0 +1,44 @@ + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: team-b + name: whereami +spec: + replicas: 1 + selector: + matchLabels: + app: whereami + template: + metadata: + labels: + app: whereami + version: v1 + spec: + containers: + - name: whereami + image: us-docker.pkg.dev/google-samples/containers/gke/whereami:v1.2.7 + resources: + limits: + cpu: "0.5" + memory: 128Mi + requests: + cpu: "0.5" + memory: 128Mi + ports: + - name: http + containerPort: 8080 \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/namespace.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/namespace.yml new file mode 100644 index 000000000..075ca0859 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/namespace.yml @@ -0,0 +1,20 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: team-b + labels: + geo: "europe-west1" \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/system/repo.yml b/fast/stages/03-gke-multitenant/config/system/repo.yml new file mode 100644 index 000000000..bfcc179a4 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/system/repo.yml @@ -0,0 +1,23 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# For the mono-repo mode, declaring a Repo resource under system/ directory is required. +# For the multi-repo mode, declaring a Repo resource under system/ directory is optional. +apiVersion: configmanagement.gke.io/v1 +kind: Repo +metadata: + name: repo +spec: + version: 1.0.0 \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md index be3be6862..d1089cfa5 100644 --- a/fast/stages/03-gke-multitenant/dev/README.md +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -63,6 +63,9 @@ everything else is in RBAC (second part of this stage) ## Addresses for ILBs +### Private Cluster +WARNING: do not use run this stage with GKE-HUB and private clusters enabled from a machine outside the clusters network + ## Filestore - later diff --git a/fast/stages/03-gke-multitenant/dev/gke-clusters.tf b/fast/stages/03-gke-multitenant/dev/gke-clusters.tf index e74750c12..8f9cb987e 100644 --- a/fast/stages/03-gke-multitenant/dev/gke-clusters.tf +++ b/fast/stages/03-gke-multitenant/dev/gke-clusters.tf @@ -43,6 +43,7 @@ module "gke-cluster" { horizontal_pod_autoscaling = true config_connector_config = true kalm_config = false + gcp_filestore_csi_driver_config = false # enable only if enable_dataplane_v2 is changed to false below network_policy_config = false istio_config = { @@ -51,15 +52,15 @@ module "gke-cluster" { } } # change these here for all clusters if absolutely needed - # authenticator_security_group = var.authenticator_security_group - enable_dataplane_v2 = true - enable_l4_ilb_subsetting = false - enable_intranode_visibility = true - enable_shielded_nodes = true - workload_identity = true + authenticator_security_group = var.authenticator_security_group + enable_dataplane_v2 = true + enable_l4_ilb_subsetting = false + enable_intranode_visibility = true + enable_shielded_nodes = true + workload_identity = true private_cluster_config = { enable_private_nodes = true - enable_private_endpoint = true + enable_private_endpoint = false master_ipv4_cidr_block = each.value.net.master_range master_global_access = true } diff --git a/fast/stages/03-gke-multitenant/dev/gke-hub.tf b/fast/stages/03-gke-multitenant/dev/gke-hub.tf new file mode 100644 index 000000000..6d21b727e --- /dev/null +++ b/fast/stages/03-gke-multitenant/dev/gke-hub.tf @@ -0,0 +1,59 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "gke-hub" { + source = "../../../../modules/gke-hub" + project_id = module.gke-project-0.project_id + features = { + configmanagement = true + mc_ingress = true + mc_servicediscovery = true + } + member_clusters = { + for cluster_id in keys(var.clusters) : + cluster_id => module.gke-cluster[cluster_id].id + } + member_features = { + configmanagement = { + binauthz = false + config_sync = { + gcp_service_account_email = null + https_proxy = null + policy_dir = "fast/stages/03-gke-multitenant/config" + secret_type = "none" + source_format = "hierarchy" + sync_branch = "fast-dev-gke-marzi" + sync_repo = "https://github.com/GoogleCloudPlatform/cloud-foundation-fabric" + sync_rev = null + } + hierarchy_controller = null + policy_controller = { + exemptable_namespaces = [ + "asm-system", + "config-management-system", + "config-management-monitoring", + "gatekeeper-system", + "kube-system", + "cos-auditd" + ] + log_denies_enabled = true + referential_rules_enabled = false + template_library_installed = true + } + version = "1.10.2" + } + } +} diff --git a/fast/stages/03-gke-multitenant/dev/main.tf b/fast/stages/03-gke-multitenant/dev/main.tf index aef450b0d..cc27ed976 100644 --- a/fast/stages/03-gke-multitenant/dev/main.tf +++ b/fast/stages/03-gke-multitenant/dev/main.tf @@ -35,7 +35,9 @@ module "gke-project-0" { "gkehub.googleapis.com", "stackdriver.googleapis.com", "container.googleapis.com", - # "trafficdirector.googleapis.com" + "multiclusterservicediscovery.googleapis.com", + "multiclusteringress.googleapis.com", + "trafficdirector.googleapis.com" ] # add here any other service ids and keys for robot accounts which are needed # service_encryption_key_ids = { @@ -66,6 +68,9 @@ module "gke-project-0" { # values = ["projects/fl01-prod-iac-core-0"] # } # } + iam = { + "roles/container.clusterViewer" = var.cluster_viewer_users + } } module "gke-dataset-resource-usage" { diff --git a/fast/stages/03-gke-multitenant/dev/variables.tf b/fast/stages/03-gke-multitenant/dev/variables.tf index 2a165b8ee..608ad51b4 100644 --- a/fast/stages/03-gke-multitenant/dev/variables.tf +++ b/fast/stages/03-gke-multitenant/dev/variables.tf @@ -18,11 +18,11 @@ # 1 project, m clusters # cloud dns for gke? -# variable "authenticator_security_group" { -# description = "Optional group used for Groups for GKE." -# type = string -# default = null -# } +variable "authenticator_security_group" { + description = "Optional group used for Groups for GKE." + type = string + default = null +} variable "billing_account" { # tfdoc:variable:source 00-bootstrap @@ -36,14 +36,15 @@ variable "billing_account" { variable "cluster_defaults" { description = "Default values for optional cluster configurations." type = object({ - cloudrun_config = bool - database_encryption_key = string - enable_binary_authorization = bool - master_authorized_ranges = map(string) - max_pods_per_node = number - pod_security_policy = bool - release_channel = string - vertical_pod_autoscaling = bool + cloudrun_config = bool + database_encryption_key = string + enable_binary_authorization = bool + master_authorized_ranges = map(string) + max_pods_per_node = number + pod_security_policy = bool + release_channel = string + vertical_pod_autoscaling = bool + gcp_filestore_csi_driver_config = bool }) default = { # TODO: review defaults @@ -55,10 +56,11 @@ variable "cluster_defaults" { rfc1918_2 = "172.16.0.0/12" rfc1918_3 = "192.168.0.0/16" } - max_pods_per_node = 110 - pod_security_policy = false - release_channel = "STABLE" - vertical_pod_autoscaling = false + max_pods_per_node = 110 + pod_security_policy = false + release_channel = "STABLE" + vertical_pod_autoscaling = false + gcp_filestore_csi_driver_config = false } } @@ -82,18 +84,25 @@ variable "clusters" { subnet = string }) overrides = object({ - cloudrun_config = bool - database_encryption_key = string - enable_binary_authorization = bool - master_authorized_ranges = map(string) - max_pods_per_node = number - pod_security_policy = bool - release_channel = string - vertical_pod_autoscaling = bool + cloudrun_config = bool + database_encryption_key = string + enable_binary_authorization = bool + master_authorized_ranges = map(string) + max_pods_per_node = number + pod_security_policy = bool + release_channel = string + vertical_pod_autoscaling = bool + gcp_filestore_csi_driver_config = bool }) })) } +variable "cluster_viewer_users" { + description = "list of users to be granted as container.clusterViewer" + type = list(any) + default = [] +} + variable "dns_domain" { description = "Domain name used for clusters, prefix by each cluster name. Leave null to disable Cloud DNS for GKE." type = string diff --git a/modules/gke-hub/main.tf b/modules/gke-hub/main.tf index 784fe88c9..910641f4e 100644 --- a/modules/gke-hub/main.tf +++ b/modules/gke-hub/main.tf @@ -40,7 +40,7 @@ resource "google_gke_hub_feature" "configmanagement" { resource "google_gke_hub_feature" "mci" { provider = google-beta - for_each = var.features.mc_ingress ? var.member_clusters : {} + for_each = var.features.mc_ingress ? try(var.member_clusters[0], {}) : {} project = var.project_id name = "multiclusteringress" location = "global" @@ -67,6 +67,12 @@ resource "google_gke_hub_feature_membership" "feature_member" { feature = google_gke_hub_feature.configmanagement["1"].name membership = google_gke_hub_membership.membership[each.key].membership_id + depends_on = [ + google_gke_hub_feature.configmanagement, + google_gke_hub_feature.mci, + google_gke_hub_feature.mcs, + ] + dynamic "configmanagement" { for_each = ( try(var.member_features.configmanagement, null) != null diff --git a/modules/gke-nodepool/main.tf b/modules/gke-nodepool/main.tf index ec9bbf97c..b1c540b5a 100644 --- a/modules/gke-nodepool/main.tf +++ b/modules/gke-nodepool/main.tf @@ -84,7 +84,7 @@ resource "google_container_node_pool" "nodepool" { location = var.location name = var.name - initial_node_count = var.initial_node_count + initial_node_count = var.node_count == null ? var.initial_node_count : null // (dmarzi) TOFIX max_pods_per_node = var.max_pods_per_node node_count = var.autoscaling_config == null ? var.node_count : null node_locations = var.node_locations