From 75a065ab68688bf215a6aed746f39df1c85d78b1 Mon Sep 17 00:00:00 2001 From: Jacopo Andrea Giola Date: Fri, 7 Apr 2023 14:38:16 +0200 Subject: [PATCH] docs: add examples folder --- examples/README.md | 8 + examples/base/config.yaml | 53 ++ examples/eks/config.yaml | 53 ++ examples/gke/config.yaml | 53 ++ hack/cni/eks-1.12.8.yaml | 1618 ++++++++++++++++++++++++++++++++ hack/cni/eks-values.yaml | 1 + hack/cni/gke-1.12.8.yaml | 1847 +++++++++++++++++++++++++++++++++++++ hack/cni/gke-values.yaml | 11 + hack/cni/kind-1.12.6.yaml | 1592 ++++++++++++++++++++++++++++++++ hack/cni/kind-1.12.8.yaml | 1617 ++++++++++++++++++++++++++++++++ hack/cni/values.yaml | 36 + 11 files changed, 6889 insertions(+) create mode 100644 examples/README.md create mode 100644 examples/base/config.yaml create mode 100644 examples/eks/config.yaml create mode 100644 examples/gke/config.yaml create mode 100644 hack/cni/eks-1.12.8.yaml create mode 100755 hack/cni/eks-values.yaml create mode 100644 hack/cni/gke-1.12.8.yaml create mode 100755 hack/cni/gke-values.yaml create mode 100644 hack/cni/kind-1.12.6.yaml create mode 100644 hack/cni/kind-1.12.8.yaml create mode 100755 hack/cni/values.yaml diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 00000000..2a6625eb --- /dev/null +++ b/examples/README.md @@ -0,0 +1,8 @@ +# Examples + +In this folder you can find three different examples for installing all the modules and add-on of the distribution +for one of the supported kubernetes flavor or engine. + +You can use the file inside one of the folder with [vab] and install it on your cluster. + +[vab]: https://github.com/mia-platform/vab "cli for handling installation and upgrade of Mia-Platform distribution" diff --git a/examples/base/config.yaml b/examples/base/config.yaml new file mode 100644 index 00000000..fc12b885 --- /dev/null +++ b/examples/base/config.yaml @@ -0,0 +1,53 @@ +kind: ClustersConfiguration +apiVersion: vab.mia-platform.eu/v1alpha1 +name: base-installation +spec: + modules: + cni/cilium/base: + version: v1.24.0 + disaster-recovery/velero/base: + version: v1.24.1 + ingress/traefik/base: + version: v1.24.1 + observability/logging-operator/base: + version: v1.24.1 + observability/prometheus-operator/base: + version: v1.24.0 + scheduling/kube-green/base: + version: v1.24.1 + security/cert-manager/base: + version: v1.24.1 + security/kyverno/base: + version: v1.24.1 + security/rbac-manager/base: + version: v1.24.0 + addOns: + cert-manager/kube-green-webhook-certificate: + version: v1.0.0 + cert-manager/oidc-proxy-certificate: + version: v1.0.0 + cert-manager/prometheus-operator-webhook-certificate: + version: v1.0.0 + logging-operator/logging-stack: + version: v1.0.2 + prometheus-operator/cert-manager: + version: v1.0.1 + prometheus-operator/kube-green: + version: v1.0.0 + prometheus-operator/kyverno: + version: v1.0.1 + prometheus-operator/logging-operator: + version: v1.0.0 + prometheus-operator/logging-stack: + version: v1.0.0 + prometheus-operator/monitoring-stack: + version: v1.0.2 + prometheus-operator/rbac-manager: + version: v1.0.1 + prometheus-operator/traefik: + version: v1.0.0 + groups: + - name: base-installation + clusters: + - name: base-installation + context: base-installation diff --git a/examples/eks/config.yaml b/examples/eks/config.yaml new file mode 100644 index 00000000..348e6e6a --- /dev/null +++ b/examples/eks/config.yaml @@ -0,0 +1,53 @@ +kind: ClustersConfiguration +apiVersion: vab.mia-platform.eu/v1alpha1 +name: eks-installation +spec: + modules: + cni/cilium/eks: + version: v1.24.0 + disaster-recovery/velero/base: + version: v1.24.1 + ingress/traefik/eks: + version: v1.24.1 + observability/logging-operator/base: + version: v1.24.1 + observability/prometheus-operator/base: + version: v1.24.0 + scheduling/kube-green/base: + version: v1.24.1 + security/cert-manager/base: + version: v1.24.1 + security/kyverno/base: + version: v1.24.1 + security/rbac-manager/base: + version: v1.24.0 + addOns: + cert-manager/kube-green-webhook-certificate: + version: v1.0.0 + cert-manager/oidc-proxy-certificate: + version: v1.0.0 + cert-manager/prometheus-operator-webhook-certificate: + version: v1.0.0 + logging-operator/logging-stack: + version: v1.0.2 + prometheus-operator/cert-manager: + version: v1.0.1 + prometheus-operator/kube-green: + version: v1.0.0 + prometheus-operator/kyverno: + version: v1.0.1 + prometheus-operator/logging-operator: + version: v1.0.0 + prometheus-operator/logging-stack: + version: v1.0.0 + prometheus-operator/monitoring-stack: + version: v1.0.2 + prometheus-operator/rbac-manager: + version: v1.0.1 + prometheus-operator/traefik: + version: v1.0.0 + groups: + - name: eks-installation + clusters: + - name: eks-installation + context: eks-installation diff --git a/examples/gke/config.yaml b/examples/gke/config.yaml new file mode 100644 index 00000000..2f73db8e --- /dev/null +++ b/examples/gke/config.yaml @@ -0,0 +1,53 @@ +kind: ClustersConfiguration +apiVersion: vab.mia-platform.eu/v1alpha1 +name: gke-installation +spec: + modules: + cni/cilium/gke: + version: v1.24.0 + disaster-recovery/velero/base: + version: v1.24.1 + ingress/traefik/gke: + version: v1.24.1 + observability/logging-operator/base: + version: v1.24.1 + observability/prometheus-operator/base: + version: v1.24.0 + scheduling/kube-green/base: + version: v1.24.1 + security/cert-manager/base: + version: v1.24.1 + security/kyverno/base: + version: v1.24.1 + security/rbac-manager/base: + version: v1.24.0 + addOns: + cert-manager/kube-green-webhook-certificate: + version: v1.0.0 + cert-manager/oidc-proxy-certificate: + version: v1.0.0 + cert-manager/prometheus-operator-webhook-certificate: + version: v1.0.0 + logging-operator/logging-stack: + version: v1.0.2 + prometheus-operator/cert-manager: + version: v1.0.1 + prometheus-operator/kube-green: + version: v1.0.0 + prometheus-operator/kyverno: + version: v1.0.1 + prometheus-operator/logging-operator: + version: v1.0.0 + prometheus-operator/logging-stack: + version: v1.0.0 + prometheus-operator/monitoring-stack: + version: v1.0.2 + prometheus-operator/rbac-manager: + version: v1.0.1 + prometheus-operator/traefik: + version: v1.0.0 + groups: + - name: gke-installation + clusters: + - name: gke-installation + context: gke-installation diff --git a/hack/cni/eks-1.12.8.yaml b/hack/cni/eks-1.12.8.yaml new file mode 100644 index 00000000..3fb82abd --- /dev/null +++ b/hack/cni/eks-1.12.8.yaml @@ -0,0 +1,1618 @@ +--- +# Source: cilium/templates/cilium-agent/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/hubble-relay/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "hubble-relay" + namespace: kube-system +--- +# Source: cilium/templates/hubble-ui/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "hubble-ui" + namespace: kube-system +--- +# Source: cilium/templates/hubble/tls-cronjob/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "hubble-generate-certs" + namespace: kube-system +--- +# Source: cilium/templates/cilium-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-config + namespace: kube-system +data: + + # Identity allocation mode selects how identities are shared between cilium + # nodes by setting how they are stored. The options are "crd" or "kvstore". + # - "crd" stores identities in kubernetes as CRDs (custom resource definition). + # These can be queried with: + # kubectl get ciliumid + # - "kvstore" stores identities in an etcd kvstore, that is + # configured below. Cilium versions before 1.6 supported only the kvstore + # backend. Upgrades from these older cilium versions should continue using + # the kvstore by commenting out the identity-allocation-mode below, or + # setting it to "kvstore". + identity-allocation-mode: crd + cilium-endpoint-gc-interval: "5m0s" + nodes-gc-interval: "5m0s" + skip-cnp-status-startup-clean: "false" + # Disable the usage of CiliumEndpoint CRD + disable-endpoint-crd: "false" + + # If you want to run cilium in debug mode change this value to true + debug: "false" + # The agent can be put into the following three policy enforcement modes + # default, always and never. + # https://docs.cilium.io/en/latest/policy/intro/#policy-enforcement-modes + enable-policy: "default" + # If you want metrics enabled in all of your Cilium agents, set the port for + # which the Cilium agents will have their metrics exposed. + # This option deprecates the "prometheus-serve-addr" in the + # "cilium-metrics-config" ConfigMap + # NOTE that this will open the port on ALL nodes where Cilium pods are + # scheduled. + prometheus-serve-addr: ":9962" + # Port to expose Envoy metrics (e.g. "9964"). Envoy metrics listener will be disabled if this + # field is not set. + proxy-prometheus-port: "9964" + # If you want metrics enabled in cilium-operator, set the port for + # which the Cilium Operator will have their metrics exposed. + # NOTE that this will open the port on the nodes where Cilium operator pod + # is scheduled. + operator-prometheus-serve-addr: ":9963" + enable-metrics: "true" + + # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 + # address. + enable-ipv4: "true" + + # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 + # address. + enable-ipv6: "false" + # Users who wish to specify their own custom CNI configuration file must set + # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. + custom-cni-conf: "false" + enable-bpf-clock-probe: "true" + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation: medium + + # The monitor aggregation interval governs the typical time between monitor + # notification events for each allowed connection. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-interval: 5s + + # The monitor aggregation flags determine which TCP flags which, upon the + # first observation, cause monitor notifications to be generated. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-flags: all + # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic + # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. + bpf-map-dynamic-size-ratio: "0.0025" + # bpf-policy-map-max specifies the maximum number of entries in endpoint + # policy map (per endpoint) + bpf-policy-map-max: "16384" + # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, + # backend and affinity maps. + bpf-lb-map-max: "65536" + # bpf-lb-bypass-fib-lookup instructs Cilium to enable the FIB lookup bypass + # optimization for nodeport reverse NAT handling. + bpf-lb-external-clusterip: "false" + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # As a result, reply packets may be dropped and the load-balancing decisions + # for established connections may change. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps: "false" + + # Regular expression matching compatible Istio sidecar istio-proxy + # container image names + sidecar-istio-proxy-image: "cilium/istio_proxy" + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name: default + # Unique ID of the cluster. Must be unique across all conneted clusters and + # in the range of 1 and 255. Only relevant when building a mesh of clusters. + cluster-id: "0" + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve + tunnel: "vxlan" + # Enables L7 proxy for L7 policy enforcement and visibility + enable-l7-proxy: "true" + # Enable chaining with another CNI plugin + # + # Supported modes: + # - none + # - aws-cni + # - flannel + # - generic-veth + # - portmap (Enables HostPort support for Cilium) + cni-chaining-mode: portmap + + enable-ipv4-masquerade: "true" + enable-ipv6-masquerade: "true" + egress-masquerade-interfaces: eth0 + + enable-xt-socket-fallback: "true" + install-iptables-rules: "true" + install-no-conntrack-iptables-rules: "false" + + auto-direct-node-routes: "false" + enable-local-redirect-policy: "false" + + kube-proxy-replacement: "disabled" + bpf-lb-sock: "false" + enable-health-check-nodeport: "true" + node-port-bind-protection: "true" + enable-auto-protect-node-port-range: "true" + enable-session-affinity: "true" + enable-svc-source-range-check: "true" + enable-l2-neigh-discovery: "true" + arping-refresh-period: "30s" + cni-uninstall: "true" + enable-endpoint-health-checking: "true" + enable-health-checking: "true" + enable-well-known-identities: "false" + enable-remote-node-identity: "true" + synchronize-k8s-nodes: "true" + operator-api-serve-addr: "127.0.0.1:9234" + # Enable Hubble gRPC service. + enable-hubble: "true" + # UNIX domain socket for Hubble server to listen to. + hubble-socket-path: "/var/run/cilium/hubble.sock" + # Address to expose Hubble metrics (e.g. ":7070"). Metrics server will be disabled if this + # field is not set. + hubble-metrics-server: ":9965" + # A space separated list of metrics to enable. See [0] for available metrics. + # + # https://github.com/cilium/hubble/blob/master/Documentation/metrics.md + hubble-metrics: + dns:query;ignoreAAAA + drop + tcp + flow + icmp + http + # An additional address for Hubble server to listen to (e.g. ":4244"). + hubble-listen-address: ":4244" + hubble-disable-tls: "false" + hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt + hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key + hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt + ipam: "cluster-pool" + cluster-pool-ipv4-cidr: "10.0.0.0/8" + cluster-pool-ipv4-mask-size: "24" + disable-cnp-status-updates: "true" + enable-vtep: "false" + vtep-endpoint: "" + vtep-cidr: "" + vtep-mask: "" + vtep-mac: "" + enable-bgp-control-plane: "false" + procfs: "/host/proc" + bpf-root: "/sys/fs/bpf" + cgroup-root: "/run/cilium/cgroupv2" + enable-k8s-terminating-endpoint: "true" + remove-cilium-node-taints: "true" + set-cilium-is-up-condition: "true" + unmanaged-pod-watcher-interval: "15" + tofqdns-dns-reject-response-code: "refused" + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "50" + tofqdns-idle-connection-grace-period: "0s" + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-min-ttl: "3600" + tofqdns-proxy-response-max-delay: "100ms" + agent-not-ready-taint-key: "ignore-taint.cluster-autoscaler.kubernetes.io/cilium-agent-not-ready" +--- +# Source: cilium/templates/hubble-relay/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hubble-relay-config + namespace: kube-system +data: + config.yaml: | + cluster-name: default + peer-service: "hubble-peer.kube-system.svc.cluster.local:443" + listen-address: :4245 + metrics-listen-address: ":9966" + dial-timeout: + retry-timeout: + sort-buffer-len-max: + sort-buffer-drain-timeout: + tls-client-cert-file: /var/lib/hubble-relay/tls/client.crt + tls-client-key-file: /var/lib/hubble-relay/tls/client.key + tls-hubble-server-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt + disable-server-tls: true +--- +# Source: cilium/templates/hubble-ui/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hubble-ui-nginx + namespace: kube-system +data: + nginx.conf: "server {\n listen 8081;\n listen [::]:8081;\n server_name localhost;\n root /app;\n index index.html;\n client_max_body_size 1G;\n\n location / {\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n\n # CORS\n add_header Access-Control-Allow-Methods \"GET, POST, PUT, HEAD, DELETE, OPTIONS\";\n add_header Access-Control-Allow-Origin *;\n add_header Access-Control-Max-Age 1728000;\n add_header Access-Control-Expose-Headers content-length,grpc-status,grpc-message;\n add_header Access-Control-Allow-Headers range,keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout;\n if ($request_method = OPTIONS) {\n return 204;\n }\n # /CORS\n\n location /api {\n proxy_http_version 1.1;\n proxy_pass_request_headers on;\n proxy_hide_header Access-Control-Allow-Origin;\n proxy_pass http://127.0.0.1:8090;\n }\n\n location / {\n try_files $uri $uri/ /index.html;\n }\n }\n}" +--- +# Source: cilium/templates/cilium-agent/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - pods + - endpoints + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get +- apiGroups: + - cilium.io + resources: + - ciliumbgploadbalancerippools + - ciliumbgppeeringpolicies + - ciliumclusterwideenvoyconfigs + - ciliumclusterwidenetworkpolicies + - ciliumegressgatewaypolicies + - ciliumegressnatpolicies + - ciliumendpoints + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumidentities + - ciliumlocalredirectpolicies + - ciliumnetworkpolicies + - ciliumnodes + verbs: + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + - ciliumendpoints + - ciliumnodes + verbs: + - create +- apiGroups: + - cilium.io + # To synchronize garbage collection of such resources + resources: + - ciliumidentities + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + verbs: + - delete + - get +- apiGroups: + - cilium.io + resources: + - ciliumnodes + - ciliumnodes/status + verbs: + - get + - update +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + - ciliumendpoints/status + - ciliumendpoints + verbs: + - patch +--- +# Source: cilium/templates/cilium-operator/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-operator +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + # To remove node taints + - nodes + # To set NetworkUnavailable false on startup + - nodes/status + verbs: + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform LB IP allocation for BGP + - services/status + verbs: + - update +- apiGroups: + - "" + resources: + # to check apiserver connectivity + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumclusterwidenetworkpolicies + verbs: + # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' + - create + - update + - deletecollection + # To update the status of the CNPs and CCNPs + - patch + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + verbs: + # Update the auto-generated CNPs and CCNPs status. + - patch + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + - ciliumidentities + verbs: + # To perform garbage collection of such resources + - delete + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + verbs: + # To synchronize garbage collection of such resources + - update +- apiGroups: + - cilium.io + resources: + - ciliumnodes + verbs: + - create + - update + - get + - list + - watch + # To perform CiliumNode garbage collector + - delete +- apiGroups: + - cilium.io + resources: + - ciliumnodes/status + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpointslices + - ciliumenvoyconfigs + verbs: + - create + - update + - get + - list + - watch + - delete +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - update + resourceNames: + - ciliumbgploadbalancerippools.cilium.io + - ciliumbgppeeringpolicies.cilium.io + - ciliumclusterwideenvoyconfigs.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumegressgatewaypolicies.cilium.io + - ciliumegressnatpolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumenvoyconfigs.cilium.io + - ciliumexternalworkloads.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodes.cilium.io +# For cilium-operator running in HA mode. +# +# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election +# between multiple running instances. +# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less +# common and fewer objects in the cluster watch "all Leases". +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +--- +# Source: cilium/templates/hubble-ui/clusterrole.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-ui +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - componentstatuses + - endpoints + - namespaces + - nodes + - pods + - services + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - "*" + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/hubble/tls-cronjob/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hubble-generate-certs +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - hubble-server-certs + - hubble-relay-client-certs + - hubble-relay-server-certs + verbs: + - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - cilium-ca + - hubble-ca-secret + verbs: + - get + - update +--- +# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/hubble-ui/clusterrolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-ui +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hubble-ui +subjects: +- kind: ServiceAccount + name: "hubble-ui" + namespace: kube-system +--- +# Source: cilium/templates/hubble/tls-cronjob/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hubble-generate-certs +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hubble-generate-certs +subjects: +- kind: ServiceAccount + name: "hubble-generate-certs" + namespace: kube-system +--- +# Source: cilium/templates/cilium-agent/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: cilium-agent + namespace: kube-system + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9964" + labels: + k8s-app: cilium +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: cilium + ports: + - name: envoy-metrics + port: 9964 + protocol: TCP + targetPort: envoy-metrics +--- +# Source: cilium/templates/hubble-relay/metrics-service.yaml +# We use a separate service from hubble-relay which can be exposed externally +kind: Service +apiVersion: v1 +metadata: + name: hubble-relay-metrics + namespace: kube-system + labels: + k8s-app: hubble-relay +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: hubble-relay + ports: + - name: metrics + port: 9966 + protocol: TCP + targetPort: prometheus +--- +# Source: cilium/templates/hubble-relay/service.yaml +kind: Service +apiVersion: v1 +metadata: + name: hubble-relay + namespace: kube-system + labels: + k8s-app: hubble-relay +spec: + type: "ClusterIP" + selector: + k8s-app: hubble-relay + ports: + - protocol: TCP + port: 80 + targetPort: 4245 +--- +# Source: cilium/templates/hubble-ui/service.yaml +kind: Service +apiVersion: v1 +metadata: + name: hubble-ui + namespace: kube-system + labels: + k8s-app: hubble-ui +spec: + type: "ClusterIP" + selector: + k8s-app: hubble-ui + ports: + - name: http + port: 80 + targetPort: 8081 +--- +# Source: cilium/templates/hubble/metrics-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hubble-metrics + namespace: kube-system + labels: + k8s-app: hubble + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9965" +spec: + clusterIP: None + type: ClusterIP + ports: + - name: hubble-metrics + port: 9965 + protocol: TCP + targetPort: hubble-metrics + selector: + k8s-app: cilium +--- +# Source: cilium/templates/hubble/peer-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hubble-peer + namespace: kube-system + labels: + k8s-app: cilium +spec: + selector: + k8s-app: cilium + ports: + - name: peer-service + port: 443 + protocol: TCP + targetPort: 4244 + internalTrafficPolicy: Local +--- +# Source: cilium/templates/cilium-agent/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium + namespace: kube-system + labels: + k8s-app: cilium +spec: + selector: + matchLabels: + k8s-app: cilium + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9962" + prometheus.io/scrape: "true" + # Set app AppArmor's profile to "unconfined". The value of this annotation + # can be modified as long users know which profiles they have available + # in AppArmor. + container.apparmor.security.beta.kubernetes.io/cilium-agent: "unconfined" + container.apparmor.security.beta.kubernetes.io/clean-cilium-state: "unconfined" + container.apparmor.security.beta.kubernetes.io/mount-cgroup: "unconfined" + container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: "unconfined" + labels: + k8s-app: cilium + spec: + containers: + - name: cilium-agent + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + command: + - cilium-agent + args: + - --config-dir=/tmp/cilium/config-map + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + failureThreshold: 105 + periodSeconds: 2 + successThreshold: 1 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: CILIUM_CNI_CHAINING_MODE + valueFrom: + configMapKeyRef: + name: cilium-config + key: cni-chaining-mode + optional: true + - name: CILIUM_CUSTOM_CNI_CONF + valueFrom: + configMapKeyRef: + name: cilium-config + key: custom-cni-conf + optional: true + lifecycle: + postStart: + exec: + command: + - "/cni-install.sh" + - "--enable-debug=false" + - "--cni-exclusive=true" + - "--log-file=/var/run/cilium/cilium-cni.log" + preStop: + exec: + command: + - /cni-uninstall.sh + ports: + - name: peer-service + containerPort: 4244 + hostPort: 4244 + protocol: TCP + - name: prometheus + containerPort: 9962 + hostPort: 9962 + protocol: TCP + - name: envoy-metrics + containerPort: 9964 + hostPort: 9964 + protocol: TCP + - name: hubble-metrics + containerPort: 9965 + hostPort: 9965 + protocol: TCP + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + # Use to set socket permission + - CHOWN + # Used to terminate envoy child process + - KILL + # Used since cilium modifies routing tables, etc... + - NET_ADMIN + # Used since cilium creates raw sockets, etc... + - NET_RAW + # Used since cilium monitor uses mmap + - IPC_LOCK + # Used in iptables. Consider removing once we are iptables-free + - SYS_MODULE + # We need it for now but might not need it for >= 5.11 specially + # for the 'SYS_RESOURCE'. + # In >= 5.8 there's already BPF and PERMON capabilities + - SYS_ADMIN + # Could be an alternative for the SYS_ADMIN for the RLIMIT_NPROC + - SYS_RESOURCE + # Both PERFMON and BPF requires kernel 5.8, container runtime + # cri-o >= v1.22.0 or containerd >= v1.5.0. + # If available, SYS_ADMIN can be removed. + #- PERFMON + #- BPF + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # Unprivileged containers need to mount /proc/sys/net from the host + # to have write access + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + # Unprivileged containers need to mount /proc/sys/kernel from the host + # to have write access + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - name: bpf-maps + mountPath: /sys/fs/bpf + # Unprivileged containers can't set mount propagation to bidirectional + # in this case we will mount the bpf fs from an init container that + # is privileged and set the mount propagation from host to container + # in Cilium. + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium + - name: etc-cni-netd + mountPath: /host/etc/cni/net.d + - name: clustermesh-secrets + mountPath: /var/lib/cilium/clustermesh + readOnly: true + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + # Needed to be able to load kernel modules + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + - name: hubble-tls + mountPath: /var/lib/cilium/tls/hubble + readOnly: true + initContainers: + # Required to mount cgroup2 filesystem on the underlying Kubernetes node. + # We use nsenter command with host's cgroup and mount namespaces enabled. + - name: mount-cgroup + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + env: + - name: CGROUP_ROOT + value: /run/cilium/cgroupv2 + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh and mount that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + add: + # Only used for 'mount' cgroup + - SYS_ADMIN + # Used for nsenter + - SYS_CHROOT + - SYS_PTRACE + - name: apply-sysctl-overwrites + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + env: + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + add: + # Required in order to access host's /etc/sysctl.d dir + - SYS_ADMIN + # Used for nsenter + - SYS_CHROOT + - SYS_PTRACE + # Mount the bpf fs if it is not mounted. We will perform this task + # from a privileged container because the mount propagation bidirectional + # only works from privileged containers. + - name: mount-bpf-fs + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + args: + - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' + command: + - /bin/bash + - -c + - -- + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + privileged: true + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + - name: clean-cilium-state + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-state + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-bpf-state + optional: true + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + # Most of the capabilities here are the same ones used in the + # cilium-agent's container because this container can be used to + # uninstall all Cilium resources, and therefore it is likely that + # will need the same capabilities. + add: + # Used since cilium modifies routing tables, etc... + - NET_ADMIN + # Used in iptables. Consider removing once we are iptables-free + - SYS_MODULE + # We need it for now but might not need it for >= 5.11 specially + # for the 'SYS_RESOURCE'. + # In >= 5.8 there's already BPF and PERMON capabilities + - SYS_ADMIN + # Could be an alternative for the SYS_ADMIN for the RLIMIT_NPROC + - SYS_RESOURCE + # Both PERFMON and BPF requires kernel 5.8, container runtime + # cri-o >= v1.22.0 or containerd >= v1.5.0. + # If available, SYS_ADMIN can be removed. + #- PERFMON + #- BPF + drop: + - ALL + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + # Required to mount cgroup filesystem from the host to cilium agent pod + - name: cilium-cgroup + mountPath: /run/cilium/cgroupv2 + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium + resources: + requests: + cpu: 100m + memory: 100Mi # wait-for-kube-proxy + # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent + - name: install-cni-binaries + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + command: + - "/install-plugin.sh" + resources: + requests: + cpu: 100m + memory: 10Mi + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cni-path + mountPath: /host/opt/cni/bin + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccount: "cilium" + serviceAccountName: "cilium" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + # To mount cgroup2 filesystem on the host + - name: hostproc + hostPath: + path: /proc + type: Directory + # To keep state between restarts / upgrades for cgroup2 filesystem + - name: cilium-cgroup + hostPath: + path: /run/cilium/cgroupv2 + type: DirectoryOrCreate + # To install cilium cni plugin in the host + - name: cni-path + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + # To install cilium cni configuration in the host + - name: etc-cni-netd + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + # To be able to load kernel modules + - name: lib-modules + hostPath: + path: /lib/modules + # To access iptables concurrently with other processes (e.g. kube-proxy) + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # To read the clustermesh configuration + - name: clustermesh-secrets + secret: + secretName: cilium-clustermesh + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + optional: true + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config + - name: host-proc-sys-net + hostPath: + path: /proc/sys/net + type: Directory + - name: host-proc-sys-kernel + hostPath: + path: /proc/sys/kernel + type: Directory + - name: hubble-tls + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-server-certs + optional: true + items: + - key: ca.crt + path: client-ca.crt + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key +--- +# Source: cilium/templates/cilium-operator/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cilium-operator + namespace: kube-system + labels: + io.cilium/app: operator + name: cilium-operator +spec: + # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go + # for more details. + replicas: 2 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9963" + prometheus.io/scrape: "true" + labels: + io.cilium/app: operator + name: cilium-operator + spec: + containers: + - name: cilium-operator + image: "quay.io/cilium/operator-generic:v1.12.8@sha256:7431f0c2001fb875b1a8901e103825394c38cd6c63a1435a3273ed20ae0e7578" + imagePullPolicy: IfNotPresent + command: + - cilium-operator-generic + args: + - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + ports: + - name: prometheus + containerPort: 9963 + hostPort: 9963 + protocol: TCP + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + volumeMounts: + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + terminationMessagePolicy: FallbackToLogsOnError + hostNetwork: true + restartPolicy: Always + priorityClassName: system-cluster-critical + serviceAccount: "cilium-operator" + serviceAccountName: "cilium-operator" + automountServiceAccountToken: true + # In HA mode, cilium-operator pods must not be scheduled on the same + # node as they will clash with each other. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + io.cilium/app: operator + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config +--- +# Source: cilium/templates/hubble-relay/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hubble-relay + namespace: kube-system + labels: + k8s-app: hubble-relay +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: hubble-relay + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + labels: + k8s-app: hubble-relay + spec: + containers: + - name: hubble-relay + image: "quay.io/cilium/hubble-relay:v1.12.8@sha256:508cf85bb1a11c13abd995e3c5fd18ed3c2f1d26cbf463a97297e8b8c9149f13" + imagePullPolicy: IfNotPresent + command: + - hubble-relay + args: + - serve + ports: + - name: grpc + containerPort: 4245 + - name: prometheus + containerPort: 9966 + protocol: TCP + readinessProbe: + tcpSocket: + port: grpc + livenessProbe: + tcpSocket: + port: grpc + volumeMounts: + - name: config + mountPath: /etc/hubble-relay + readOnly: true + - name: tls + mountPath: /var/lib/hubble-relay/tls + readOnly: true + terminationMessagePolicy: FallbackToLogsOnError + restartPolicy: Always + priorityClassName: + serviceAccount: "hubble-relay" + serviceAccountName: "hubble-relay" + automountServiceAccountToken: false + terminationGracePeriodSeconds: 1 + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + volumes: + - name: config + configMap: + name: hubble-relay-config + items: + - key: config.yaml + path: config.yaml + - name: tls + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-relay-client-certs + items: + - key: ca.crt + path: hubble-server-ca.crt + - key: tls.crt + path: client.crt + - key: tls.key + path: client.key +--- +# Source: cilium/templates/hubble-ui/deployment.yaml +kind: Deployment +apiVersion: apps/v1 +metadata: + name: hubble-ui + namespace: kube-system + labels: + k8s-app: hubble-ui +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: hubble-ui + template: + metadata: + annotations: + labels: + k8s-app: hubble-ui + spec: + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsUser: 1001 + priorityClassName: + serviceAccount: "hubble-ui" + serviceAccountName: "hubble-ui" + automountServiceAccountToken: true + containers: + - name: frontend + image: "quay.io/cilium/hubble-ui:v0.9.2@sha256:d3596efc94a41c6b772b9afe6fe47c17417658956e04c3e2a28d293f2670663e" + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8081 + volumeMounts: + - name: hubble-ui-nginx-conf + mountPath: /etc/nginx/conf.d/default.conf + subPath: nginx.conf + - name: tmp-dir + mountPath: /tmp + terminationMessagePolicy: FallbackToLogsOnError + - name: backend + image: "quay.io/cilium/hubble-ui-backend:v0.9.2@sha256:a3ac4d5b87889c9f7cc6323e86d3126b0d382933bd64f44382a92778b0cde5d7" + imagePullPolicy: IfNotPresent + env: + - name: EVENTS_SERVER_PORT + value: "8090" + - name: FLOWS_API_ADDR + value: "hubble-relay:80" + ports: + - name: grpc + containerPort: 8090 + volumeMounts: + terminationMessagePolicy: FallbackToLogsOnError + nodeSelector: + kubernetes.io/os: linux + volumes: + - configMap: + defaultMode: 420 + name: hubble-ui-nginx + name: hubble-ui-nginx-conf + - emptyDir: {} + name: tmp-dir +--- +# Source: cilium/templates/hubble/tls-cronjob/job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: hubble-generate-certs-c26869f410 + namespace: kube-system + labels: + k8s-app: hubble-generate-certs +spec: + template: + metadata: + labels: + k8s-app: hubble-generate-certs + spec: + containers: + - name: certgen + image: "quay.io/cilium/certgen:v0.1.8@sha256:4a456552a5f192992a6edcec2febb1c54870d665173a33dc7d876129b199ddbd" + imagePullPolicy: IfNotPresent + command: + - "/usr/bin/cilium-certgen" + # Because this is executed as a job, we pass the values as command + # line args instead of via config map. This allows users to inspect + # the values used in past runs by inspecting the completed pod. + args: + - "--cilium-namespace=kube-system" + - "--ca-generate" + - "--ca-reuse-secret" + - "--hubble-server-cert-generate" + - "--hubble-server-cert-common-name=*.default.hubble-grpc.cilium.io" + - "--hubble-server-cert-validity-duration=94608000s" + - "--hubble-relay-client-cert-generate" + - "--hubble-relay-client-cert-validity-duration=94608000s" + hostNetwork: true + serviceAccount: "hubble-generate-certs" + serviceAccountName: "hubble-generate-certs" + automountServiceAccountToken: true + restartPolicy: OnFailure + ttlSecondsAfterFinished: 1800 +--- +# Source: cilium/templates/hubble/tls-cronjob/cronjob.yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: hubble-generate-certs + namespace: kube-system + labels: + k8s-app: hubble-generate-certs +spec: + schedule: "0 0 1 */4 *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + metadata: + labels: + k8s-app: hubble-generate-certs + spec: + containers: + - name: certgen + image: "quay.io/cilium/certgen:v0.1.8@sha256:4a456552a5f192992a6edcec2febb1c54870d665173a33dc7d876129b199ddbd" + imagePullPolicy: IfNotPresent + command: + - "/usr/bin/cilium-certgen" + # Because this is executed as a job, we pass the values as command + # line args instead of via config map. This allows users to inspect + # the values used in past runs by inspecting the completed pod. + args: + - "--cilium-namespace=kube-system" + - "--ca-generate" + - "--ca-reuse-secret" + - "--hubble-server-cert-generate" + - "--hubble-server-cert-common-name=*.default.hubble-grpc.cilium.io" + - "--hubble-server-cert-validity-duration=94608000s" + - "--hubble-relay-client-cert-generate" + - "--hubble-relay-client-cert-validity-duration=94608000s" + hostNetwork: true + serviceAccount: "hubble-generate-certs" + serviceAccountName: "hubble-generate-certs" + automountServiceAccountToken: true + restartPolicy: OnFailure + ttlSecondsAfterFinished: 1800 diff --git a/hack/cni/eks-values.yaml b/hack/cni/eks-values.yaml new file mode 100755 index 00000000..9e04e298 --- /dev/null +++ b/hack/cni/eks-values.yaml @@ -0,0 +1 @@ +egressMasqueradeInterfaces: eth0 diff --git a/hack/cni/gke-1.12.8.yaml b/hack/cni/gke-1.12.8.yaml new file mode 100644 index 00000000..04785c9a --- /dev/null +++ b/hack/cni/gke-1.12.8.yaml @@ -0,0 +1,1847 @@ +--- +# Source: cilium/templates/cilium-agent/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/hubble-relay/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "hubble-relay" + namespace: kube-system +--- +# Source: cilium/templates/hubble-ui/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "hubble-ui" + namespace: kube-system +--- +# Source: cilium/templates/hubble/tls-cronjob/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "hubble-generate-certs" + namespace: kube-system +--- +# Source: cilium/templates/cilium-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-config + namespace: kube-system +data: + + # Identity allocation mode selects how identities are shared between cilium + # nodes by setting how they are stored. The options are "crd" or "kvstore". + # - "crd" stores identities in kubernetes as CRDs (custom resource definition). + # These can be queried with: + # kubectl get ciliumid + # - "kvstore" stores identities in an etcd kvstore, that is + # configured below. Cilium versions before 1.6 supported only the kvstore + # backend. Upgrades from these older cilium versions should continue using + # the kvstore by commenting out the identity-allocation-mode below, or + # setting it to "kvstore". + identity-allocation-mode: crd + cilium-endpoint-gc-interval: "5m0s" + nodes-gc-interval: "5m0s" + skip-cnp-status-startup-clean: "false" + # Disable the usage of CiliumEndpoint CRD + disable-endpoint-crd: "false" + + # If you want to run cilium in debug mode change this value to true + debug: "false" + # The agent can be put into the following three policy enforcement modes + # default, always and never. + # https://docs.cilium.io/en/latest/policy/intro/#policy-enforcement-modes + enable-policy: "default" + # If you want metrics enabled in all of your Cilium agents, set the port for + # which the Cilium agents will have their metrics exposed. + # This option deprecates the "prometheus-serve-addr" in the + # "cilium-metrics-config" ConfigMap + # NOTE that this will open the port on ALL nodes where Cilium pods are + # scheduled. + prometheus-serve-addr: ":9962" + # Port to expose Envoy metrics (e.g. "9964"). Envoy metrics listener will be disabled if this + # field is not set. + proxy-prometheus-port: "9964" + # If you want metrics enabled in cilium-operator, set the port for + # which the Cilium Operator will have their metrics exposed. + # NOTE that this will open the port on the nodes where Cilium operator pod + # is scheduled. + operator-prometheus-serve-addr: ":9963" + enable-metrics: "true" + + # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 + # address. + enable-ipv4: "true" + + # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 + # address. + enable-ipv6: "false" + # Users who wish to specify their own custom CNI configuration file must set + # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. + custom-cni-conf: "false" + enable-bpf-clock-probe: "true" + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation: medium + + # The monitor aggregation interval governs the typical time between monitor + # notification events for each allowed connection. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-interval: 5s + + # The monitor aggregation flags determine which TCP flags which, upon the + # first observation, cause monitor notifications to be generated. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-flags: all + # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic + # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. + bpf-map-dynamic-size-ratio: "0.0025" + # bpf-policy-map-max specifies the maximum number of entries in endpoint + # policy map (per endpoint) + bpf-policy-map-max: "16384" + # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, + # backend and affinity maps. + bpf-lb-map-max: "65536" + # bpf-lb-bypass-fib-lookup instructs Cilium to enable the FIB lookup bypass + # optimization for nodeport reverse NAT handling. + bpf-lb-external-clusterip: "false" + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # As a result, reply packets may be dropped and the load-balancing decisions + # for established connections may change. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps: "false" + + # Regular expression matching compatible Istio sidecar istio-proxy + # container image names + sidecar-istio-proxy-image: "cilium/istio_proxy" + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name: default + # Unique ID of the cluster. Must be unique across all conneted clusters and + # in the range of 1 and 255. Only relevant when building a mesh of clusters. + cluster-id: "0" + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve + tunnel: "disabled" + enable-endpoint-routes: "true" + enable-local-node-route: "false" + # Enables L7 proxy for L7 policy enforcement and visibility + enable-l7-proxy: "true" + # Enable chaining with another CNI plugin + # + # Supported modes: + # - none + # - aws-cni + # - flannel + # - generic-veth + # - portmap (Enables HostPort support for Cilium) + cni-chaining-mode: portmap + + enable-ipv4-masquerade: "true" + enable-ipv6-masquerade: "true" + + enable-xt-socket-fallback: "true" + install-iptables-rules: "true" + install-no-conntrack-iptables-rules: "false" + + auto-direct-node-routes: "false" + enable-local-redirect-policy: "false" + ipv4-native-routing-cidr: 10.0.0.0/16 + + kube-proxy-replacement: "disabled" + bpf-lb-sock: "false" + enable-health-check-nodeport: "true" + node-port-bind-protection: "true" + enable-auto-protect-node-port-range: "true" + enable-session-affinity: "true" + enable-svc-source-range-check: "true" + enable-l2-neigh-discovery: "true" + arping-refresh-period: "30s" + cni-uninstall: "true" + enable-endpoint-health-checking: "true" + enable-health-checking: "true" + enable-well-known-identities: "false" + enable-remote-node-identity: "true" + synchronize-k8s-nodes: "true" + operator-api-serve-addr: "127.0.0.1:9234" + # Enable Hubble gRPC service. + enable-hubble: "true" + # UNIX domain socket for Hubble server to listen to. + hubble-socket-path: "/var/run/cilium/hubble.sock" + # Address to expose Hubble metrics (e.g. ":7070"). Metrics server will be disabled if this + # field is not set. + hubble-metrics-server: ":9965" + # A space separated list of metrics to enable. See [0] for available metrics. + # + # https://github.com/cilium/hubble/blob/master/Documentation/metrics.md + hubble-metrics: + dns:query;ignoreAAAA + drop + tcp + flow + icmp + http + # An additional address for Hubble server to listen to (e.g. ":4244"). + hubble-listen-address: ":4244" + hubble-disable-tls: "false" + hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt + hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key + hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt + ipam: "kubernetes" + disable-cnp-status-updates: "true" + enable-vtep: "false" + vtep-endpoint: "" + vtep-cidr: "" + vtep-mask: "" + vtep-mac: "" + enable-bgp-control-plane: "false" + procfs: "/host/proc" + bpf-root: "/sys/fs/bpf" + cgroup-root: "/run/cilium/cgroupv2" + enable-k8s-terminating-endpoint: "true" + remove-cilium-node-taints: "true" + set-cilium-is-up-condition: "true" + unmanaged-pod-watcher-interval: "15" + tofqdns-dns-reject-response-code: "refused" + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "50" + tofqdns-idle-connection-grace-period: "0s" + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-min-ttl: "3600" + tofqdns-proxy-response-max-delay: "100ms" + agent-not-ready-taint-key: "ignore-taint.cluster-autoscaler.kubernetes.io/cilium-agent-not-ready" +--- +# Source: cilium/templates/hubble-relay/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hubble-relay-config + namespace: kube-system +data: + config.yaml: | + cluster-name: default + peer-service: "hubble-peer.kube-system.svc.cluster.local:443" + listen-address: :4245 + metrics-listen-address: ":9966" + dial-timeout: + retry-timeout: + sort-buffer-len-max: + sort-buffer-drain-timeout: + tls-client-cert-file: /var/lib/hubble-relay/tls/client.crt + tls-client-key-file: /var/lib/hubble-relay/tls/client.key + tls-hubble-server-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt + disable-server-tls: true +--- +# Source: cilium/templates/hubble-ui/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hubble-ui-nginx + namespace: kube-system +data: + nginx.conf: "server {\n listen 8081;\n listen [::]:8081;\n server_name localhost;\n root /app;\n index index.html;\n client_max_body_size 1G;\n\n location / {\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n\n # CORS\n add_header Access-Control-Allow-Methods \"GET, POST, PUT, HEAD, DELETE, OPTIONS\";\n add_header Access-Control-Allow-Origin *;\n add_header Access-Control-Max-Age 1728000;\n add_header Access-Control-Expose-Headers content-length,grpc-status,grpc-message;\n add_header Access-Control-Allow-Headers range,keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout;\n if ($request_method = OPTIONS) {\n return 204;\n }\n # /CORS\n\n location /api {\n proxy_http_version 1.1;\n proxy_pass_request_headers on;\n proxy_hide_header Access-Control-Allow-Origin;\n proxy_pass http://127.0.0.1:8090;\n }\n\n location / {\n try_files $uri $uri/ /index.html;\n }\n }\n}" +--- +# Source: cilium/templates/cilium-agent/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - pods + - endpoints + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get +- apiGroups: + - cilium.io + resources: + - ciliumbgploadbalancerippools + - ciliumbgppeeringpolicies + - ciliumclusterwideenvoyconfigs + - ciliumclusterwidenetworkpolicies + - ciliumegressgatewaypolicies + - ciliumegressnatpolicies + - ciliumendpoints + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumidentities + - ciliumlocalredirectpolicies + - ciliumnetworkpolicies + - ciliumnodes + verbs: + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + - ciliumendpoints + - ciliumnodes + verbs: + - create +- apiGroups: + - cilium.io + # To synchronize garbage collection of such resources + resources: + - ciliumidentities + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + verbs: + - delete + - get +- apiGroups: + - cilium.io + resources: + - ciliumnodes + - ciliumnodes/status + verbs: + - get + - update +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + - ciliumendpoints/status + - ciliumendpoints + verbs: + - patch +--- +# Source: cilium/templates/cilium-operator/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-operator +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + # To remove node taints + - nodes + # To set NetworkUnavailable false on startup + - nodes/status + verbs: + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform LB IP allocation for BGP + - services/status + verbs: + - update +- apiGroups: + - "" + resources: + # to check apiserver connectivity + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumclusterwidenetworkpolicies + verbs: + # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' + - create + - update + - deletecollection + # To update the status of the CNPs and CCNPs + - patch + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + verbs: + # Update the auto-generated CNPs and CCNPs status. + - patch + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + - ciliumidentities + verbs: + # To perform garbage collection of such resources + - delete + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + verbs: + # To synchronize garbage collection of such resources + - update +- apiGroups: + - cilium.io + resources: + - ciliumnodes + verbs: + - create + - update + - get + - list + - watch + # To perform CiliumNode garbage collector + - delete +- apiGroups: + - cilium.io + resources: + - ciliumnodes/status + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpointslices + - ciliumenvoyconfigs + verbs: + - create + - update + - get + - list + - watch + - delete +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - update + resourceNames: + - ciliumbgploadbalancerippools.cilium.io + - ciliumbgppeeringpolicies.cilium.io + - ciliumclusterwideenvoyconfigs.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumegressgatewaypolicies.cilium.io + - ciliumegressnatpolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumenvoyconfigs.cilium.io + - ciliumexternalworkloads.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodes.cilium.io +# For cilium-operator running in HA mode. +# +# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election +# between multiple running instances. +# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less +# common and fewer objects in the cluster watch "all Leases". +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +--- +# Source: cilium/templates/hubble-ui/clusterrole.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-ui +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - componentstatuses + - endpoints + - namespaces + - nodes + - pods + - services + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - "*" + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/hubble/tls-cronjob/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hubble-generate-certs +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - hubble-server-certs + - hubble-relay-client-certs + - hubble-relay-server-certs + verbs: + - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - cilium-ca + - hubble-ca-secret + verbs: + - get + - update +--- +# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/hubble-ui/clusterrolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-ui +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hubble-ui +subjects: +- kind: ServiceAccount + name: "hubble-ui" + namespace: kube-system +--- +# Source: cilium/templates/hubble/tls-cronjob/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hubble-generate-certs +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hubble-generate-certs +subjects: +- kind: ServiceAccount + name: "hubble-generate-certs" + namespace: kube-system +--- +# Source: cilium/templates/cilium-agent/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: cilium-agent + namespace: kube-system + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9964" + labels: + k8s-app: cilium +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: cilium + ports: + - name: envoy-metrics + port: 9964 + protocol: TCP + targetPort: envoy-metrics +--- +# Source: cilium/templates/hubble-relay/metrics-service.yaml +# We use a separate service from hubble-relay which can be exposed externally +kind: Service +apiVersion: v1 +metadata: + name: hubble-relay-metrics + namespace: kube-system + labels: + k8s-app: hubble-relay +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: hubble-relay + ports: + - name: metrics + port: 9966 + protocol: TCP + targetPort: prometheus +--- +# Source: cilium/templates/hubble-relay/service.yaml +kind: Service +apiVersion: v1 +metadata: + name: hubble-relay + namespace: kube-system + labels: + k8s-app: hubble-relay +spec: + type: "ClusterIP" + selector: + k8s-app: hubble-relay + ports: + - protocol: TCP + port: 80 + targetPort: 4245 +--- +# Source: cilium/templates/hubble-ui/service.yaml +kind: Service +apiVersion: v1 +metadata: + name: hubble-ui + namespace: kube-system + labels: + k8s-app: hubble-ui +spec: + type: "ClusterIP" + selector: + k8s-app: hubble-ui + ports: + - name: http + port: 80 + targetPort: 8081 +--- +# Source: cilium/templates/hubble/metrics-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hubble-metrics + namespace: kube-system + labels: + k8s-app: hubble + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9965" +spec: + clusterIP: None + type: ClusterIP + ports: + - name: hubble-metrics + port: 9965 + protocol: TCP + targetPort: hubble-metrics + selector: + k8s-app: cilium +--- +# Source: cilium/templates/hubble/peer-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hubble-peer + namespace: kube-system + labels: + k8s-app: cilium +spec: + selector: + k8s-app: cilium + ports: + - name: peer-service + port: 443 + protocol: TCP + targetPort: 4244 + internalTrafficPolicy: Local +--- +# Source: cilium/templates/cilium-agent/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium + namespace: kube-system + labels: + k8s-app: cilium +spec: + selector: + matchLabels: + k8s-app: cilium + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9962" + prometheus.io/scrape: "true" + # Set app AppArmor's profile to "unconfined". The value of this annotation + # can be modified as long users know which profiles they have available + # in AppArmor. + container.apparmor.security.beta.kubernetes.io/cilium-agent: "unconfined" + container.apparmor.security.beta.kubernetes.io/clean-cilium-state: "unconfined" + container.apparmor.security.beta.kubernetes.io/mount-cgroup: "unconfined" + container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: "unconfined" + labels: + k8s-app: cilium + spec: + containers: + - name: cilium-agent + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + command: + - cilium-agent + args: + - --config-dir=/tmp/cilium/config-map + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + failureThreshold: 105 + periodSeconds: 2 + successThreshold: 1 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: CILIUM_CNI_CHAINING_MODE + valueFrom: + configMapKeyRef: + name: cilium-config + key: cni-chaining-mode + optional: true + - name: CILIUM_CUSTOM_CNI_CONF + valueFrom: + configMapKeyRef: + name: cilium-config + key: custom-cni-conf + optional: true + lifecycle: + postStart: + exec: + command: + - "/cni-install.sh" + - "--enable-debug=false" + - "--cni-exclusive=true" + - "--log-file=/var/run/cilium/cilium-cni.log" + preStop: + exec: + command: + - /cni-uninstall.sh + ports: + - name: peer-service + containerPort: 4244 + hostPort: 4244 + protocol: TCP + - name: prometheus + containerPort: 9962 + hostPort: 9962 + protocol: TCP + - name: envoy-metrics + containerPort: 9964 + hostPort: 9964 + protocol: TCP + - name: hubble-metrics + containerPort: 9965 + hostPort: 9965 + protocol: TCP + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + # Use to set socket permission + - CHOWN + # Used to terminate envoy child process + - KILL + # Used since cilium modifies routing tables, etc... + - NET_ADMIN + # Used since cilium creates raw sockets, etc... + - NET_RAW + # Used since cilium monitor uses mmap + - IPC_LOCK + # Used in iptables. Consider removing once we are iptables-free + - SYS_MODULE + # We need it for now but might not need it for >= 5.11 specially + # for the 'SYS_RESOURCE'. + # In >= 5.8 there's already BPF and PERMON capabilities + - SYS_ADMIN + # Could be an alternative for the SYS_ADMIN for the RLIMIT_NPROC + - SYS_RESOURCE + # Both PERFMON and BPF requires kernel 5.8, container runtime + # cri-o >= v1.22.0 or containerd >= v1.5.0. + # If available, SYS_ADMIN can be removed. + #- PERFMON + #- BPF + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # Unprivileged containers need to mount /proc/sys/net from the host + # to have write access + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + # Unprivileged containers need to mount /proc/sys/kernel from the host + # to have write access + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - name: bpf-maps + mountPath: /sys/fs/bpf + # Unprivileged containers can't set mount propagation to bidirectional + # in this case we will mount the bpf fs from an init container that + # is privileged and set the mount propagation from host to container + # in Cilium. + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium + - name: etc-cni-netd + mountPath: /host/etc/cni/net.d + - name: clustermesh-secrets + mountPath: /var/lib/cilium/clustermesh + readOnly: true + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + # Needed to be able to load kernel modules + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + - name: hubble-tls + mountPath: /var/lib/cilium/tls/hubble + readOnly: true + initContainers: + # Required to mount cgroup2 filesystem on the underlying Kubernetes node. + # We use nsenter command with host's cgroup and mount namespaces enabled. + - name: mount-cgroup + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + env: + - name: CGROUP_ROOT + value: /run/cilium/cgroupv2 + - name: BIN_PATH + value: /home/kubernetes/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh and mount that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + add: + # Only used for 'mount' cgroup + - SYS_ADMIN + # Used for nsenter + - SYS_CHROOT + - SYS_PTRACE + - name: apply-sysctl-overwrites + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + env: + - name: BIN_PATH + value: /home/kubernetes/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + add: + # Required in order to access host's /etc/sysctl.d dir + - SYS_ADMIN + # Used for nsenter + - SYS_CHROOT + - SYS_PTRACE + # Mount the bpf fs if it is not mounted. We will perform this task + # from a privileged container because the mount propagation bidirectional + # only works from privileged containers. + - name: mount-bpf-fs + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + args: + - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' + command: + - /bin/bash + - -c + - -- + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + privileged: true + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + - name: wait-for-node-init + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + until test -s "/tmp/cilium-bootstrap.d/cilium-bootstrap-time"; do + echo "Waiting on node-init to run..."; + sleep 1; + done + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cilium-bootstrap-file-dir + mountPath: "/tmp/cilium-bootstrap.d" + - name: clean-cilium-state + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-state + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-bpf-state + optional: true + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + # Most of the capabilities here are the same ones used in the + # cilium-agent's container because this container can be used to + # uninstall all Cilium resources, and therefore it is likely that + # will need the same capabilities. + add: + # Used since cilium modifies routing tables, etc... + - NET_ADMIN + # Used in iptables. Consider removing once we are iptables-free + - SYS_MODULE + # We need it for now but might not need it for >= 5.11 specially + # for the 'SYS_RESOURCE'. + # In >= 5.8 there's already BPF and PERMON capabilities + - SYS_ADMIN + # Could be an alternative for the SYS_ADMIN for the RLIMIT_NPROC + - SYS_RESOURCE + # Both PERFMON and BPF requires kernel 5.8, container runtime + # cri-o >= v1.22.0 or containerd >= v1.5.0. + # If available, SYS_ADMIN can be removed. + #- PERFMON + #- BPF + drop: + - ALL + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + # Required to mount cgroup filesystem from the host to cilium agent pod + - name: cilium-cgroup + mountPath: /run/cilium/cgroupv2 + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium + resources: + requests: + cpu: 100m + memory: 100Mi # wait-for-kube-proxy + # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent + - name: install-cni-binaries + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + command: + - "/install-plugin.sh" + resources: + requests: + cpu: 100m + memory: 10Mi + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cni-path + mountPath: /host/opt/cni/bin + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccount: "cilium" + serviceAccountName: "cilium" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + # To mount cgroup2 filesystem on the host + - name: hostproc + hostPath: + path: /proc + type: Directory + # To keep state between restarts / upgrades for cgroup2 filesystem + - name: cilium-cgroup + hostPath: + path: /run/cilium/cgroupv2 + type: DirectoryOrCreate + # To install cilium cni plugin in the host + - name: cni-path + hostPath: + path: /home/kubernetes/bin + type: DirectoryOrCreate + # To install cilium cni configuration in the host + - name: etc-cni-netd + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + # To be able to load kernel modules + - name: lib-modules + hostPath: + path: /lib/modules + # To access iptables concurrently with other processes (e.g. kube-proxy) + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: cilium-bootstrap-file-dir + hostPath: + path: "/tmp/cilium-bootstrap.d" + type: DirectoryOrCreate + # To read the clustermesh configuration + - name: clustermesh-secrets + secret: + secretName: cilium-clustermesh + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + optional: true + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config + - name: host-proc-sys-net + hostPath: + path: /proc/sys/net + type: Directory + - name: host-proc-sys-kernel + hostPath: + path: /proc/sys/kernel + type: Directory + - name: hubble-tls + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-server-certs + optional: true + items: + - key: ca.crt + path: client-ca.crt + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key +--- +# Source: cilium/templates/cilium-nodeinit/daemonset.yaml +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: cilium-node-init + namespace: kube-system + labels: + app: cilium-node-init +spec: + selector: + matchLabels: + app: cilium-node-init + updateStrategy: + type: RollingUpdate + template: + metadata: + annotations: + # Set app AppArmor's profile to "unconfined". The value of this annotation + # can be modified as long users know which profiles they have available + # in AppArmor. + container.apparmor.security.beta.kubernetes.io/node-init: "unconfined" + labels: + app: cilium-node-init + spec: + containers: + - name: node-init + image: "quay.io/cilium/startup-script:d69851597ea019af980891a4628fb36b7880ec26" + imagePullPolicy: IfNotPresent + lifecycle: + env: + # STARTUP_SCRIPT is the script run on node bootstrap. Node + # bootstrapping can be customized in this script. This script is invoked + # using nsenter, so it runs in the host's network and mount namespace using + # the host's userland tools! + - name: STARTUP_SCRIPT + value: | + #!/bin/bash + + set -o errexit + set -o pipefail + set -o nounset + + echo "Link information:" + ip link + + echo "Routing table:" + ip route + + echo "Addressing:" + ip -4 a + ip -6 a + if ip link show cbr0; then + echo "Detected cbr0 bridge. Deleting interface..." + ip link del cbr0 + fi + # Check if we're running on a GKE containerd flavor as indicated by the presence + # of the '--container-runtime-endpoint' flag in '/etc/default/kubelet'. + GKE_KUBERNETES_BIN_DIR="/home/kubernetes/bin" + KUBELET_DEFAULTS_FILE="/etc/default/kubelet" + if [[ -f "${GKE_KUBERNETES_BIN_DIR}/gke" ]] && [[ $(grep -cF -- '--container-runtime-endpoint' "${KUBELET_DEFAULTS_FILE}") == "1" ]]; then + echo "GKE *_containerd flavor detected..." + + # (GKE *_containerd) Upon node restarts, GKE's containerd images seem to reset + # the /etc directory and our changes to the kubelet and Cilium's CNI + # configuration are removed. This leaves room for containerd and its CNI to + # take over pods previously managed by Cilium, causing Cilium to lose + # ownership over these pods. We rely on the empirical observation that + # /home/kubernetes/bin/kubelet is not changed across node reboots, and replace + # it with a wrapper script that performs some initialization steps when + # required and then hands over control to the real kubelet. + + # Only create the kubelet wrapper if we haven't previously done so. + if [[ ! -f "${GKE_KUBERNETES_BIN_DIR}/the-kubelet" ]]; + then + echo "Installing the kubelet wrapper..." + + # Rename the real kubelet. + mv "${GKE_KUBERNETES_BIN_DIR}/kubelet" "${GKE_KUBERNETES_BIN_DIR}/the-kubelet" + + # Initialize the kubelet wrapper which lives in the place of the real kubelet. + touch "${GKE_KUBERNETES_BIN_DIR}/kubelet" + chmod a+x "${GKE_KUBERNETES_BIN_DIR}/kubelet" + + # Populate the kubelet wrapper. It will perform the initialization steps we + # need and then become the kubelet. + cat <<'EOF' | tee "${GKE_KUBERNETES_BIN_DIR}/kubelet" + #!/bin/bash + + set -euo pipefail + + CNI_CONF_DIR="/etc/cni/net.d" + CONTAINERD_CONFIG="/etc/containerd/config.toml" + + # kubelet version string format is "Kubernetes v1.24-gke.900" + K8S_VERSION=$(/home/kubernetes/bin/the-kubelet --version) + + # Helper to check if a version string, passed as first parameter, is greater than or + # equal the one passed as second parameter. + function version_gte() { + [[ "$(printf '%s\n' "${2}" "${1}" | sort -V | head -n1)" = "${2}" ]] && return + } + + # Only stop and start containerd if the Cilium CNI configuration does not exist, + # or if the 'conf_template' property is present in the containerd config file, + # in order to avoid unnecessarily restarting containerd. + if [[ -z "$(find "${CNI_CONF_DIR}" -type f -name '*cilium*')" || \ + "$(grep -cE '^\s+conf_template' "${CONTAINERD_CONFIG}")" != "0" ]]; + then + # Stop containerd as it starts by creating a CNI configuration from a template + # causing pods to start with IPs assigned by GKE's CNI. + # 'disable --now' is used instead of stop as this script runs concurrently + # with containerd on node startup, and hence containerd might not have been + # started yet, in which case 'disable' prevents it from starting. + echo "Disabling and stopping containerd" + systemctl disable --now containerd + + # Remove any pre-existing files in the CNI configuration directory. We skip + # any possibly existing Cilium configuration file for the obvious reasons. + echo "Removing undesired CNI configuration files" + find "${CNI_CONF_DIR}" -type f -not -name '*cilium*' -exec rm {} \; + + # As mentioned above, the containerd configuration needs a little tweak in + # order not to create the default CNI configuration, so we update its config. + echo "Fixing containerd configuration" + sed -Ei 's/^(\s+conf_template)/\#\1/g' "${CONTAINERD_CONFIG}" + + if version_gte "${K8S_VERSION#"Kubernetes "}" "v1.24"; then + # Starting from GKE node version 1.24, containerd version used is 1.6. + # Since that version containerd no longer allows missing configuration for the CNI, + # not even for pods with hostNetwork set to true. Thus, we add a temporary one. + # This will be replaced with the real config by cni-install.sh script from the + # agent pod. + echo -e "{\n\t"cniVersion": "0.3.1",\n\t"name": "cilium",\n\t"type": "cilium-cni"\n}" > /etc/cni/net.d/05-cilium.conf + fi + + # Start containerd. It won't create it's CNI configuration file anymore. + echo "Enabling and starting containerd" + systemctl enable --now containerd + fi + + # Become the real kubelet and, for k8s < 1.24, pass it additional dockershim + # flags (and place these last so they have precedence). + if version_gte "${K8S_VERSION#"Kubernetes "}" "v1.24"; then + exec /home/kubernetes/bin/the-kubelet "${@}" + else + exec /home/kubernetes/bin/the-kubelet "${@}" --network-plugin=cni --cni-bin-dir=/home/kubernetes/bin + fi + EOF + else + echo "Kubelet wrapper already exists, skipping..." + fi + else + # kubelet version string format is "Kubernetes v1.24-gke.900" + K8S_VERSION=$(kubelet --version) + + # Helper to check if a version string, passed as first parameter, is greater than or + # equal the one passed as second parameter. + function version_gte() { + [[ "$(printf '%s\n' "${2}" "${1}" | sort -V | head -n1)" = "${2}" ]] && return + } + + # Dockershim flags have been removed since k8s 1.24. + if ! version_gte "${K8S_VERSION#"Kubernetes "}" "v1.24"; then + # (Generic) Alter the kubelet configuration to run in CNI mode + echo "Changing kubelet configuration to --network-plugin=cni --cni-bin-dir=/home/kubernetes/bin" + mkdir -p /home/kubernetes/bin + sed -i "s:--network-plugin=kubenet:--network-plugin=cni\ --cni-bin-dir=/home/kubernetes/bin:g" "${KUBELET_DEFAULTS_FILE}" + fi + fi + echo "Restarting the kubelet..." + systemctl restart kubelet + # If Cilium is configured to manage masquerading of traffic leaving the node, + # we need to disable the IP-MASQ chain because even if ip-masq-agent + # is not installed, the node init script installs some default rules into + # the IP-MASQ chain. + # If we remove the jump to that ip-masq chain, then we ensure the ip masquerade + # configuration is solely managed by Cilium. + # Also, if Cilium is installed, it may be expected that it would be solely responsible + # for the networking configuration on that node. So provide the same functionality + # as the --disable-snat-flag for existing GKE clusters. + iptables -w -t nat -D POSTROUTING -m comment --comment "ip-masq: ensure nat POSTROUTING directs all non-LOCAL destination traffic to our custom IP-MASQ chain" -m addrtype ! --dst-type LOCAL -j IP-MASQ || true + mkdir -p "/tmp/cilium-bootstrap.d" + date > "/tmp/cilium-bootstrap.d/cilium-bootstrap-time" + echo "Node initialization complete" + + resources: + requests: + cpu: 100m + memory: 100Mi + securityContext: + capabilities: + add: + - SYS_MODULE + - NET_ADMIN + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + privileged: false + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePolicy: FallbackToLogsOnError + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + hostPID: true + hostNetwork: true + priorityClassName: system-node-critical +--- +# Source: cilium/templates/cilium-operator/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cilium-operator + namespace: kube-system + labels: + io.cilium/app: operator + name: cilium-operator +spec: + # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go + # for more details. + replicas: 2 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9963" + prometheus.io/scrape: "true" + labels: + io.cilium/app: operator + name: cilium-operator + spec: + containers: + - name: cilium-operator + image: "quay.io/cilium/operator-generic:v1.12.8@sha256:7431f0c2001fb875b1a8901e103825394c38cd6c63a1435a3273ed20ae0e7578" + imagePullPolicy: IfNotPresent + command: + - cilium-operator-generic + args: + - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + ports: + - name: prometheus + containerPort: 9963 + hostPort: 9963 + protocol: TCP + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + volumeMounts: + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + terminationMessagePolicy: FallbackToLogsOnError + hostNetwork: true + restartPolicy: Always + priorityClassName: system-cluster-critical + serviceAccount: "cilium-operator" + serviceAccountName: "cilium-operator" + automountServiceAccountToken: true + # In HA mode, cilium-operator pods must not be scheduled on the same + # node as they will clash with each other. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + io.cilium/app: operator + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config +--- +# Source: cilium/templates/hubble-relay/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hubble-relay + namespace: kube-system + labels: + k8s-app: hubble-relay +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: hubble-relay + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + labels: + k8s-app: hubble-relay + spec: + containers: + - name: hubble-relay + image: "quay.io/cilium/hubble-relay:v1.12.8@sha256:508cf85bb1a11c13abd995e3c5fd18ed3c2f1d26cbf463a97297e8b8c9149f13" + imagePullPolicy: IfNotPresent + command: + - hubble-relay + args: + - serve + ports: + - name: grpc + containerPort: 4245 + - name: prometheus + containerPort: 9966 + protocol: TCP + readinessProbe: + tcpSocket: + port: grpc + livenessProbe: + tcpSocket: + port: grpc + volumeMounts: + - name: config + mountPath: /etc/hubble-relay + readOnly: true + - name: tls + mountPath: /var/lib/hubble-relay/tls + readOnly: true + terminationMessagePolicy: FallbackToLogsOnError + restartPolicy: Always + priorityClassName: + serviceAccount: "hubble-relay" + serviceAccountName: "hubble-relay" + automountServiceAccountToken: false + terminationGracePeriodSeconds: 1 + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + volumes: + - name: config + configMap: + name: hubble-relay-config + items: + - key: config.yaml + path: config.yaml + - name: tls + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-relay-client-certs + items: + - key: ca.crt + path: hubble-server-ca.crt + - key: tls.crt + path: client.crt + - key: tls.key + path: client.key +--- +# Source: cilium/templates/hubble-ui/deployment.yaml +kind: Deployment +apiVersion: apps/v1 +metadata: + name: hubble-ui + namespace: kube-system + labels: + k8s-app: hubble-ui +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: hubble-ui + template: + metadata: + annotations: + labels: + k8s-app: hubble-ui + spec: + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsUser: 1001 + priorityClassName: + serviceAccount: "hubble-ui" + serviceAccountName: "hubble-ui" + automountServiceAccountToken: true + containers: + - name: frontend + image: "quay.io/cilium/hubble-ui:v0.9.2@sha256:d3596efc94a41c6b772b9afe6fe47c17417658956e04c3e2a28d293f2670663e" + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8081 + volumeMounts: + - name: hubble-ui-nginx-conf + mountPath: /etc/nginx/conf.d/default.conf + subPath: nginx.conf + - name: tmp-dir + mountPath: /tmp + terminationMessagePolicy: FallbackToLogsOnError + - name: backend + image: "quay.io/cilium/hubble-ui-backend:v0.9.2@sha256:a3ac4d5b87889c9f7cc6323e86d3126b0d382933bd64f44382a92778b0cde5d7" + imagePullPolicy: IfNotPresent + env: + - name: EVENTS_SERVER_PORT + value: "8090" + - name: FLOWS_API_ADDR + value: "hubble-relay:80" + ports: + - name: grpc + containerPort: 8090 + volumeMounts: + terminationMessagePolicy: FallbackToLogsOnError + nodeSelector: + kubernetes.io/os: linux + volumes: + - configMap: + defaultMode: 420 + name: hubble-ui-nginx + name: hubble-ui-nginx-conf + - emptyDir: {} + name: tmp-dir +--- +# Source: cilium/templates/hubble/tls-cronjob/job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: hubble-generate-certs-c26869f410 + namespace: kube-system + labels: + k8s-app: hubble-generate-certs +spec: + template: + metadata: + labels: + k8s-app: hubble-generate-certs + spec: + containers: + - name: certgen + image: "quay.io/cilium/certgen:v0.1.8@sha256:4a456552a5f192992a6edcec2febb1c54870d665173a33dc7d876129b199ddbd" + imagePullPolicy: IfNotPresent + command: + - "/usr/bin/cilium-certgen" + # Because this is executed as a job, we pass the values as command + # line args instead of via config map. This allows users to inspect + # the values used in past runs by inspecting the completed pod. + args: + - "--cilium-namespace=kube-system" + - "--ca-generate" + - "--ca-reuse-secret" + - "--hubble-server-cert-generate" + - "--hubble-server-cert-common-name=*.default.hubble-grpc.cilium.io" + - "--hubble-server-cert-validity-duration=94608000s" + - "--hubble-relay-client-cert-generate" + - "--hubble-relay-client-cert-validity-duration=94608000s" + hostNetwork: true + serviceAccount: "hubble-generate-certs" + serviceAccountName: "hubble-generate-certs" + automountServiceAccountToken: true + restartPolicy: OnFailure + ttlSecondsAfterFinished: 1800 +--- +# Source: cilium/templates/hubble/tls-cronjob/cronjob.yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: hubble-generate-certs + namespace: kube-system + labels: + k8s-app: hubble-generate-certs +spec: + schedule: "0 0 1 */4 *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + metadata: + labels: + k8s-app: hubble-generate-certs + spec: + containers: + - name: certgen + image: "quay.io/cilium/certgen:v0.1.8@sha256:4a456552a5f192992a6edcec2febb1c54870d665173a33dc7d876129b199ddbd" + imagePullPolicy: IfNotPresent + command: + - "/usr/bin/cilium-certgen" + # Because this is executed as a job, we pass the values as command + # line args instead of via config map. This allows users to inspect + # the values used in past runs by inspecting the completed pod. + args: + - "--cilium-namespace=kube-system" + - "--ca-generate" + - "--ca-reuse-secret" + - "--hubble-server-cert-generate" + - "--hubble-server-cert-common-name=*.default.hubble-grpc.cilium.io" + - "--hubble-server-cert-validity-duration=94608000s" + - "--hubble-relay-client-cert-generate" + - "--hubble-relay-client-cert-validity-duration=94608000s" + hostNetwork: true + serviceAccount: "hubble-generate-certs" + serviceAccountName: "hubble-generate-certs" + automountServiceAccountToken: true + restartPolicy: OnFailure + ttlSecondsAfterFinished: 1800 diff --git a/hack/cni/gke-values.yaml b/hack/cni/gke-values.yaml new file mode 100755 index 00000000..b14e6313 --- /dev/null +++ b/hack/cni/gke-values.yaml @@ -0,0 +1,11 @@ +gke: + enabled: true +cni: + binPath: /home/kubernetes/bin +ipv4NativeRoutingCIDR: 10.0.0.0/16 +nodeinit: + enabled: true + reconfigureKubelet: true + removeCbrBridge: true +ipam: + mode: kubernetes diff --git a/hack/cni/kind-1.12.6.yaml b/hack/cni/kind-1.12.6.yaml new file mode 100644 index 00000000..809c4df4 --- /dev/null +++ b/hack/cni/kind-1.12.6.yaml @@ -0,0 +1,1592 @@ +--- +# Source: cilium/templates/cilium-agent/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/hubble-relay/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "hubble-relay" + namespace: kube-system +--- +# Source: cilium/templates/hubble-ui/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "hubble-ui" + namespace: kube-system +--- +# Source: cilium/templates/hubble/tls-cronjob/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "hubble-generate-certs" + namespace: kube-system +--- +# Source: cilium/templates/cilium-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-config + namespace: kube-system +data: + + # Identity allocation mode selects how identities are shared between cilium + # nodes by setting how they are stored. The options are "crd" or "kvstore". + # - "crd" stores identities in kubernetes as CRDs (custom resource definition). + # These can be queried with: + # kubectl get ciliumid + # - "kvstore" stores identities in an etcd kvstore, that is + # configured below. Cilium versions before 1.6 supported only the kvstore + # backend. Upgrades from these older cilium versions should continue using + # the kvstore by commenting out the identity-allocation-mode below, or + # setting it to "kvstore". + identity-allocation-mode: crd + cilium-endpoint-gc-interval: "5m0s" + nodes-gc-interval: "5m0s" + skip-cnp-status-startup-clean: "false" + # Disable the usage of CiliumEndpoint CRD + disable-endpoint-crd: "false" + + # If you want to run cilium in debug mode change this value to true + debug: "false" + # The agent can be put into the following three policy enforcement modes + # default, always and never. + # https://docs.cilium.io/en/latest/policy/intro/#policy-enforcement-modes + enable-policy: "default" + # If you want metrics enabled in all of your Cilium agents, set the port for + # which the Cilium agents will have their metrics exposed. + # This option deprecates the "prometheus-serve-addr" in the + # "cilium-metrics-config" ConfigMap + # NOTE that this will open the port on ALL nodes where Cilium pods are + # scheduled. + prometheus-serve-addr: ":9962" + # Port to expose Envoy metrics (e.g. "9964"). Envoy metrics listener will be disabled if this + # field is not set. + proxy-prometheus-port: "9964" + # If you want metrics enabled in cilium-operator, set the port for + # which the Cilium Operator will have their metrics exposed. + # NOTE that this will open the port on the nodes where Cilium operator pod + # is scheduled. + operator-prometheus-serve-addr: ":9963" + enable-metrics: "true" + + # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 + # address. + enable-ipv4: "true" + + # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 + # address. + enable-ipv6: "false" + # Users who wish to specify their own custom CNI configuration file must set + # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. + custom-cni-conf: "false" + enable-bpf-clock-probe: "true" + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation: medium + + # The monitor aggregation interval governs the typical time between monitor + # notification events for each allowed connection. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-interval: 5s + + # The monitor aggregation flags determine which TCP flags which, upon the + # first observation, cause monitor notifications to be generated. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-flags: all + # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic + # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. + bpf-map-dynamic-size-ratio: "0.0025" + # bpf-policy-map-max specifies the maximum number of entries in endpoint + # policy map (per endpoint) + bpf-policy-map-max: "16384" + # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, + # backend and affinity maps. + bpf-lb-map-max: "65536" + # bpf-lb-bypass-fib-lookup instructs Cilium to enable the FIB lookup bypass + # optimization for nodeport reverse NAT handling. + bpf-lb-external-clusterip: "false" + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # As a result, reply packets may be dropped and the load-balancing decisions + # for established connections may change. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps: "false" + + # Regular expression matching compatible Istio sidecar istio-proxy + # container image names + sidecar-istio-proxy-image: "cilium/istio_proxy" + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name: default + # Unique ID of the cluster. Must be unique across all conneted clusters and + # in the range of 1 and 255. Only relevant when building a mesh of clusters. + cluster-id: "0" + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve + tunnel: "vxlan" + # Enables L7 proxy for L7 policy enforcement and visibility + enable-l7-proxy: "true" + # Enable chaining with another CNI plugin + # + # Supported modes: + # - none + # - aws-cni + # - flannel + # - generic-veth + # - portmap (Enables HostPort support for Cilium) + cni-chaining-mode: portmap + + enable-ipv4-masquerade: "true" + enable-ipv6-masquerade: "true" + + enable-xt-socket-fallback: "true" + install-iptables-rules: "true" + install-no-conntrack-iptables-rules: "false" + + auto-direct-node-routes: "false" + enable-local-redirect-policy: "false" + + kube-proxy-replacement: "disabled" + bpf-lb-sock: "false" + enable-health-check-nodeport: "true" + node-port-bind-protection: "true" + enable-auto-protect-node-port-range: "true" + enable-session-affinity: "true" + enable-svc-source-range-check: "true" + enable-l2-neigh-discovery: "true" + arping-refresh-period: "30s" + enable-endpoint-health-checking: "true" + enable-health-checking: "true" + enable-well-known-identities: "false" + enable-remote-node-identity: "true" + synchronize-k8s-nodes: "true" + operator-api-serve-addr: "127.0.0.1:9234" + # Enable Hubble gRPC service. + enable-hubble: "true" + # UNIX domain socket for Hubble server to listen to. + hubble-socket-path: "/var/run/cilium/hubble.sock" + # Address to expose Hubble metrics (e.g. ":7070"). Metrics server will be disabled if this + # field is not set. + hubble-metrics-server: ":9965" + # A space separated list of metrics to enable. See [0] for available metrics. + # + # https://github.com/cilium/hubble/blob/master/Documentation/metrics.md + hubble-metrics: + dns:query;ignoreAAAA + drop + tcp + flow + icmp + http + # An additional address for Hubble server to listen to (e.g. ":4244"). + hubble-listen-address: ":4244" + hubble-disable-tls: "false" + hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt + hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key + hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt + ipam: "cluster-pool" + cluster-pool-ipv4-cidr: "10.0.0.0/8" + cluster-pool-ipv4-mask-size: "24" + disable-cnp-status-updates: "true" + enable-vtep: "false" + vtep-endpoint: "" + vtep-cidr: "" + vtep-mask: "" + vtep-mac: "" + enable-bgp-control-plane: "false" + procfs: "/host/proc" + bpf-root: "/sys/fs/bpf" + cgroup-root: "/run/cilium/cgroupv2" + enable-k8s-terminating-endpoint: "true" + remove-cilium-node-taints: "true" + set-cilium-is-up-condition: "true" + unmanaged-pod-watcher-interval: "15" + tofqdns-dns-reject-response-code: "refused" + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "50" + tofqdns-idle-connection-grace-period: "0s" + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-min-ttl: "3600" + tofqdns-proxy-response-max-delay: "100ms" + agent-not-ready-taint-key: "ignore-taint.cluster-autoscaler.kubernetes.io/cilium-agent-not-ready" +--- +# Source: cilium/templates/hubble-relay/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hubble-relay-config + namespace: kube-system +data: + config.yaml: | + cluster-name: default + peer-service: "hubble-peer.kube-system.svc.cluster.local:443" + listen-address: :4245 + metrics-listen-address: ":9966" + dial-timeout: + retry-timeout: + sort-buffer-len-max: + sort-buffer-drain-timeout: + tls-client-cert-file: /var/lib/hubble-relay/tls/client.crt + tls-client-key-file: /var/lib/hubble-relay/tls/client.key + tls-hubble-server-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt + disable-server-tls: true +--- +# Source: cilium/templates/hubble-ui/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hubble-ui-nginx + namespace: kube-system +data: + nginx.conf: "server {\n listen 8081;\n listen [::]:8081;\n server_name localhost;\n root /app;\n index index.html;\n client_max_body_size 1G;\n\n location / {\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n\n # CORS\n add_header Access-Control-Allow-Methods \"GET, POST, PUT, HEAD, DELETE, OPTIONS\";\n add_header Access-Control-Allow-Origin *;\n add_header Access-Control-Max-Age 1728000;\n add_header Access-Control-Expose-Headers content-length,grpc-status,grpc-message;\n add_header Access-Control-Allow-Headers range,keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout;\n if ($request_method = OPTIONS) {\n return 204;\n }\n # /CORS\n\n location /api {\n proxy_http_version 1.1;\n proxy_pass_request_headers on;\n proxy_hide_header Access-Control-Allow-Origin;\n proxy_pass http://127.0.0.1:8090;\n }\n\n location / {\n try_files $uri $uri/ /index.html;\n }\n }\n}" +--- +# Source: cilium/templates/cilium-agent/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - pods + - endpoints + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get +- apiGroups: + - cilium.io + resources: + - ciliumbgploadbalancerippools + - ciliumbgppeeringpolicies + - ciliumclusterwideenvoyconfigs + - ciliumclusterwidenetworkpolicies + - ciliumegressgatewaypolicies + - ciliumegressnatpolicies + - ciliumendpoints + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumidentities + - ciliumlocalredirectpolicies + - ciliumnetworkpolicies + - ciliumnodes + verbs: + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + - ciliumendpoints + - ciliumnodes + verbs: + - create +- apiGroups: + - cilium.io + # To synchronize garbage collection of such resources + resources: + - ciliumidentities + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + verbs: + - delete + - get +- apiGroups: + - cilium.io + resources: + - ciliumnodes + - ciliumnodes/status + verbs: + - get + - update +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + - ciliumendpoints/status + - ciliumendpoints + verbs: + - patch +--- +# Source: cilium/templates/cilium-operator/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-operator +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + # To remove node taints + - nodes + # To set NetworkUnavailable false on startup + - nodes/status + verbs: + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform LB IP allocation for BGP + - services/status + verbs: + - update +- apiGroups: + - "" + resources: + # to check apiserver connectivity + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumclusterwidenetworkpolicies + verbs: + # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' + - create + - update + - deletecollection + # To update the status of the CNPs and CCNPs + - patch + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + verbs: + # Update the auto-generated CNPs and CCNPs status. + - patch + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + - ciliumidentities + verbs: + # To perform garbage collection of such resources + - delete + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + verbs: + # To synchronize garbage collection of such resources + - update +- apiGroups: + - cilium.io + resources: + - ciliumnodes + verbs: + - create + - update + - get + - list + - watch + # To perform CiliumNode garbage collector + - delete +- apiGroups: + - cilium.io + resources: + - ciliumnodes/status + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpointslices + - ciliumenvoyconfigs + verbs: + - create + - update + - get + - list + - watch + - delete +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - update + resourceNames: + - ciliumbgploadbalancerippools.cilium.io + - ciliumbgppeeringpolicies.cilium.io + - ciliumclusterwideenvoyconfigs.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumegressgatewaypolicies.cilium.io + - ciliumegressnatpolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumenvoyconfigs.cilium.io + - ciliumexternalworkloads.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodes.cilium.io +# For cilium-operator running in HA mode. +# +# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election +# between multiple running instances. +# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less +# common and fewer objects in the cluster watch "all Leases". +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +--- +# Source: cilium/templates/hubble-ui/clusterrole.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-ui +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - componentstatuses + - endpoints + - namespaces + - nodes + - pods + - services + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - "*" + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/hubble/tls-cronjob/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hubble-generate-certs +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - hubble-server-certs + - hubble-relay-client-certs + - hubble-relay-server-certs + verbs: + - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - cilium-ca + - hubble-ca-secret + verbs: + - get + - update +--- +# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/hubble-ui/clusterrolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-ui +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hubble-ui +subjects: +- kind: ServiceAccount + name: "hubble-ui" + namespace: kube-system +--- +# Source: cilium/templates/hubble/tls-cronjob/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hubble-generate-certs +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hubble-generate-certs +subjects: +- kind: ServiceAccount + name: "hubble-generate-certs" + namespace: kube-system +--- +# Source: cilium/templates/cilium-agent/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: cilium-agent + namespace: kube-system + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9964" + labels: + k8s-app: cilium +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: cilium + ports: + - name: envoy-metrics + port: 9964 + protocol: TCP + targetPort: envoy-metrics +--- +# Source: cilium/templates/hubble-relay/metrics-service.yaml +# We use a separate service from hubble-relay which can be exposed externally +kind: Service +apiVersion: v1 +metadata: + name: hubble-relay-metrics + namespace: kube-system + labels: + k8s-app: hubble-relay +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: hubble-relay + ports: + - name: metrics + port: 9966 + protocol: TCP + targetPort: prometheus +--- +# Source: cilium/templates/hubble-relay/service.yaml +kind: Service +apiVersion: v1 +metadata: + name: hubble-relay + namespace: kube-system + labels: + k8s-app: hubble-relay +spec: + type: "ClusterIP" + selector: + k8s-app: hubble-relay + ports: + - protocol: TCP + port: 80 + targetPort: 4245 +--- +# Source: cilium/templates/hubble-ui/service.yaml +kind: Service +apiVersion: v1 +metadata: + name: hubble-ui + namespace: kube-system + labels: + k8s-app: hubble-ui +spec: + type: "ClusterIP" + selector: + k8s-app: hubble-ui + ports: + - name: http + port: 80 + targetPort: 8081 +--- +# Source: cilium/templates/hubble/metrics-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hubble-metrics + namespace: kube-system + labels: + k8s-app: hubble + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9965" +spec: + clusterIP: None + type: ClusterIP + ports: + - name: hubble-metrics + port: 9965 + protocol: TCP + targetPort: hubble-metrics + selector: + k8s-app: cilium +--- +# Source: cilium/templates/hubble/peer-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hubble-peer + namespace: kube-system + labels: + k8s-app: cilium +spec: + selector: + k8s-app: cilium + ports: + - name: peer-service + port: 443 + protocol: TCP + targetPort: 4244 + internalTrafficPolicy: Local +--- +# Source: cilium/templates/cilium-agent/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium + namespace: kube-system + labels: + k8s-app: cilium +spec: + selector: + matchLabels: + k8s-app: cilium + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9962" + prometheus.io/scrape: "true" + # Set app AppArmor's profile to "unconfined". The value of this annotation + # can be modified as long users know which profiles they have available + # in AppArmor. + container.apparmor.security.beta.kubernetes.io/cilium-agent: "unconfined" + container.apparmor.security.beta.kubernetes.io/clean-cilium-state: "unconfined" + container.apparmor.security.beta.kubernetes.io/mount-cgroup: "unconfined" + container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: "unconfined" + labels: + k8s-app: cilium + spec: + containers: + - name: cilium-agent + image: "quay.io/cilium/cilium:v1.12.6@sha256:454134506b0448c756398d3e8df68d474acde2a622ab58d0c7e8b272b5867d0d" + imagePullPolicy: IfNotPresent + command: + - cilium-agent + args: + - --config-dir=/tmp/cilium/config-map + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + failureThreshold: 105 + periodSeconds: 2 + successThreshold: 1 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: CILIUM_CNI_CHAINING_MODE + valueFrom: + configMapKeyRef: + name: cilium-config + key: cni-chaining-mode + optional: true + - name: CILIUM_CUSTOM_CNI_CONF + valueFrom: + configMapKeyRef: + name: cilium-config + key: custom-cni-conf + optional: true + lifecycle: + postStart: + exec: + command: + - "/cni-install.sh" + - "--enable-debug=false" + - "--cni-exclusive=true" + - "--log-file=/var/run/cilium/cilium-cni.log" + preStop: + exec: + command: + - /cni-uninstall.sh + ports: + - name: peer-service + containerPort: 4244 + hostPort: 4244 + protocol: TCP + - name: prometheus + containerPort: 9962 + hostPort: 9962 + protocol: TCP + - name: envoy-metrics + containerPort: 9964 + hostPort: 9964 + protocol: TCP + - name: hubble-metrics + containerPort: 9965 + hostPort: 9965 + protocol: TCP + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + # Use to set socket permission + - CHOWN + # Used to terminate envoy child process + - KILL + # Used since cilium modifies routing tables, etc... + - NET_ADMIN + # Used since cilium creates raw sockets, etc... + - NET_RAW + # Used since cilium monitor uses mmap + - IPC_LOCK + # Used in iptables. Consider removing once we are iptables-free + - SYS_MODULE + # We need it for now but might not need it for >= 5.11 specially + # for the 'SYS_RESOURCE'. + # In >= 5.8 there's already BPF and PERMON capabilities + - SYS_ADMIN + # Could be an alternative for the SYS_ADMIN for the RLIMIT_NPROC + - SYS_RESOURCE + # Both PERFMON and BPF requires kernel 5.8, container runtime + # cri-o >= v1.22.0 or containerd >= v1.5.0. + # If available, SYS_ADMIN can be removed. + #- PERFMON + #- BPF + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # Unprivileged containers need to mount /proc/sys/net from the host + # to have write access + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + # Unprivileged containers need to mount /proc/sys/kernel from the host + # to have write access + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - name: bpf-maps + mountPath: /sys/fs/bpf + # Unprivileged containers can't set mount propagation to bidirectional + # in this case we will mount the bpf fs from an init container that + # is privileged and set the mount propagation from host to container + # in Cilium. + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium + - name: cni-path + mountPath: /host/opt/cni/bin + - name: etc-cni-netd + mountPath: /host/etc/cni/net.d + - name: clustermesh-secrets + mountPath: /var/lib/cilium/clustermesh + readOnly: true + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + # Needed to be able to load kernel modules + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + - name: hubble-tls + mountPath: /var/lib/cilium/tls/hubble + readOnly: true + initContainers: + # Required to mount cgroup2 filesystem on the underlying Kubernetes node. + # We use nsenter command with host's cgroup and mount namespaces enabled. + - name: mount-cgroup + image: "quay.io/cilium/cilium:v1.12.6@sha256:454134506b0448c756398d3e8df68d474acde2a622ab58d0c7e8b272b5867d0d" + imagePullPolicy: IfNotPresent + env: + - name: CGROUP_ROOT + value: /run/cilium/cgroupv2 + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh and mount that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + add: + # Only used for 'mount' cgroup + - SYS_ADMIN + # Used for nsenter + - SYS_CHROOT + - SYS_PTRACE + - name: apply-sysctl-overwrites + image: "quay.io/cilium/cilium:v1.12.6@sha256:454134506b0448c756398d3e8df68d474acde2a622ab58d0c7e8b272b5867d0d" + imagePullPolicy: IfNotPresent + env: + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + add: + # Required in order to access host's /etc/sysctl.d dir + - SYS_ADMIN + # Used for nsenter + - SYS_CHROOT + - SYS_PTRACE + # Mount the bpf fs if it is not mounted. We will perform this task + # from a privileged container because the mount propagation bidirectional + # only works from privileged containers. + - name: mount-bpf-fs + image: "quay.io/cilium/cilium:v1.12.6@sha256:454134506b0448c756398d3e8df68d474acde2a622ab58d0c7e8b272b5867d0d" + imagePullPolicy: IfNotPresent + args: + - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' + command: + - /bin/bash + - -c + - -- + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + privileged: true + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + - name: clean-cilium-state + image: "quay.io/cilium/cilium:v1.12.6@sha256:454134506b0448c756398d3e8df68d474acde2a622ab58d0c7e8b272b5867d0d" + imagePullPolicy: IfNotPresent + command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-state + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-bpf-state + optional: true + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + # Most of the capabilities here are the same ones used in the + # cilium-agent's container because this container can be used to + # uninstall all Cilium resources, and therefore it is likely that + # will need the same capabilities. + add: + # Used since cilium modifies routing tables, etc... + - NET_ADMIN + # Used in iptables. Consider removing once we are iptables-free + - SYS_MODULE + # We need it for now but might not need it for >= 5.11 specially + # for the 'SYS_RESOURCE'. + # In >= 5.8 there's already BPF and PERMON capabilities + - SYS_ADMIN + # Could be an alternative for the SYS_ADMIN for the RLIMIT_NPROC + - SYS_RESOURCE + # Both PERFMON and BPF requires kernel 5.8, container runtime + # cri-o >= v1.22.0 or containerd >= v1.5.0. + # If available, SYS_ADMIN can be removed. + #- PERFMON + #- BPF + drop: + - ALL + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + # Required to mount cgroup filesystem from the host to cilium agent pod + - name: cilium-cgroup + mountPath: /run/cilium/cgroupv2 + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium + resources: + requests: + cpu: 100m + memory: 100Mi # wait-for-kube-proxy + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccount: "cilium" + serviceAccountName: "cilium" + terminationGracePeriodSeconds: 1 + hostNetwork: true + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + # To mount cgroup2 filesystem on the host + - name: hostproc + hostPath: + path: /proc + type: Directory + # To keep state between restarts / upgrades for cgroup2 filesystem + - name: cilium-cgroup + hostPath: + path: /run/cilium/cgroupv2 + type: DirectoryOrCreate + # To install cilium cni plugin in the host + - name: cni-path + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + # To install cilium cni configuration in the host + - name: etc-cni-netd + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + # To be able to load kernel modules + - name: lib-modules + hostPath: + path: /lib/modules + # To access iptables concurrently with other processes (e.g. kube-proxy) + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # To read the clustermesh configuration + - name: clustermesh-secrets + secret: + secretName: cilium-clustermesh + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + optional: true + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config + - name: host-proc-sys-net + hostPath: + path: /proc/sys/net + type: Directory + - name: host-proc-sys-kernel + hostPath: + path: /proc/sys/kernel + type: Directory + - name: hubble-tls + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-server-certs + optional: true + items: + - key: ca.crt + path: client-ca.crt + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key +--- +# Source: cilium/templates/cilium-operator/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cilium-operator + namespace: kube-system + labels: + io.cilium/app: operator + name: cilium-operator +spec: + # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go + # for more details. + replicas: 2 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9963" + prometheus.io/scrape: "true" + labels: + io.cilium/app: operator + name: cilium-operator + spec: + containers: + - name: cilium-operator + image: "quay.io/cilium/operator-generic:v1.12.6@sha256:eec4430d222cb2967d42d3b404d2606e66468de47ae85e0a3ca3f58f00a5e017" + imagePullPolicy: IfNotPresent + command: + - cilium-operator-generic + args: + - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + ports: + - name: prometheus + containerPort: 9963 + hostPort: 9963 + protocol: TCP + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + volumeMounts: + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + terminationMessagePolicy: FallbackToLogsOnError + hostNetwork: true + restartPolicy: Always + priorityClassName: system-cluster-critical + serviceAccount: "cilium-operator" + serviceAccountName: "cilium-operator" + # In HA mode, cilium-operator pods must not be scheduled on the same + # node as they will clash with each other. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + io.cilium/app: operator + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config +--- +# Source: cilium/templates/hubble-relay/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hubble-relay + namespace: kube-system + labels: + k8s-app: hubble-relay +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: hubble-relay + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + labels: + k8s-app: hubble-relay + spec: + containers: + - name: hubble-relay + image: "quay.io/cilium/hubble-relay:v1.12.6@sha256:27a68a16f0ee7ed6ba690e91847de6931a5511f85a7f939320df216486764cb9" + imagePullPolicy: IfNotPresent + command: + - hubble-relay + args: + - serve + ports: + - name: grpc + containerPort: 4245 + - name: prometheus + containerPort: 9966 + protocol: TCP + readinessProbe: + tcpSocket: + port: grpc + livenessProbe: + tcpSocket: + port: grpc + volumeMounts: + - name: config + mountPath: /etc/hubble-relay + readOnly: true + - name: tls + mountPath: /var/lib/hubble-relay/tls + readOnly: true + terminationMessagePolicy: FallbackToLogsOnError + restartPolicy: Always + priorityClassName: + serviceAccount: "hubble-relay" + serviceAccountName: "hubble-relay" + automountServiceAccountToken: false + terminationGracePeriodSeconds: 1 + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + volumes: + - name: config + configMap: + name: hubble-relay-config + items: + - key: config.yaml + path: config.yaml + - name: tls + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-relay-client-certs + items: + - key: ca.crt + path: hubble-server-ca.crt + - key: tls.crt + path: client.crt + - key: tls.key + path: client.key +--- +# Source: cilium/templates/hubble-ui/deployment.yaml +kind: Deployment +apiVersion: apps/v1 +metadata: + name: hubble-ui + namespace: kube-system + labels: + k8s-app: hubble-ui +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: hubble-ui + template: + metadata: + annotations: + labels: + k8s-app: hubble-ui + spec: + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsUser: 1001 + priorityClassName: + serviceAccount: "hubble-ui" + serviceAccountName: "hubble-ui" + containers: + - name: frontend + image: "quay.io/cilium/hubble-ui:v0.9.2@sha256:d3596efc94a41c6b772b9afe6fe47c17417658956e04c3e2a28d293f2670663e" + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8081 + volumeMounts: + - name: hubble-ui-nginx-conf + mountPath: /etc/nginx/conf.d/default.conf + subPath: nginx.conf + - name: tmp-dir + mountPath: /tmp + terminationMessagePolicy: FallbackToLogsOnError + - name: backend + image: "quay.io/cilium/hubble-ui-backend:v0.9.2@sha256:a3ac4d5b87889c9f7cc6323e86d3126b0d382933bd64f44382a92778b0cde5d7" + imagePullPolicy: IfNotPresent + env: + - name: EVENTS_SERVER_PORT + value: "8090" + - name: FLOWS_API_ADDR + value: "hubble-relay:80" + ports: + - name: grpc + containerPort: 8090 + volumeMounts: + terminationMessagePolicy: FallbackToLogsOnError + nodeSelector: + kubernetes.io/os: linux + volumes: + - configMap: + defaultMode: 420 + name: hubble-ui-nginx + name: hubble-ui-nginx-conf + - emptyDir: {} + name: tmp-dir +--- +# Source: cilium/templates/hubble/tls-cronjob/job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: hubble-generate-certs-394f790584 + namespace: kube-system + labels: + k8s-app: hubble-generate-certs +spec: + template: + metadata: + labels: + k8s-app: hubble-generate-certs + spec: + containers: + - name: certgen + image: "quay.io/cilium/certgen:v0.1.8@sha256:4a456552a5f192992a6edcec2febb1c54870d665173a33dc7d876129b199ddbd" + imagePullPolicy: IfNotPresent + command: + - "/usr/bin/cilium-certgen" + # Because this is executed as a job, we pass the values as command + # line args instead of via config map. This allows users to inspect + # the values used in past runs by inspecting the completed pod. + args: + - "--cilium-namespace=kube-system" + - "--ca-generate" + - "--ca-reuse-secret" + - "--hubble-server-cert-generate" + - "--hubble-server-cert-common-name=*.default.hubble-grpc.cilium.io" + - "--hubble-server-cert-validity-duration=94608000s" + - "--hubble-relay-client-cert-generate" + - "--hubble-relay-client-cert-validity-duration=94608000s" + hostNetwork: true + serviceAccount: "hubble-generate-certs" + serviceAccountName: "hubble-generate-certs" + restartPolicy: OnFailure + ttlSecondsAfterFinished: 1800 +--- +# Source: cilium/templates/hubble/tls-cronjob/cronjob.yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: hubble-generate-certs + namespace: kube-system + labels: + k8s-app: hubble-generate-certs +spec: + schedule: "0 0 1 */4 *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + metadata: + labels: + k8s-app: hubble-generate-certs + spec: + containers: + - name: certgen + image: "quay.io/cilium/certgen:v0.1.8@sha256:4a456552a5f192992a6edcec2febb1c54870d665173a33dc7d876129b199ddbd" + imagePullPolicy: IfNotPresent + command: + - "/usr/bin/cilium-certgen" + # Because this is executed as a job, we pass the values as command + # line args instead of via config map. This allows users to inspect + # the values used in past runs by inspecting the completed pod. + args: + - "--cilium-namespace=kube-system" + - "--ca-generate" + - "--ca-reuse-secret" + - "--hubble-server-cert-generate" + - "--hubble-server-cert-common-name=*.default.hubble-grpc.cilium.io" + - "--hubble-server-cert-validity-duration=94608000s" + - "--hubble-relay-client-cert-generate" + - "--hubble-relay-client-cert-validity-duration=94608000s" + hostNetwork: true + serviceAccount: "hubble-generate-certs" + serviceAccountName: "hubble-generate-certs" + restartPolicy: OnFailure + ttlSecondsAfterFinished: 1800 diff --git a/hack/cni/kind-1.12.8.yaml b/hack/cni/kind-1.12.8.yaml new file mode 100644 index 00000000..7ae212c5 --- /dev/null +++ b/hack/cni/kind-1.12.8.yaml @@ -0,0 +1,1617 @@ +--- +# Source: cilium/templates/cilium-agent/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/hubble-relay/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "hubble-relay" + namespace: kube-system +--- +# Source: cilium/templates/hubble-ui/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "hubble-ui" + namespace: kube-system +--- +# Source: cilium/templates/hubble/tls-cronjob/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "hubble-generate-certs" + namespace: kube-system +--- +# Source: cilium/templates/cilium-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-config + namespace: kube-system +data: + + # Identity allocation mode selects how identities are shared between cilium + # nodes by setting how they are stored. The options are "crd" or "kvstore". + # - "crd" stores identities in kubernetes as CRDs (custom resource definition). + # These can be queried with: + # kubectl get ciliumid + # - "kvstore" stores identities in an etcd kvstore, that is + # configured below. Cilium versions before 1.6 supported only the kvstore + # backend. Upgrades from these older cilium versions should continue using + # the kvstore by commenting out the identity-allocation-mode below, or + # setting it to "kvstore". + identity-allocation-mode: crd + cilium-endpoint-gc-interval: "5m0s" + nodes-gc-interval: "5m0s" + skip-cnp-status-startup-clean: "false" + # Disable the usage of CiliumEndpoint CRD + disable-endpoint-crd: "false" + + # If you want to run cilium in debug mode change this value to true + debug: "false" + # The agent can be put into the following three policy enforcement modes + # default, always and never. + # https://docs.cilium.io/en/latest/policy/intro/#policy-enforcement-modes + enable-policy: "default" + # If you want metrics enabled in all of your Cilium agents, set the port for + # which the Cilium agents will have their metrics exposed. + # This option deprecates the "prometheus-serve-addr" in the + # "cilium-metrics-config" ConfigMap + # NOTE that this will open the port on ALL nodes where Cilium pods are + # scheduled. + prometheus-serve-addr: ":9962" + # Port to expose Envoy metrics (e.g. "9964"). Envoy metrics listener will be disabled if this + # field is not set. + proxy-prometheus-port: "9964" + # If you want metrics enabled in cilium-operator, set the port for + # which the Cilium Operator will have their metrics exposed. + # NOTE that this will open the port on the nodes where Cilium operator pod + # is scheduled. + operator-prometheus-serve-addr: ":9963" + enable-metrics: "true" + + # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 + # address. + enable-ipv4: "true" + + # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 + # address. + enable-ipv6: "false" + # Users who wish to specify their own custom CNI configuration file must set + # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. + custom-cni-conf: "false" + enable-bpf-clock-probe: "true" + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation: medium + + # The monitor aggregation interval governs the typical time between monitor + # notification events for each allowed connection. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-interval: 5s + + # The monitor aggregation flags determine which TCP flags which, upon the + # first observation, cause monitor notifications to be generated. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-flags: all + # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic + # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. + bpf-map-dynamic-size-ratio: "0.0025" + # bpf-policy-map-max specifies the maximum number of entries in endpoint + # policy map (per endpoint) + bpf-policy-map-max: "16384" + # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, + # backend and affinity maps. + bpf-lb-map-max: "65536" + # bpf-lb-bypass-fib-lookup instructs Cilium to enable the FIB lookup bypass + # optimization for nodeport reverse NAT handling. + bpf-lb-external-clusterip: "false" + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # As a result, reply packets may be dropped and the load-balancing decisions + # for established connections may change. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps: "false" + + # Regular expression matching compatible Istio sidecar istio-proxy + # container image names + sidecar-istio-proxy-image: "cilium/istio_proxy" + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name: default + # Unique ID of the cluster. Must be unique across all conneted clusters and + # in the range of 1 and 255. Only relevant when building a mesh of clusters. + cluster-id: "0" + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve + tunnel: "vxlan" + # Enables L7 proxy for L7 policy enforcement and visibility + enable-l7-proxy: "true" + # Enable chaining with another CNI plugin + # + # Supported modes: + # - none + # - aws-cni + # - flannel + # - generic-veth + # - portmap (Enables HostPort support for Cilium) + cni-chaining-mode: portmap + + enable-ipv4-masquerade: "true" + enable-ipv6-masquerade: "true" + + enable-xt-socket-fallback: "true" + install-iptables-rules: "true" + install-no-conntrack-iptables-rules: "false" + + auto-direct-node-routes: "false" + enable-local-redirect-policy: "false" + + kube-proxy-replacement: "disabled" + bpf-lb-sock: "false" + enable-health-check-nodeport: "true" + node-port-bind-protection: "true" + enable-auto-protect-node-port-range: "true" + enable-session-affinity: "true" + enable-svc-source-range-check: "true" + enable-l2-neigh-discovery: "true" + arping-refresh-period: "30s" + cni-uninstall: "true" + enable-endpoint-health-checking: "true" + enable-health-checking: "true" + enable-well-known-identities: "false" + enable-remote-node-identity: "true" + synchronize-k8s-nodes: "true" + operator-api-serve-addr: "127.0.0.1:9234" + # Enable Hubble gRPC service. + enable-hubble: "true" + # UNIX domain socket for Hubble server to listen to. + hubble-socket-path: "/var/run/cilium/hubble.sock" + # Address to expose Hubble metrics (e.g. ":7070"). Metrics server will be disabled if this + # field is not set. + hubble-metrics-server: ":9965" + # A space separated list of metrics to enable. See [0] for available metrics. + # + # https://github.com/cilium/hubble/blob/master/Documentation/metrics.md + hubble-metrics: + dns:query;ignoreAAAA + drop + tcp + flow + icmp + http + # An additional address for Hubble server to listen to (e.g. ":4244"). + hubble-listen-address: ":4244" + hubble-disable-tls: "false" + hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt + hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key + hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt + ipam: "cluster-pool" + cluster-pool-ipv4-cidr: "10.0.0.0/8" + cluster-pool-ipv4-mask-size: "24" + disable-cnp-status-updates: "true" + enable-vtep: "false" + vtep-endpoint: "" + vtep-cidr: "" + vtep-mask: "" + vtep-mac: "" + enable-bgp-control-plane: "false" + procfs: "/host/proc" + bpf-root: "/sys/fs/bpf" + cgroup-root: "/run/cilium/cgroupv2" + enable-k8s-terminating-endpoint: "true" + remove-cilium-node-taints: "true" + set-cilium-is-up-condition: "true" + unmanaged-pod-watcher-interval: "15" + tofqdns-dns-reject-response-code: "refused" + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "50" + tofqdns-idle-connection-grace-period: "0s" + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-min-ttl: "3600" + tofqdns-proxy-response-max-delay: "100ms" + agent-not-ready-taint-key: "ignore-taint.cluster-autoscaler.kubernetes.io/cilium-agent-not-ready" +--- +# Source: cilium/templates/hubble-relay/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hubble-relay-config + namespace: kube-system +data: + config.yaml: | + cluster-name: default + peer-service: "hubble-peer.kube-system.svc.cluster.local:443" + listen-address: :4245 + metrics-listen-address: ":9966" + dial-timeout: + retry-timeout: + sort-buffer-len-max: + sort-buffer-drain-timeout: + tls-client-cert-file: /var/lib/hubble-relay/tls/client.crt + tls-client-key-file: /var/lib/hubble-relay/tls/client.key + tls-hubble-server-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt + disable-server-tls: true +--- +# Source: cilium/templates/hubble-ui/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hubble-ui-nginx + namespace: kube-system +data: + nginx.conf: "server {\n listen 8081;\n listen [::]:8081;\n server_name localhost;\n root /app;\n index index.html;\n client_max_body_size 1G;\n\n location / {\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n\n # CORS\n add_header Access-Control-Allow-Methods \"GET, POST, PUT, HEAD, DELETE, OPTIONS\";\n add_header Access-Control-Allow-Origin *;\n add_header Access-Control-Max-Age 1728000;\n add_header Access-Control-Expose-Headers content-length,grpc-status,grpc-message;\n add_header Access-Control-Allow-Headers range,keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout;\n if ($request_method = OPTIONS) {\n return 204;\n }\n # /CORS\n\n location /api {\n proxy_http_version 1.1;\n proxy_pass_request_headers on;\n proxy_hide_header Access-Control-Allow-Origin;\n proxy_pass http://127.0.0.1:8090;\n }\n\n location / {\n try_files $uri $uri/ /index.html;\n }\n }\n}" +--- +# Source: cilium/templates/cilium-agent/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - pods + - endpoints + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get +- apiGroups: + - cilium.io + resources: + - ciliumbgploadbalancerippools + - ciliumbgppeeringpolicies + - ciliumclusterwideenvoyconfigs + - ciliumclusterwidenetworkpolicies + - ciliumegressgatewaypolicies + - ciliumegressnatpolicies + - ciliumendpoints + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumidentities + - ciliumlocalredirectpolicies + - ciliumnetworkpolicies + - ciliumnodes + verbs: + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + - ciliumendpoints + - ciliumnodes + verbs: + - create +- apiGroups: + - cilium.io + # To synchronize garbage collection of such resources + resources: + - ciliumidentities + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + verbs: + - delete + - get +- apiGroups: + - cilium.io + resources: + - ciliumnodes + - ciliumnodes/status + verbs: + - get + - update +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + - ciliumendpoints/status + - ciliumendpoints + verbs: + - patch +--- +# Source: cilium/templates/cilium-operator/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-operator +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + # To remove node taints + - nodes + # To set NetworkUnavailable false on startup + - nodes/status + verbs: + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform LB IP allocation for BGP + - services/status + verbs: + - update +- apiGroups: + - "" + resources: + # to check apiserver connectivity + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumclusterwidenetworkpolicies + verbs: + # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' + - create + - update + - deletecollection + # To update the status of the CNPs and CCNPs + - patch + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + verbs: + # Update the auto-generated CNPs and CCNPs status. + - patch + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + - ciliumidentities + verbs: + # To perform garbage collection of such resources + - delete + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + verbs: + # To synchronize garbage collection of such resources + - update +- apiGroups: + - cilium.io + resources: + - ciliumnodes + verbs: + - create + - update + - get + - list + - watch + # To perform CiliumNode garbage collector + - delete +- apiGroups: + - cilium.io + resources: + - ciliumnodes/status + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpointslices + - ciliumenvoyconfigs + verbs: + - create + - update + - get + - list + - watch + - delete +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - update + resourceNames: + - ciliumbgploadbalancerippools.cilium.io + - ciliumbgppeeringpolicies.cilium.io + - ciliumclusterwideenvoyconfigs.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumegressgatewaypolicies.cilium.io + - ciliumegressnatpolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumenvoyconfigs.cilium.io + - ciliumexternalworkloads.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodes.cilium.io +# For cilium-operator running in HA mode. +# +# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election +# between multiple running instances. +# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less +# common and fewer objects in the cluster watch "all Leases". +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +--- +# Source: cilium/templates/hubble-ui/clusterrole.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-ui +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - componentstatuses + - endpoints + - namespaces + - nodes + - pods + - services + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - "*" + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/hubble/tls-cronjob/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hubble-generate-certs +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - hubble-server-certs + - hubble-relay-client-certs + - hubble-relay-server-certs + verbs: + - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - cilium-ca + - hubble-ca-secret + verbs: + - get + - update +--- +# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/hubble-ui/clusterrolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-ui +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hubble-ui +subjects: +- kind: ServiceAccount + name: "hubble-ui" + namespace: kube-system +--- +# Source: cilium/templates/hubble/tls-cronjob/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hubble-generate-certs +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hubble-generate-certs +subjects: +- kind: ServiceAccount + name: "hubble-generate-certs" + namespace: kube-system +--- +# Source: cilium/templates/cilium-agent/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: cilium-agent + namespace: kube-system + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9964" + labels: + k8s-app: cilium +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: cilium + ports: + - name: envoy-metrics + port: 9964 + protocol: TCP + targetPort: envoy-metrics +--- +# Source: cilium/templates/hubble-relay/metrics-service.yaml +# We use a separate service from hubble-relay which can be exposed externally +kind: Service +apiVersion: v1 +metadata: + name: hubble-relay-metrics + namespace: kube-system + labels: + k8s-app: hubble-relay +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: hubble-relay + ports: + - name: metrics + port: 9966 + protocol: TCP + targetPort: prometheus +--- +# Source: cilium/templates/hubble-relay/service.yaml +kind: Service +apiVersion: v1 +metadata: + name: hubble-relay + namespace: kube-system + labels: + k8s-app: hubble-relay +spec: + type: "ClusterIP" + selector: + k8s-app: hubble-relay + ports: + - protocol: TCP + port: 80 + targetPort: 4245 +--- +# Source: cilium/templates/hubble-ui/service.yaml +kind: Service +apiVersion: v1 +metadata: + name: hubble-ui + namespace: kube-system + labels: + k8s-app: hubble-ui +spec: + type: "ClusterIP" + selector: + k8s-app: hubble-ui + ports: + - name: http + port: 80 + targetPort: 8081 +--- +# Source: cilium/templates/hubble/metrics-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hubble-metrics + namespace: kube-system + labels: + k8s-app: hubble + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9965" +spec: + clusterIP: None + type: ClusterIP + ports: + - name: hubble-metrics + port: 9965 + protocol: TCP + targetPort: hubble-metrics + selector: + k8s-app: cilium +--- +# Source: cilium/templates/hubble/peer-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hubble-peer + namespace: kube-system + labels: + k8s-app: cilium +spec: + selector: + k8s-app: cilium + ports: + - name: peer-service + port: 443 + protocol: TCP + targetPort: 4244 + internalTrafficPolicy: Local +--- +# Source: cilium/templates/cilium-agent/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium + namespace: kube-system + labels: + k8s-app: cilium +spec: + selector: + matchLabels: + k8s-app: cilium + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9962" + prometheus.io/scrape: "true" + # Set app AppArmor's profile to "unconfined". The value of this annotation + # can be modified as long users know which profiles they have available + # in AppArmor. + container.apparmor.security.beta.kubernetes.io/cilium-agent: "unconfined" + container.apparmor.security.beta.kubernetes.io/clean-cilium-state: "unconfined" + container.apparmor.security.beta.kubernetes.io/mount-cgroup: "unconfined" + container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: "unconfined" + labels: + k8s-app: cilium + spec: + containers: + - name: cilium-agent + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + command: + - cilium-agent + args: + - --config-dir=/tmp/cilium/config-map + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + failureThreshold: 105 + periodSeconds: 2 + successThreshold: 1 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: CILIUM_CNI_CHAINING_MODE + valueFrom: + configMapKeyRef: + name: cilium-config + key: cni-chaining-mode + optional: true + - name: CILIUM_CUSTOM_CNI_CONF + valueFrom: + configMapKeyRef: + name: cilium-config + key: custom-cni-conf + optional: true + lifecycle: + postStart: + exec: + command: + - "/cni-install.sh" + - "--enable-debug=false" + - "--cni-exclusive=true" + - "--log-file=/var/run/cilium/cilium-cni.log" + preStop: + exec: + command: + - /cni-uninstall.sh + ports: + - name: peer-service + containerPort: 4244 + hostPort: 4244 + protocol: TCP + - name: prometheus + containerPort: 9962 + hostPort: 9962 + protocol: TCP + - name: envoy-metrics + containerPort: 9964 + hostPort: 9964 + protocol: TCP + - name: hubble-metrics + containerPort: 9965 + hostPort: 9965 + protocol: TCP + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + # Use to set socket permission + - CHOWN + # Used to terminate envoy child process + - KILL + # Used since cilium modifies routing tables, etc... + - NET_ADMIN + # Used since cilium creates raw sockets, etc... + - NET_RAW + # Used since cilium monitor uses mmap + - IPC_LOCK + # Used in iptables. Consider removing once we are iptables-free + - SYS_MODULE + # We need it for now but might not need it for >= 5.11 specially + # for the 'SYS_RESOURCE'. + # In >= 5.8 there's already BPF and PERMON capabilities + - SYS_ADMIN + # Could be an alternative for the SYS_ADMIN for the RLIMIT_NPROC + - SYS_RESOURCE + # Both PERFMON and BPF requires kernel 5.8, container runtime + # cri-o >= v1.22.0 or containerd >= v1.5.0. + # If available, SYS_ADMIN can be removed. + #- PERFMON + #- BPF + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # Unprivileged containers need to mount /proc/sys/net from the host + # to have write access + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + # Unprivileged containers need to mount /proc/sys/kernel from the host + # to have write access + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - name: bpf-maps + mountPath: /sys/fs/bpf + # Unprivileged containers can't set mount propagation to bidirectional + # in this case we will mount the bpf fs from an init container that + # is privileged and set the mount propagation from host to container + # in Cilium. + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium + - name: etc-cni-netd + mountPath: /host/etc/cni/net.d + - name: clustermesh-secrets + mountPath: /var/lib/cilium/clustermesh + readOnly: true + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + # Needed to be able to load kernel modules + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + - name: hubble-tls + mountPath: /var/lib/cilium/tls/hubble + readOnly: true + initContainers: + # Required to mount cgroup2 filesystem on the underlying Kubernetes node. + # We use nsenter command with host's cgroup and mount namespaces enabled. + - name: mount-cgroup + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + env: + - name: CGROUP_ROOT + value: /run/cilium/cgroupv2 + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh and mount that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + add: + # Only used for 'mount' cgroup + - SYS_ADMIN + # Used for nsenter + - SYS_CHROOT + - SYS_PTRACE + - name: apply-sysctl-overwrites + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + env: + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + add: + # Required in order to access host's /etc/sysctl.d dir + - SYS_ADMIN + # Used for nsenter + - SYS_CHROOT + - SYS_PTRACE + # Mount the bpf fs if it is not mounted. We will perform this task + # from a privileged container because the mount propagation bidirectional + # only works from privileged containers. + - name: mount-bpf-fs + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + args: + - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' + command: + - /bin/bash + - -c + - -- + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + privileged: true + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + - name: clean-cilium-state + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-state + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-bpf-state + optional: true + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + # Most of the capabilities here are the same ones used in the + # cilium-agent's container because this container can be used to + # uninstall all Cilium resources, and therefore it is likely that + # will need the same capabilities. + add: + # Used since cilium modifies routing tables, etc... + - NET_ADMIN + # Used in iptables. Consider removing once we are iptables-free + - SYS_MODULE + # We need it for now but might not need it for >= 5.11 specially + # for the 'SYS_RESOURCE'. + # In >= 5.8 there's already BPF and PERMON capabilities + - SYS_ADMIN + # Could be an alternative for the SYS_ADMIN for the RLIMIT_NPROC + - SYS_RESOURCE + # Both PERFMON and BPF requires kernel 5.8, container runtime + # cri-o >= v1.22.0 or containerd >= v1.5.0. + # If available, SYS_ADMIN can be removed. + #- PERFMON + #- BPF + drop: + - ALL + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + # Required to mount cgroup filesystem from the host to cilium agent pod + - name: cilium-cgroup + mountPath: /run/cilium/cgroupv2 + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium + resources: + requests: + cpu: 100m + memory: 100Mi # wait-for-kube-proxy + # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent + - name: install-cni-binaries + image: "quay.io/cilium/cilium:v1.12.8@sha256:b6c3c48b380334b8f08dba6e0c28d906c0d722b8c2beb0d506b3cea27f66f78d" + imagePullPolicy: IfNotPresent + command: + - "/install-plugin.sh" + resources: + requests: + cpu: 100m + memory: 10Mi + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cni-path + mountPath: /host/opt/cni/bin + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccount: "cilium" + serviceAccountName: "cilium" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + # To mount cgroup2 filesystem on the host + - name: hostproc + hostPath: + path: /proc + type: Directory + # To keep state between restarts / upgrades for cgroup2 filesystem + - name: cilium-cgroup + hostPath: + path: /run/cilium/cgroupv2 + type: DirectoryOrCreate + # To install cilium cni plugin in the host + - name: cni-path + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + # To install cilium cni configuration in the host + - name: etc-cni-netd + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + # To be able to load kernel modules + - name: lib-modules + hostPath: + path: /lib/modules + # To access iptables concurrently with other processes (e.g. kube-proxy) + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # To read the clustermesh configuration + - name: clustermesh-secrets + secret: + secretName: cilium-clustermesh + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + optional: true + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config + - name: host-proc-sys-net + hostPath: + path: /proc/sys/net + type: Directory + - name: host-proc-sys-kernel + hostPath: + path: /proc/sys/kernel + type: Directory + - name: hubble-tls + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-server-certs + optional: true + items: + - key: ca.crt + path: client-ca.crt + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key +--- +# Source: cilium/templates/cilium-operator/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cilium-operator + namespace: kube-system + labels: + io.cilium/app: operator + name: cilium-operator +spec: + # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go + # for more details. + replicas: 2 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9963" + prometheus.io/scrape: "true" + labels: + io.cilium/app: operator + name: cilium-operator + spec: + containers: + - name: cilium-operator + image: "quay.io/cilium/operator-generic:v1.12.8@sha256:7431f0c2001fb875b1a8901e103825394c38cd6c63a1435a3273ed20ae0e7578" + imagePullPolicy: IfNotPresent + command: + - cilium-operator-generic + args: + - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + ports: + - name: prometheus + containerPort: 9963 + hostPort: 9963 + protocol: TCP + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + volumeMounts: + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + terminationMessagePolicy: FallbackToLogsOnError + hostNetwork: true + restartPolicy: Always + priorityClassName: system-cluster-critical + serviceAccount: "cilium-operator" + serviceAccountName: "cilium-operator" + automountServiceAccountToken: true + # In HA mode, cilium-operator pods must not be scheduled on the same + # node as they will clash with each other. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + io.cilium/app: operator + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config +--- +# Source: cilium/templates/hubble-relay/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hubble-relay + namespace: kube-system + labels: + k8s-app: hubble-relay +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: hubble-relay + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + labels: + k8s-app: hubble-relay + spec: + containers: + - name: hubble-relay + image: "quay.io/cilium/hubble-relay:v1.12.8@sha256:508cf85bb1a11c13abd995e3c5fd18ed3c2f1d26cbf463a97297e8b8c9149f13" + imagePullPolicy: IfNotPresent + command: + - hubble-relay + args: + - serve + ports: + - name: grpc + containerPort: 4245 + - name: prometheus + containerPort: 9966 + protocol: TCP + readinessProbe: + tcpSocket: + port: grpc + livenessProbe: + tcpSocket: + port: grpc + volumeMounts: + - name: config + mountPath: /etc/hubble-relay + readOnly: true + - name: tls + mountPath: /var/lib/hubble-relay/tls + readOnly: true + terminationMessagePolicy: FallbackToLogsOnError + restartPolicy: Always + priorityClassName: + serviceAccount: "hubble-relay" + serviceAccountName: "hubble-relay" + automountServiceAccountToken: false + terminationGracePeriodSeconds: 1 + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + volumes: + - name: config + configMap: + name: hubble-relay-config + items: + - key: config.yaml + path: config.yaml + - name: tls + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-relay-client-certs + items: + - key: ca.crt + path: hubble-server-ca.crt + - key: tls.crt + path: client.crt + - key: tls.key + path: client.key +--- +# Source: cilium/templates/hubble-ui/deployment.yaml +kind: Deployment +apiVersion: apps/v1 +metadata: + name: hubble-ui + namespace: kube-system + labels: + k8s-app: hubble-ui +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: hubble-ui + template: + metadata: + annotations: + labels: + k8s-app: hubble-ui + spec: + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsUser: 1001 + priorityClassName: + serviceAccount: "hubble-ui" + serviceAccountName: "hubble-ui" + automountServiceAccountToken: true + containers: + - name: frontend + image: "quay.io/cilium/hubble-ui:v0.9.2@sha256:d3596efc94a41c6b772b9afe6fe47c17417658956e04c3e2a28d293f2670663e" + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8081 + volumeMounts: + - name: hubble-ui-nginx-conf + mountPath: /etc/nginx/conf.d/default.conf + subPath: nginx.conf + - name: tmp-dir + mountPath: /tmp + terminationMessagePolicy: FallbackToLogsOnError + - name: backend + image: "quay.io/cilium/hubble-ui-backend:v0.9.2@sha256:a3ac4d5b87889c9f7cc6323e86d3126b0d382933bd64f44382a92778b0cde5d7" + imagePullPolicy: IfNotPresent + env: + - name: EVENTS_SERVER_PORT + value: "8090" + - name: FLOWS_API_ADDR + value: "hubble-relay:80" + ports: + - name: grpc + containerPort: 8090 + volumeMounts: + terminationMessagePolicy: FallbackToLogsOnError + nodeSelector: + kubernetes.io/os: linux + volumes: + - configMap: + defaultMode: 420 + name: hubble-ui-nginx + name: hubble-ui-nginx-conf + - emptyDir: {} + name: tmp-dir +--- +# Source: cilium/templates/hubble/tls-cronjob/job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: hubble-generate-certs-c26869f410 + namespace: kube-system + labels: + k8s-app: hubble-generate-certs +spec: + template: + metadata: + labels: + k8s-app: hubble-generate-certs + spec: + containers: + - name: certgen + image: "quay.io/cilium/certgen:v0.1.8@sha256:4a456552a5f192992a6edcec2febb1c54870d665173a33dc7d876129b199ddbd" + imagePullPolicy: IfNotPresent + command: + - "/usr/bin/cilium-certgen" + # Because this is executed as a job, we pass the values as command + # line args instead of via config map. This allows users to inspect + # the values used in past runs by inspecting the completed pod. + args: + - "--cilium-namespace=kube-system" + - "--ca-generate" + - "--ca-reuse-secret" + - "--hubble-server-cert-generate" + - "--hubble-server-cert-common-name=*.default.hubble-grpc.cilium.io" + - "--hubble-server-cert-validity-duration=94608000s" + - "--hubble-relay-client-cert-generate" + - "--hubble-relay-client-cert-validity-duration=94608000s" + hostNetwork: true + serviceAccount: "hubble-generate-certs" + serviceAccountName: "hubble-generate-certs" + automountServiceAccountToken: true + restartPolicy: OnFailure + ttlSecondsAfterFinished: 1800 +--- +# Source: cilium/templates/hubble/tls-cronjob/cronjob.yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: hubble-generate-certs + namespace: kube-system + labels: + k8s-app: hubble-generate-certs +spec: + schedule: "0 0 1 */4 *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + metadata: + labels: + k8s-app: hubble-generate-certs + spec: + containers: + - name: certgen + image: "quay.io/cilium/certgen:v0.1.8@sha256:4a456552a5f192992a6edcec2febb1c54870d665173a33dc7d876129b199ddbd" + imagePullPolicy: IfNotPresent + command: + - "/usr/bin/cilium-certgen" + # Because this is executed as a job, we pass the values as command + # line args instead of via config map. This allows users to inspect + # the values used in past runs by inspecting the completed pod. + args: + - "--cilium-namespace=kube-system" + - "--ca-generate" + - "--ca-reuse-secret" + - "--hubble-server-cert-generate" + - "--hubble-server-cert-common-name=*.default.hubble-grpc.cilium.io" + - "--hubble-server-cert-validity-duration=94608000s" + - "--hubble-relay-client-cert-generate" + - "--hubble-relay-client-cert-validity-duration=94608000s" + hostNetwork: true + serviceAccount: "hubble-generate-certs" + serviceAccountName: "hubble-generate-certs" + automountServiceAccountToken: true + restartPolicy: OnFailure + ttlSecondsAfterFinished: 1800 diff --git a/hack/cni/values.yaml b/hack/cni/values.yaml new file mode 100755 index 00000000..82f17a83 --- /dev/null +++ b/hack/cni/values.yaml @@ -0,0 +1,36 @@ +upgradeCompatibility: "1.12" +agentNotReadyTaintKey: ignore-taint.cluster-autoscaler.kubernetes.io/cilium-agent-not-ready +cni: + chainingMode: portmap +sessionAffinity: true +hubble: + metrics: + enabled: + - dns:query;ignoreAAAA + - drop + - tcp + - flow + - icmp + - http + serviceMonitor: + enabled: false + tls: + auto: + method: cronJob + relay: + enabled: true + prometheus: + enabled: true + serviceMonitor: + enabled: false + ui: + enabled: true +prometheus: + enabled: true + serviceMonitor: + enabled: false +operator: + prometheus: + enabled: true + serviceMonitor: + enabled: false