From 927d6ab6e39477c9561d8fc962bd7257dc9f2262 Mon Sep 17 00:00:00 2001 From: l1b0k Date: Tue, 15 Oct 2024 11:27:45 +0800 Subject: [PATCH 1/3] add ct full logging Signed-off-by: l1b0k --- policy/cilium/0033-logging.patch | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 policy/cilium/0033-logging.patch diff --git a/policy/cilium/0033-logging.patch b/policy/cilium/0033-logging.patch new file mode 100644 index 00000000..a0c85985 --- /dev/null +++ b/policy/cilium/0033-logging.patch @@ -0,0 +1,51 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: l1b0k +Date: Mon, 14 Oct 2024 16:33:42 +0800 +Subject: logging + +Signed-off-by: l1b0k +--- + pkg/maps/ctmap/ctmap.go | 19 +++++++++++++------ + 1 file changed, 13 insertions(+), 6 deletions(-) + +diff --git a/pkg/maps/ctmap/ctmap.go b/pkg/maps/ctmap/ctmap.go +index 868871811b..052cab28e9 100644 +--- a/pkg/maps/ctmap/ctmap.go ++++ b/pkg/maps/ctmap/ctmap.go +@@ -496,6 +496,13 @@ func doGC4(m *Map, filter *GCFilter) gcStats { + globalDeleteLock[m.mapType].Lock() + stats.dumpError = m.DumpReliablyWithCallback(filterCallback, stats.DumpStats) + globalDeleteLock[m.mapType].Unlock() ++ ++ log.Infof("gc map %s id %d max %d deleted %d alived %d", m.Name(), m.InnerID, m.MaxEntries, stats.deleted, stats.aliveEntries) ++ cur := stats.aliveEntries + stats.deleted ++ if float64(cur)/float64(m.MaxEntries) >= 0.9 { ++ log.Infof("ConntrackFull table %s current %d maxEntries %d", m.Name(), cur, m.MaxEntries) ++ } ++ + return stats + } + +@@ -566,13 +573,13 @@ func GC(m *Map, filter *GCFilter) int { + // The consumer of the buffer invokes the function. + // + // The SNAT is being used for the following cases: +-// 1. By NodePort BPF on an intermediate node before fwd'ing request from outside ++// 1. By NodePort BPF on an intermediate node before fwd'ing request from outside + // to a destination node. +-// 2. A packet from local endpoint sent to outside (BPF-masq). +-// 3. A packet from a host local application (i.e. running in the host netns) +-// This is needed to prevent SNAT from hijacking such connections. +-// 4. By DSR on a backend node to SNAT responses with service IP+port before +-// sending to a client. ++// 2. A packet from local endpoint sent to outside (BPF-masq). ++// 3. A packet from a host local application (i.e. running in the host netns) ++// This is needed to prevent SNAT from hijacking such connections. ++// 4. By DSR on a backend node to SNAT responses with service IP+port before ++// sending to a client. + // + // In the case of 1-3, we always create a CT_EGRESS CT entry. This allows the + // CT GC to remove corresponding SNAT entries. In the case of 4, will create +-- +2.47.0 + From c1db82892ca9404b6e525f6ae3088fb03d448ea1 Mon Sep 17 00:00:00 2001 From: l1b0k Date: Tue, 15 Oct 2024 15:43:20 +0800 Subject: [PATCH 2/3] update policy image Signed-off-by: l1b0k --- .github/workflows/check.yml | 6 +++--- Dockerfile | 16 ++++++++-------- Dockerfile.controlplane | 6 +++--- Dockerfile.policy | 4 ++-- Makefile | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index d94cbe7d..5f956e01 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: 1.21.3 + go-version: 1.23.2 - name: Test run: | go=$(which go) @@ -35,7 +35,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: 1.21.3 + go-version: 1.23.2 - name: Check module vendoring run: | go mod tidy @@ -49,7 +49,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: 1.21.3 + go-version: 1.23.2 cache: false - name: Run golangci-lint uses: golangci/golangci-lint-action@v4 diff --git a/Dockerfile b/Dockerfile index ff3d19dc..09ba114a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,21 @@ -ARG TERWAY_POLICY_IMAGE=registry-cn-zhangjiakou.ack.aliyuncs.com/acs/terway:policy-75c98940@sha256:6dbdffee0cdc5c29239d487b4e567046d7dd23f61d67dbbbce1a2e5db9b210dc +ARG TERWAY_POLICY_IMAGE=registry-cn-zhangjiakou.ack.aliyuncs.com/acs/terway:policy-927d6ab6@sha256:dbcc2cef1164b7ce0de7700cefbdece7ca0281d84e5db91ce96488f1a2c00ed7 ARG UBUNTU_IMAGE=registry.cn-hangzhou.aliyuncs.com/acs/ubuntu:22.04-update ARG CILIUM_LLVM_IMAGE=quay.io/cilium/cilium-llvm:547db7ec9a750b8f888a506709adb41f135b952e@sha256:4d6fa0aede3556c5fb5a9c71bc6b9585475ac9b1064f516d4c45c8fb691c9d9e ARG CILIUM_BPFTOOL_IMAGE=quay.io/cilium/cilium-bpftool:78448c1a37ff2b790d5e25c3d8b8ec3e96e6405f@sha256:99a9453a921a8de99899ef82e0822f0c03f65d97005c064e231c06247ad8597d ARG CILIUM_IPROUTE2_IMAGE=quay.io/cilium/cilium-iproute2:3570d58349efb2d6b0342369a836998c93afd291@sha256:1abcd7a5d2117190ab2690a163ee9cd135bc9e4cf8a4df662a8f993044c79342 ARG CILIUM_IPTABLES_IMAGE=quay.io/cilium/iptables-20.04:e6f83206c57e606282056903ffd3aab0183bdaed@sha256:7ce0de449d356a5259021dc13f2b00a8bddfbea57a1c91ff8f146d455cace9e5 -FROM --platform=$TARGETPLATFORM ${TERWAY_POLICY_IMAGE} as policy-dist -FROM --platform=$TARGETPLATFORM ${CILIUM_LLVM_IMAGE} as llvm-dist -FROM --platform=$TARGETPLATFORM ${CILIUM_BPFTOOL_IMAGE} as bpftool-dist -FROM --platform=$TARGETPLATFORM ${CILIUM_IPROUTE2_IMAGE} as iproute2-dist -FROM --platform=$TARGETPLATFORM ${CILIUM_IPTABLES_IMAGE} as iptables-dist +FROM --platform=$TARGETPLATFORM ${TERWAY_POLICY_IMAGE} AS policy-dist +FROM --platform=$TARGETPLATFORM ${CILIUM_LLVM_IMAGE} AS llvm-dist +FROM --platform=$TARGETPLATFORM ${CILIUM_BPFTOOL_IMAGE} AS bpftool-dist +FROM --platform=$TARGETPLATFORM ${CILIUM_IPROUTE2_IMAGE} AS iproute2-dist +FROM --platform=$TARGETPLATFORM ${CILIUM_IPTABLES_IMAGE} AS iptables-dist -FROM --platform=$BUILDPLATFORM golang:1.21.3 as builder +FROM --platform=$BUILDPLATFORM golang:1.23.2 AS builder ARG GOPROXY ARG TARGETOS ARG TARGETARCH -ENV GOPROXY $GOPROXY +ENV GOPROXY=$GOPROXY WORKDIR /go/src/github.com/AliyunContainerService/terway/ COPY go.sum go.mod ./ RUN go mod download diff --git a/Dockerfile.controlplane b/Dockerfile.controlplane index 88bf21dc..1752376d 100644 --- a/Dockerfile.controlplane +++ b/Dockerfile.controlplane @@ -1,8 +1,8 @@ -FROM --platform=$BUILDPLATFORM golang:1.21.3 as builder +FROM --platform=$BUILDPLATFORM golang:1.23.2 AS builder ARG GOPROXY ARG TARGETOS ARG TARGETARCH -ENV GOPROXY $GOPROXY +ENV GOPROXY=$GOPROXY WORKDIR /go/src/github.com/AliyunContainerService/terway/ COPY go.sum go.mod ./ RUN go mod download @@ -15,7 +15,7 @@ RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -tags default_build -X \"github.com/AliyunContainerService/terway/pkg/aliyun/credential.kubernetesAlicloudIdentity=terway-controlplane/`git rev-parse --short HEAD 2>/dev/null`\"" \ -o terway-controlplane cmd/terway-controlplane/terway-controlplane.go -FROM --platform=$TARGETPLATFORM debian:stable-slim as cert +FROM --platform=$TARGETPLATFORM debian:stable-slim AS cert RUN apt-get update && apt-get -uy upgrade RUN apt-get -y install ca-certificates && update-ca-certificates diff --git a/Dockerfile.policy b/Dockerfile.policy index cf28caff..4fff7bdb 100644 --- a/Dockerfile.policy +++ b/Dockerfile.policy @@ -1,4 +1,4 @@ -FROM --platform=$TARGETPLATFORM calico/go-build:v0.90 as felix-builder +FROM --platform=$TARGETPLATFORM calico/go-build:v0.90 AS felix-builder ARG GOPROXY ARG GIT_VERSION ENV GOPROXY $GOPROXY @@ -21,7 +21,7 @@ RUN cd /go/src/github.com/projectcalico/calico && \ FROM --platform=$TARGETPLATFORM quay.io/cilium/cilium-builder:1d3ec0f0b74a32048a9716c7a8ce1eee851ca0ec@sha256:9fab9eb021456705d99b014d2f9e59aff9f50aa1a296aa55e984f3e947a62120 as cilium-builder ARG GOPROXY -ENV GOPROXY $GOPROXY +ENV GOPROXY=$GOPROXY ARG CILIUM_SHA="" ARG GIT_VERSION="" LABEL cilium-sha=${CILIUM_SHA} diff --git a/Makefile b/Makefile index 19c35fe1..a19a8d66 100644 --- a/Makefile +++ b/Makefile @@ -113,7 +113,7 @@ GOLANGCI_LINT = $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION) ## Tool Versions CONTROLLER_TOOLS_VERSION ?= v0.14.0 ENVTEST_VERSION ?= latest -GOLANGCI_LINT_VERSION ?= v1.54.2 +GOLANGCI_LINT_VERSION ?= v1.55.2 .PHONY: controller-gen controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. From 573884f3298994454bc28a0ac27fc60fbcef52e1 Mon Sep 17 00:00:00 2001 From: l1b0k Date: Tue, 15 Oct 2024 19:13:28 +0800 Subject: [PATCH 3/3] fix lint Signed-off-by: l1b0k --- .github/workflows/build-policy.yml | 8 +++---- .github/workflows/check.yml | 4 ++-- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/release.yml | 2 +- Makefile | 2 +- pkg/controller/webhook/mutating.go | 7 +++--- plugin/driver/utils/netlink_linux.go | 32 +++++++++++++-------------- 7 files changed, 29 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build-policy.yml b/.github/workflows/build-policy.yml index 38b7e3b8..b87ed5b1 100644 --- a/.github/workflows/build-policy.yml +++ b/.github/workflows/build-policy.yml @@ -17,12 +17,12 @@ jobs: build-policy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Cache Docker layers uses: actions/cache@v3 @@ -51,7 +51,7 @@ jobs: if: ${{ github.event_name != 'pull_request' && github.event.action != 'unassigned' }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v6 with: context: . file: ./Dockerfile.policy diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 5f956e01..8bb5f2bc 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -52,9 +52,9 @@ jobs: go-version: 1.23.2 cache: false - name: Run golangci-lint - uses: golangci/golangci-lint-action@v4 + uses: golangci/golangci-lint-action@v6 with: - version: v1.55 + version: v1.61 args: --config=.golangci.yml super-linter: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index fda94871..6029095b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -36,7 +36,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Initialize CodeQL uses: github/codeql-action/init@v3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a4ff77c3..af99c2ea 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,7 @@ jobs: release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build Changelog id: github_release uses: mikepenz/release-changelog-builder-action@v1 diff --git a/Makefile b/Makefile index a19a8d66..1d3e8954 100644 --- a/Makefile +++ b/Makefile @@ -113,7 +113,7 @@ GOLANGCI_LINT = $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION) ## Tool Versions CONTROLLER_TOOLS_VERSION ?= v0.14.0 ENVTEST_VERSION ?= latest -GOLANGCI_LINT_VERSION ?= v1.55.2 +GOLANGCI_LINT_VERSION ?= v1.61.0 .PHONY: controller-gen controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. diff --git a/pkg/controller/webhook/mutating.go b/pkg/controller/webhook/mutating.go index 15c3e576..55993ebc 100644 --- a/pkg/controller/webhook/mutating.go +++ b/pkg/controller/webhook/mutating.go @@ -18,6 +18,7 @@ package webhook import ( "context" + "errors" "fmt" "net/http" "strconv" @@ -35,7 +36,7 @@ import ( "gomodules.xyz/jsonpatch/v2" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" + k8sErr "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/json" @@ -101,7 +102,7 @@ func podWebhook(ctx context.Context, req *webhook.AdmissionRequest, client clien if err != nil { msg := fmt.Sprintf("error get previous podENI conf, %s", err) l.Error(err, msg) - return webhook.Errored(1, fmt.Errorf(msg)) + return webhook.Errored(1, errors.New(msg)) } // 1. check pod annotation config first @@ -348,7 +349,7 @@ func getPreviousZone(ctx context.Context, client client.Client, pod *corev1.Pod) Name: pod.Name, }, podENI) if err != nil { - if errors.IsNotFound(err) { + if k8sErr.IsNotFound(err) { return "", nil } return "", err diff --git a/plugin/driver/utils/netlink_linux.go b/plugin/driver/utils/netlink_linux.go index 9bf26e70..fca99e66 100644 --- a/plugin/driver/utils/netlink_linux.go +++ b/plugin/driver/utils/netlink_linux.go @@ -33,7 +33,7 @@ func NetlinkFamily(ip net.IP) int { func LinkSetName(link netlink.Link, name string) error { cmd := fmt.Sprintf("ip link set %s name %s", link.Attrs().Name, name) - Log.Infof(cmd) + Log.Info(cmd) err := netlink.LinkSetName(link, name) if err != nil { return fmt.Errorf("error %s, %w", cmd, err) @@ -43,7 +43,7 @@ func LinkSetName(link netlink.Link, name string) error { func LinkAdd(link netlink.Link) error { cmd := fmt.Sprintf("ip link add %s type %s", link.Attrs().Name, link.Type()) - Log.Infof(cmd) + Log.Info(cmd) err := netlink.LinkAdd(link) if err != nil { return fmt.Errorf("error %s, %w", cmd, err) @@ -53,7 +53,7 @@ func LinkAdd(link netlink.Link) error { func LinkSetUp(link netlink.Link) error { cmd := fmt.Sprintf("ip link set %s up", link.Attrs().Name) - Log.Infof(cmd) + Log.Info(cmd) err := netlink.LinkSetUp(link) if err != nil { return fmt.Errorf("error %s, %w", cmd, err) @@ -63,7 +63,7 @@ func LinkSetUp(link netlink.Link) error { func LinkSetDown(link netlink.Link) error { cmd := fmt.Sprintf("ip link set %s down", link.Attrs().Name) - Log.Infof(cmd) + Log.Info(cmd) err := netlink.LinkSetDown(link) if err != nil { return fmt.Errorf("error %s, %w", cmd, err) @@ -73,7 +73,7 @@ func LinkSetDown(link netlink.Link) error { func LinkDel(link netlink.Link) error { cmd := fmt.Sprintf("ip link del %s", link.Attrs().Name) - Log.Infof(cmd) + Log.Info(cmd) err := netlink.LinkDel(link) if err != nil { if _, ok := err.(netlink.LinkNotFoundError); ok { @@ -86,7 +86,7 @@ func LinkDel(link netlink.Link) error { func LinkSetMTU(link netlink.Link, mtu int) error { cmd := fmt.Sprintf("ip link set %s mtu %d", link.Attrs().Name, mtu) - Log.Infof(cmd) + Log.Info(cmd) err := netlink.LinkSetMTU(link, mtu) if err != nil { return fmt.Errorf("error %s, %w", cmd, err) @@ -96,7 +96,7 @@ func LinkSetMTU(link netlink.Link, mtu int) error { func AddrDel(link netlink.Link, addr *netlink.Addr) error { cmd := fmt.Sprintf("ip addr del %s dev %s", addr.IPNet.String(), link.Attrs().Name) - Log.Infof(cmd) + Log.Info(cmd) err := netlink.AddrDel(link, addr) if err != nil { return fmt.Errorf("error %s, %w", cmd, err) @@ -106,7 +106,7 @@ func AddrDel(link netlink.Link, addr *netlink.Addr) error { func AddrReplace(link netlink.Link, addr *netlink.Addr) error { cmd := fmt.Sprintf("ip addr replace %s dev %s", addr.IPNet.String(), link.Attrs().Name) - Log.Infof(cmd) + Log.Info(cmd) err := netlink.AddrReplace(link, addr) if err != nil { return fmt.Errorf("error %s, %w", cmd, err) @@ -116,7 +116,7 @@ func AddrReplace(link netlink.Link, addr *netlink.Addr) error { func RouteReplace(route *netlink.Route) error { cmd := fmt.Sprintf("ip route replace %s", route.String()) - Log.Infof(cmd) + Log.Info(cmd) err := netlink.RouteReplace(route) if err != nil { return fmt.Errorf("error %s, %w", cmd, err) @@ -126,7 +126,7 @@ func RouteReplace(route *netlink.Route) error { func RouteDel(route *netlink.Route) error { cmd := fmt.Sprintf("ip route del %s", route.String()) - Log.Infof(cmd) + Log.Info(cmd) err := netlink.RouteDel(route) if err != nil { return fmt.Errorf("error %s, %w", cmd, err) @@ -136,7 +136,7 @@ func RouteDel(route *netlink.Route) error { func NeighSet(neigh *netlink.Neigh) error { cmd := fmt.Sprintf("ip neigh replace %s", neigh.String()) - Log.Infof(cmd) + Log.Info(cmd) err := netlink.NeighSet(neigh) if err != nil { return fmt.Errorf("error %s, %w", cmd, err) @@ -146,7 +146,7 @@ func NeighSet(neigh *netlink.Neigh) error { func RuleAdd(rule *netlink.Rule) error { cmd := fmt.Sprintf("ip rule add %s", rule.String()) - Log.Infof(cmd) + Log.Info(cmd) err := netlink.RuleAdd(rule) if err != nil { return fmt.Errorf("error %s, %w", cmd, err) @@ -156,7 +156,7 @@ func RuleAdd(rule *netlink.Rule) error { func RuleDel(rule *netlink.Rule) error { cmd := fmt.Sprintf("ip rule del %s", rule.String()) - Log.Infof(cmd) + Log.Info(cmd) err := netlink.RuleDel(rule) if err != nil { rule.IifName = "" @@ -172,7 +172,7 @@ func RuleDel(rule *netlink.Rule) error { func LinkSetNsFd(link netlink.Link, netNS ns.NetNS) error { cmd := fmt.Sprintf("ip link set %s netns %s", link.Attrs().Name, netNS.Path()) - Log.Infof(cmd) + Log.Info(cmd) err := netlink.LinkSetNsFd(link, int(netNS.Fd())) if err != nil { return fmt.Errorf("error %s, %w", cmd, err) @@ -182,7 +182,7 @@ func LinkSetNsFd(link netlink.Link, netNS ns.NetNS) error { func QdiscReplace(qdisc netlink.Qdisc) error { cmd := fmt.Sprintf("tc qdisc replace %s", qdisc.Attrs().String()) - Log.Infof(cmd) + Log.Info(cmd) err := netlink.QdiscReplace(qdisc) if err != nil { return fmt.Errorf("error %s, %w", cmd, err) @@ -191,7 +191,7 @@ func QdiscReplace(qdisc netlink.Qdisc) error { } func QdiscDel(qdisc netlink.Qdisc) error { cmd := fmt.Sprintf("tc qdisc del %s", qdisc.Attrs().String()) - Log.Infof(cmd) + Log.Info(cmd) err := netlink.QdiscDel(qdisc) if err != nil { return fmt.Errorf("error %s, %w", cmd, err)