e2e.yaml

name: E2E CI

on:
  release:
    types: [created]
  pull_request:
    types: [opened, synchronize, reopened, edited]
    branches:
      - main
    paths:
      - "**.go"
      - "!**_test.go" # exclude test files to ignore unit test changes
      - "test/**" # include test files in e2e again
      - "!**.md"
      - "Dockerfile.release"
      - ".github/workflows/e2e.yaml"
      - "charts/**"
      - "manifests/**"

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  IMAGE_NAME: ghcr.io/${{ github.repository }}-ci:PR${{ github.event.number }}
  VCLUSTER_SUFFIX: vcluster
  VCLUSTER_NAME: vcluster
  VCLUSTER_NAMESPACE: vcluster

jobs:
  build-and-push-syncer-image:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - run: git fetch --force --tags
      - name: Set up Go
        uses: actions/setup-go@v4
        with:
          cache: false
          go-version: "1.21"
      - name: Setup Just
        uses: extractions/setup-just@v1
      - name: Setup Syft
        uses: anchore/sbom-action/download-syft@v0.14.3
      - name: Setup GoReleaser
        uses: goreleaser/goreleaser-action@v5
        with:
          install-only: true
          version: latest
      - name: Build and save syncer image
        run: |
          set -x
          TELEMETRY_PRIVATE_KEY="" goreleaser build --single-target --snapshot --id vcluster --clean --output ./vcluster
          docker build -t ${{ env.IMAGE_NAME }} -f Dockerfile.release --build-arg TARGETARCH=amd64 --build-arg TARGETOS=linux .
          docker save -o vcluster_syncer ${{ env.IMAGE_NAME }}
      - name: Upload syncer image to artifact
        uses: actions/upload-artifact@v3
        with:
          name: vcluster_syncer
          path: ./vcluster_syncer
          retention-days: 7

  build-vcluster-cli:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - run: git fetch --force --tags
      - name: Set up Go
        uses: actions/setup-go@v4
        with:
          go-version: "1.21"
          cache: false
      - name: Setup Just
        uses: extractions/setup-just@v1
      - name: Setup Syft
        uses: anchore/sbom-action/download-syft@v0.14.3
      - name: Setup GoReleaser
        uses: goreleaser/goreleaser-action@v5
        with:
          install-only: true
      - name: Build vcluster cli
        run: |
          set -x
          TELEMETRY_PRIVATE_KEY="" goreleaser build --single-target --snapshot --id vcluster-cli --clean --output ./vcluster
      - name: Upload vcluster cli to artifact
        uses: actions/upload-artifact@v3
        with:
          name: vcluster
          path: ./vcluster
          retention-days: 7

  get-testsuites-dir:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - id: set-paths-matrix
        run: |
          set -x
          sudo apt-get install -y jq
          paths=$(ls -d ./test/e2e*)
          echo "matrix=$(printf '%s\n' "${paths}" | jq -R . | jq -cs .)" >> "$GITHUB_OUTPUT"
    outputs:
      matrix: ${{ steps.set-paths-matrix.outputs.matrix }}

  e2e-tests:
    name: Execute test suites
    needs: [build-and-push-syncer-image, build-vcluster-cli, get-testsuites-dir]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        distribution: ["k3s", "k8s", "k0s", "eks"]
        test-suite-path: ${{fromJson(needs.get-testsuites-dir.outputs.matrix)}}
        multinamespace-mode: ["false", "true"]
        exclude:
          - distribution: "eks"
            test-suite-path: "./test/e2e_scheduler"
          - distribution: "k8s"
            multinamespace-mode: "true"
          - distribution: "k0s"
            multinamespace-mode: "true"
          - distribution: "eks"
            multinamespace-mode: "true"
          - distribution: "k3s"
            multinamespace-mode: "true"
            test-suite-path: "./test/e2e_target_namespace"
          - distribution: "k3s"
            multinamespace-mode: "true"
            test-suite-path: "./test/e2e_isolation_mode"

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Go
        uses: actions/setup-go@v4
        with:
          go-version: "1.21"
          cache: false

      - uses: azure/setup-helm@v3
        name: Setup Helm
        with:
          version: "v3.11.0"

      - name: Set up kind k8s cluster
        uses: engineerd/setup-kind@v0.5.0
        with:
          version: "v0.20.0"
          image: kindest/node:v1.27.3

      - name: Testing kind cluster set-up
        run: |
          set -x
          kubectl cluster-info
          kubectl get pods -n kube-system
          echo "kubectl config current-context:" $(kubectl config current-context)
          echo "KUBECONFIG env var:" ${KUBECONFIG}

      - name: Download vcluster cli
        uses: actions/download-artifact@v3
        with:
          name: vcluster

      - name: Download syncer image
        uses: actions/download-artifact@v3
        with:
          name: vcluster_syncer

      # - name: Setup upterm session for debugging
      #   uses: lhotari/action-upterm@v1

      - name: Create vcluster
        id: create-vcluster
        run: |
          set -x

          extraArgs=()
          if [ ${{ matrix.multinamespace-mode }} == "true" ]; then
            extraArgs+=( -f ./test/multins_values.yaml )
          fi
          if [ ${{ matrix.test-suite-path }} == "./test/e2e_metrics_proxy" ]; then
            helm repo add metrics-server https://kubernetes-sigs.github.io/metrics-server/
            helm upgrade --install metrics-server metrics-server/metrics-server --set args={--kubelet-insecure-tls} --set containerPort=4443 -n kube-system
          fi

          if [ ${{ matrix.test-suite-path }} == "./test/e2e_target_namespace" ]; then
            kubectl apply -f ${{ matrix.test-suite-path }}/role.yaml
          fi

          sudo apt-get install -y sed

          sed -i "s|REPLACE_IMAGE_NAME|${{ env.IMAGE_NAME }}|g" ${{ matrix.test-suite-path }}/../commonValues.yaml

          kind load image-archive vcluster_syncer

          chmod +x vcluster && sudo mv vcluster /usr/bin

          vcluster create ${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} \
          --create-namespace \
          --debug \
          --connect=false \
          --distro=${{ matrix.distribution }} \
          --local-chart-dir ./charts/${{ matrix.distribution }} \
          -f ./test/commonValues.yaml \
          -f ${{ matrix.test-suite-path }}/values.yaml \
          "${extraArgs[@]}"
        continue-on-error: true

      - name: Wait until vcluster is ready
        id: wait-until-vcluster-is-ready
        if: steps.create-vcluster.outcome == 'success'
        run: |
          set -x
          kubectl wait --for=condition=ready pod -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} --timeout=300s
        continue-on-error: true

      - name: Collect deployment information in case vcluster fails to start
        if: steps.wait-until-vcluster-is-ready.outcome != 'success'
        run: |
          set -x
          kubectl get pods -o yaml -n ${{ env.VCLUSTER_NAMESPACE }}
          echo "======================================================================================================================"
          kubectl get events -n ${{ env.VCLUSTER_NAMESPACE }} --sort-by='.lastTimestamp'
          echo "======================================================================================================================"
          kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c syncer --tail=500
          echo "======================================================================================================================"
          kubectl describe pods -n ${{ env.VCLUSTER_NAMESPACE }}
          exit 1

      # Skips NetworkPolicy tests because they require network plugin with support (e.g. Calico)
      - name: Execute e2e tests
        id: execute-e2e-tests
        working-directory: ${{ matrix.test-suite-path }}
        run: |
          set -x
          VCLUSTER_SUFFIX=${{ env.VCLUSTER_SUFFIX }} VCLUSTER_NAME=${{ env.VCLUSTER_NAME }} VCLUSTER_NAMESPACE=${{ env.VCLUSTER_NAMESPACE }} MULTINAMESPACE_MODE=${{ matrix.multinamespace-mode }} go test -v -ginkgo.v -ginkgo.skip='.*NetworkPolicy.*' -ginkgo.fail-fast
        continue-on-error: true

      - name: Print logs if e2e tests fail
        if: steps.execute-e2e-tests.outcome == 'failure'
        run: |
          set -x
          kubectl get pods -o yaml -n ${{ env.VCLUSTER_NAMESPACE }}
          echo "======================================================================================================================"
          kubectl get events -n ${{ env.VCLUSTER_NAMESPACE }} --sort-by='.lastTimestamp'
          echo "======================================================================================================================"
          kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c syncer --tail=500
          echo "======================================================================================================================"
          kubectl describe pods -n ${{ env.VCLUSTER_NAMESPACE }}
          exit 1