develop.yml

on:
  schedule:
    - cron: "0 15 * * 1-5"
  merge_group:
  pull_request:
    types: [ opened, synchronize, reopened, ready_for_review ]
    paths-ignore:
      - 'docs/**'
      - 'config/**'
      - '**.md'
      - '.dockerignore'
      - 'docker/**'
      - '.gitignore'
      - 'grafana/**'
  workflow_dispatch:

name: CI

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  check-typos-and-docs:
    name: Check typos and docs
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v4
      - uses: crate-ci/typos@master
      - name: Check the config docs
        run: |
          make config-docs && \
          git diff --name-only --exit-code ./config/config.md  \
          || (echo "'config/config.md' is not up-to-date, please run 'make config-docs'." && exit 1)

  license-header-check:
    runs-on: ubuntu-20.04
    name: Check License Header
    steps:
      - uses: actions/checkout@v4
      - uses: korandoru/hawkeye@v5

  check:
    name: Check
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ windows-2022, ubuntu-20.04 ]
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - name: Run cargo check
        run: cargo check --locked --workspace --all-targets

  toml:
    name: Toml Check
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - name: Install taplo
        run: cargo +stable install taplo-cli --version ^0.9 --locked --force
      - name: Run taplo
        run: taplo format --check

  build:
    name: Build GreptimeDB binaries
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ ubuntu-20.04 ]
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - uses: Swatinem/rust-cache@v2
        with:
          # Shares across multiple jobs
          shared-key: "build-binaries"
          cache-all-crates: "true"
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Install cargo-gc-bin
        shell: bash
        run: cargo install cargo-gc-bin --force
      - name: Build greptime binaries
        shell: bash
        # `cargo gc` will invoke `cargo build` with specified args
        run: cargo gc -- --bin greptime --bin sqlness-runner --features pg_kvbackend
      - name: Pack greptime binaries
        shell: bash
        run: |
          mkdir bins && \
          mv ./target/debug/greptime bins && \
          mv ./target/debug/sqlness-runner bins
      - name: Print greptime binaries info
        run: ls -lh bins
      - name: Upload artifacts
        uses: ./.github/actions/upload-artifacts
        with:
          artifacts-dir: bins
          version: current

  fuzztest:
    name: Fuzz Test
    needs: build
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        target: [ "fuzz_create_table", "fuzz_alter_table", "fuzz_create_database", "fuzz_create_logical_table", "fuzz_alter_logical_table", "fuzz_insert", "fuzz_insert_logical_table" ]
    steps:
      - name: Remove unused software
        run: |
          echo "Disk space before:"
          df -h
          [[ -d /usr/share/dotnet ]] && sudo rm -rf /usr/share/dotnet
          [[ -d /usr/local/lib/android ]] && sudo rm -rf /usr/local/lib/android
          [[ -d /opt/ghc ]] && sudo rm -rf /opt/ghc
          [[ -d /opt/hostedtoolcache/CodeQL ]] && sudo rm -rf /opt/hostedtoolcache/CodeQL
          sudo docker image prune --all --force
          sudo docker builder prune -a
          echo "Disk space after:"
          df -h
      - uses: actions/checkout@v4
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - name: Set Rust Fuzz
        shell: bash
        run: |
          sudo apt-get install -y libfuzzer-14-dev
          rustup install nightly
          cargo +nightly install cargo-fuzz cargo-gc-bin --force
      - name: Download pre-built binaries
        uses: actions/download-artifact@v4
        with:
          name: bins
          path: .
      - name: Unzip binaries
        run: |
          tar -xvf ./bins.tar.gz
          rm ./bins.tar.gz
      - name: Run GreptimeDB
        run: |
          ./bins/greptime standalone start&
      - name: Fuzz Test
        uses: ./.github/actions/fuzz-test
        env:
          CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
          GT_MYSQL_ADDR: 127.0.0.1:4002
        with:
          target: ${{ matrix.target }}
          max-total-time: 120

  unstable-fuzztest:
    name: Unstable Fuzz Test
    needs: build-greptime-ci
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      matrix:
        target: [ "unstable_fuzz_create_table_standalone" ]
    steps:
      - name: Remove unused software
        run: |
          echo "Disk space before:"
          df -h
          [[ -d /usr/share/dotnet ]] && sudo rm -rf /usr/share/dotnet
          [[ -d /usr/local/lib/android ]] && sudo rm -rf /usr/local/lib/android
          [[ -d /opt/ghc ]] && sudo rm -rf /opt/ghc
          [[ -d /opt/hostedtoolcache/CodeQL ]] && sudo rm -rf /opt/hostedtoolcache/CodeQL
          sudo docker image prune --all --force
          sudo docker builder prune -a
          echo "Disk space after:"
          df -h
      - uses: actions/checkout@v4
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - name: Set Rust Fuzz
        shell: bash
        run: |
          sudo apt update && sudo apt install -y libfuzzer-14-dev
          cargo install cargo-fuzz cargo-gc-bin --force
      - name: Download pre-built binariy
        uses: actions/download-artifact@v4
        with:
          name: bin
          path: .
      - name: Unzip bianry
        run: |
          tar -xvf ./bin.tar.gz
          rm ./bin.tar.gz
      - name: Run Fuzz Test
        uses: ./.github/actions/fuzz-test
        env:
          CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
          GT_MYSQL_ADDR: 127.0.0.1:4002
          GT_FUZZ_BINARY_PATH: ./bin/greptime
          GT_FUZZ_INSTANCE_ROOT_DIR: /tmp/unstable-greptime/
        with:
          target: ${{ matrix.target }}
          max-total-time: 120
          unstable: 'true'
      - name: Upload unstable fuzz test logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: unstable-fuzz-logs
          path: /tmp/unstable-greptime/
          retention-days: 3

  build-greptime-ci:
    name: Build GreptimeDB binary (profile-CI)
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ ubuntu-20.04 ]
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - uses: Swatinem/rust-cache@v2
        with:
          # Shares across multiple jobs
          shared-key: "build-greptime-ci"
          cache-all-crates: "true"
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Install cargo-gc-bin
        shell: bash
        run: cargo install cargo-gc-bin --force
      - name: Build greptime bianry
        shell: bash
        # `cargo gc` will invoke `cargo build` with specified args
        run: cargo gc --profile ci -- --bin greptime --features pg_kvbackend
      - name: Pack greptime binary
        shell: bash
        run: |
          mkdir bin && \
          mv ./target/ci/greptime bin
      - name: Print greptime binaries info
        run: ls -lh bin
      - name: Upload artifacts
        uses: ./.github/actions/upload-artifacts
        with:
          artifacts-dir: bin
          version: current

  distributed-fuzztest:
    name: Fuzz Test (Distributed, ${{ matrix.mode.name }}, ${{ matrix.target }})
    runs-on: ubuntu-latest
    needs:  build-greptime-ci
    timeout-minutes: 60
    strategy:
      matrix:
        target: [ "fuzz_create_table", "fuzz_alter_table", "fuzz_create_database", "fuzz_create_logical_table", "fuzz_alter_logical_table", "fuzz_insert", "fuzz_insert_logical_table" ]
        mode:
          - name: "Remote WAL"
            minio: true
            kafka: true
            values: "with-remote-wal.yaml"
    steps:
      - name: Remove unused software
        run: |
          echo "Disk space before:"
          df -h
          [[ -d /usr/share/dotnet ]] && sudo rm -rf /usr/share/dotnet
          [[ -d /usr/local/lib/android ]] && sudo rm -rf /usr/local/lib/android
          [[ -d /opt/ghc ]] && sudo rm -rf /opt/ghc
          [[ -d /opt/hostedtoolcache/CodeQL ]] && sudo rm -rf /opt/hostedtoolcache/CodeQL
          sudo docker image prune --all --force
          sudo docker builder prune -a
          echo "Disk space after:"
          df -h
      - uses: actions/checkout@v4
      - name: Setup Kind
        uses: ./.github/actions/setup-kind
      - if: matrix.mode.minio
        name: Setup Minio
        uses: ./.github/actions/setup-minio
      - if: matrix.mode.kafka
        name: Setup Kafka cluser
        uses: ./.github/actions/setup-kafka-cluster
      - name: Setup Etcd cluser
        uses: ./.github/actions/setup-etcd-cluster
      # Prepares for fuzz tests
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - name: Set Rust Fuzz
        shell: bash
        run: |
          sudo apt-get install -y libfuzzer-14-dev
          rustup install nightly
          cargo +nightly install cargo-fuzz cargo-gc-bin --force
      # Downloads ci image
      - name: Download pre-built binariy
        uses: actions/download-artifact@v4
        with:
          name: bin
          path: .
      - name: Unzip binary
        run: |
          tar -xvf ./bin.tar.gz
          rm ./bin.tar.gz
      - name: Build and push GreptimeDB image
        uses: ./.github/actions/build-and-push-ci-image
      - name: Wait for etcd
        run: |
          kubectl wait \
            --for=condition=Ready \
            pod -l app.kubernetes.io/instance=etcd \
            --timeout=120s \
            -n etcd-cluster
      - if: matrix.mode.minio
        name: Wait for minio
        run: |
          kubectl wait \
            --for=condition=Ready \
            pod -l app=minio \
            --timeout=120s \
            -n minio
      - if: matrix.mode.kafka
        name: Wait for kafka
        run: |
          kubectl wait \
            --for=condition=Ready \
            pod -l app.kubernetes.io/instance=kafka \
            --timeout=120s \
            -n kafka-cluster
      - name: Print etcd info
        shell: bash
        run: kubectl get all --show-labels -n etcd-cluster
      # Setup cluster for test
      - name: Setup GreptimeDB cluster
        uses: ./.github/actions/setup-greptimedb-cluster
        with:
          image-registry: localhost:5001
          values-filename: ${{ matrix.mode.values }}
      - name: Port forward (mysql)
        run: |
          kubectl port-forward service/my-greptimedb-frontend 4002:4002 -n my-greptimedb&
      - name: Fuzz Test
        uses: ./.github/actions/fuzz-test
        env:
          CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
          GT_MYSQL_ADDR: 127.0.0.1:4002
        with:
          target: ${{ matrix.target }}
          max-total-time: 120
      - name: Describe Nodes
        if: failure()
        shell: bash
        run: |
          kubectl describe nodes
      - name: Export kind logs
        if: failure()
        shell: bash
        run: |
          kind export logs /tmp/kind
      - name: Upload logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: fuzz-tests-kind-logs-${{ matrix.mode.name }}-${{ matrix.target }}
          path: /tmp/kind
          retention-days: 3
      - name: Delete cluster
        if: success()
        shell: bash
        run: |
          kind delete cluster
          docker stop $(docker ps -a -q)
          docker rm $(docker ps -a -q)
          docker system prune -f

  distributed-fuzztest-with-chaos:
    name: Fuzz Test with Chaos (Distributed, ${{ matrix.mode.name }}, ${{ matrix.target }})
    runs-on: ubuntu-latest
    needs:  build-greptime-ci
    timeout-minutes: 60
    strategy:
      matrix:
        target: ["fuzz_migrate_mito_regions", "fuzz_migrate_metric_regions", "fuzz_failover_mito_regions", "fuzz_failover_metric_regions"]
        mode:
          - name: "Remote WAL"
            minio: true
            kafka: true
            values: "with-remote-wal.yaml"
        include:
          - target: "fuzz_migrate_mito_regions"
            mode:
              name: "Local WAL"
              minio: true
              kafka: false
              values: "with-minio.yaml"
          - target: "fuzz_migrate_metric_regions"
            mode:
              name: "Local WAL"
              minio: true
              kafka: false
              values: "with-minio.yaml"
    steps:
      - name: Remove unused software
        run: |
          echo "Disk space before:"
          df -h
          [[ -d /usr/share/dotnet ]] && sudo rm -rf /usr/share/dotnet
          [[ -d /usr/local/lib/android ]] && sudo rm -rf /usr/local/lib/android
          [[ -d /opt/ghc ]] && sudo rm -rf /opt/ghc
          [[ -d /opt/hostedtoolcache/CodeQL ]] && sudo rm -rf /opt/hostedtoolcache/CodeQL
          sudo docker image prune --all --force
          sudo docker builder prune -a
          echo "Disk space after:"
          df -h
      - uses: actions/checkout@v4
      - name: Setup Kind
        uses: ./.github/actions/setup-kind
      - name: Setup Chaos Mesh
        uses: ./.github/actions/setup-chaos
      - if: matrix.mode.minio
        name: Setup Minio
        uses: ./.github/actions/setup-minio
      - if: matrix.mode.kafka
        name: Setup Kafka cluser
        uses: ./.github/actions/setup-kafka-cluster
      - name: Setup Etcd cluser
        uses: ./.github/actions/setup-etcd-cluster
      # Prepares for fuzz tests
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - name: Set Rust Fuzz
        shell: bash
        run: |
          sudo apt-get install -y libfuzzer-14-dev
          rustup install nightly
          cargo +nightly install cargo-fuzz cargo-gc-bin --force
      # Downloads ci image
      - name: Download pre-built binariy
        uses: actions/download-artifact@v4
        with:
          name: bin
          path: .
      - name: Unzip binary
        run: |
          tar -xvf ./bin.tar.gz
          rm ./bin.tar.gz
      - name: Build and push GreptimeDB image
        uses: ./.github/actions/build-and-push-ci-image
      - name: Wait for etcd
        run: |
          kubectl wait \
            --for=condition=Ready \
            pod -l app.kubernetes.io/instance=etcd \
            --timeout=120s \
            -n etcd-cluster
      - if: matrix.mode.minio
        name: Wait for minio
        run: |
          kubectl wait \
            --for=condition=Ready \
            pod -l app=minio \
            --timeout=120s \
            -n minio
      - if: matrix.mode.kafka
        name: Wait for kafka
        run: |
          kubectl wait \
            --for=condition=Ready \
            pod -l app.kubernetes.io/instance=kafka \
            --timeout=120s \
            -n kafka-cluster
      - name: Print etcd info
        shell: bash
        run: kubectl get all --show-labels -n etcd-cluster
      # Setup cluster for test
      - name: Setup GreptimeDB cluster
        uses: ./.github/actions/setup-greptimedb-cluster
        with:
          image-registry: localhost:5001
          values-filename: ${{ matrix.mode.values }}
          enable-region-failover: ${{ matrix.mode.kafka }}
      - name: Port forward (mysql)
        run: |
          kubectl port-forward service/my-greptimedb-frontend 4002:4002 -n my-greptimedb&
      - name: Fuzz Test
        uses: ./.github/actions/fuzz-test
        env:
          CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
          GT_MYSQL_ADDR: 127.0.0.1:4002
        with:
          target: ${{ matrix.target }}
          max-total-time: 120
      - name: Describe Nodes
        if: failure()
        shell: bash
        run: |
          kubectl describe nodes
      - name: Export kind logs
        if: failure()
        shell: bash
        run: |
          kind export logs /tmp/kind
      - name: Upload logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: fuzz-tests-kind-logs-${{ matrix.mode.name }}-${{ matrix.target }}
          path: /tmp/kind
          retention-days: 3
      - name: Delete cluster
        if: success()
        shell: bash
        run: |
          kind delete cluster
          docker stop $(docker ps -a -q)
          docker rm $(docker ps -a -q)
          docker system prune -f

  sqlness:
    name: Sqlness Test (${{ matrix.mode.name }})
    needs: build
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ ubuntu-20.04 ]
        mode:
          - name: "Basic"
            opts: ""
            kafka: false
          - name: "Remote WAL"
            opts: "-w kafka -k 127.0.0.1:9092"
            kafka: true
          - name: "Pg Kvbackend"
            opts: "--setup-pg"
            kafka: false
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - if: matrix.mode.kafka
        name: Setup kafka server
        working-directory: tests-integration/fixtures
        run: docker compose up -d --wait kafka
      - name: Download pre-built binaries
        uses: actions/download-artifact@v4
        with:
          name: bins
          path: .
      - name: Unzip binaries
        run: tar -xvf ./bins.tar.gz
      - name: Run sqlness
        run: RUST_BACKTRACE=1 ./bins/sqlness-runner ${{ matrix.mode.opts }} -c ./tests/cases --bins-dir ./bins --preserve-state
      - name: Upload sqlness logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: sqlness-logs-${{ matrix.mode.name }}
          path: /tmp/sqlness*
          retention-days: 3

  fmt:
    name: Rustfmt
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          components: rustfmt
      - name: Check format
        run: make fmt-check

  clippy:
    name: Clippy
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          components: clippy
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
        with:
          # Shares across multiple jobs
          # Shares with `Check` job
          shared-key: "check-lint"
          cache-all-crates: "true"
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Run cargo clippy
        run: make clippy

  conflict-check:
    name: Check for conflict
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Merge Conflict Finder
        uses: olivernybroe/action-conflict-finder@v4.0

  coverage:
    if: github.event.pull_request.draft == false
    runs-on: ubuntu-20.04-8-cores
    timeout-minutes: 60
    needs:  [conflict-check, clippy, fmt]
    steps:
      - uses: actions/checkout@v4
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: rui314/setup-mold@v1
      - name: Install toolchain
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          components: llvm-tools
          cache: false
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
        with:
          # Shares cross multiple jobs
          shared-key: "coverage-test"
          cache-all-crates: "true"
          save-if: ${{ github.ref == 'refs/heads/main' }}
      # Disabled temporarily to see performance
      # - name: Docker Cache
      #   uses: ScribeMD/docker-cache@0.5.0
      #   with:
      #     key: docker-${{ runner.os }}-coverage
      - name: Install latest nextest release
        uses: taiki-e/install-action@nextest
      - name: Install cargo-llvm-cov
        uses: taiki-e/install-action@cargo-llvm-cov
      - name: Setup external services
        working-directory: tests-integration/fixtures
        run: docker compose up -d --wait
      - name: Run nextest cases
        run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F dashboard -F pg_kvbackend
        env:
          CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
          RUST_BACKTRACE: 1
          CARGO_INCREMENTAL: 0
          GT_S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
          GT_S3_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
          GT_S3_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
          GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
          GT_MINIO_BUCKET: greptime
          GT_MINIO_ACCESS_KEY_ID: superpower_ci_user
          GT_MINIO_ACCESS_KEY: superpower_password
          GT_MINIO_REGION: us-west-2
          GT_MINIO_ENDPOINT_URL: http://127.0.0.1:9000
          GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
          GT_POSTGRES_ENDPOINTS: postgres://greptimedb:admin@127.0.0.1:5432/postgres
          GT_KAFKA_ENDPOINTS: 127.0.0.1:9092
          GT_KAFKA_SASL_ENDPOINTS: 127.0.0.1:9093
          UNITTEST_LOG_DIR: "__unittest_logs"
      - name: Codecov upload
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./lcov.info
          flags: rust
          fail_ci_if_error: false
          verbose: true

  # compat:
  #   name: Compatibility Test
  #   needs: build
  #   runs-on: ubuntu-20.04
  #   timeout-minutes: 60
  #   steps:
  #     - uses: actions/checkout@v4
  #     - name: Download pre-built binaries
  #       uses: actions/download-artifact@v4
  #       with:
  #         name: bins
  #         path: .
  #     - name: Unzip binaries
  #       run: |
  #         mkdir -p ./bins/current
  #         tar -xvf ./bins.tar.gz --strip-components=1 -C ./bins/current
  #     - run: ./tests/compat/test-compat.sh 0.6.0