diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index 02f1ccb692..0f33ae90c9 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -92,21 +92,21 @@ jobs: with: dotnet-version: ${{ matrix.dotnet }} -# - name: Install shared software dependencies -# uses: ./.github/workflows/install-shared-dependencies -# with: -# os: ${{ matrix.host.OS }} -# target: ${{ matrix.host.TARGET }} -# github-token: ${{ secrets.GITHUB_TOKEN }} -# engine-version: ${{ matrix.engine.version }} + - name: Install shared software dependencies + uses: ./.github/workflows/install-shared-dependencies + with: + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: ${{ matrix.engine.version }} -# - name: Test dotnet ${{ matrix.dotnet }} -# working-directory: csharp -# run: dotnet test --framework net${{ matrix.dotnet }} "-l:html;LogFileName=TestReport.html" --results-directory . -warnaserror + - name: Test dotnet ${{ matrix.dotnet }} + working-directory: csharp + run: dotnet test --framework net${{ matrix.dotnet }} "-l:html;LogFileName=TestReport.html" --results-directory . -warnaserror -# - uses: ./.github/workflows/test-benchmark -# with: -# language-flag: -csharp -dotnet-framework net${{ matrix.dotnet }} + - uses: ./.github/workflows/test-benchmark + with: + language-flag: -csharp -dotnet-framework net${{ matrix.dotnet }} - name: Upload test reports if: always() diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 7d87e02ed3..562af25723 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -55,9 +55,6 @@ run-name: jobs: get-matrices: runs-on: ubuntu-latest - # Avoid running on schedule for forks - if: (github.event_name != 'schedule') || github.event_name == 'push' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' -# if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') || github.event_name == 'push' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' outputs: engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} @@ -93,7 +90,49 @@ jobs: go-version: ${{ matrix.go }} cache-dependency-path: go/go.sum - lint-rust: + - name: Install shared software dependencies + uses: ./.github/workflows/install-shared-dependencies + with: + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: ${{ matrix.engine.version }} + + - name: Install tools for Go ${{ matrix.go }} + working-directory: ./go + run: make install-tools-go${{ matrix.go }} + + - name: Set LD_LIBRARY_PATH + if: ${{ matrix.os == 'ubuntu-latest' }} + run: echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/go/target/release/deps/" >> $GITHUB_ENV + + - name: Build client + working-directory: ./go + run: make build + + - name: Run linters # TODO move + working-directory: ./go + run: make lint-ci + + - name: Run tests + working-directory: ./go + run: make test + + - uses: ./.github/workflows/test-benchmark + with: + language-flag: -go + + - name: Upload logs and reports + if: always() + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: test-report-go-${{ matrix.go }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }} + path: | + utils/clusters/** + benchmarks/results/** + + lint: timeout-minutes: 10 runs-on: ubuntu-latest steps: @@ -105,3 +144,94 @@ jobs: with: cargo-toml-folder: go github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Go ${{ matrix.go }} + uses: actions/setup-go@v5 + with: +# go-version: ${{ matrix.go }} + cache-dependency-path: go/go.sum + +# - name: Install tools for Go ${{ matrix.go }} +# working-directory: ./go +# run: # install-tools-go${{ matrix.go }} + + - name: Install and run linters + working-directory: go + run: | + make install-dev-tools + echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/go/target/release/deps/" >> $GITHUB_ENV + make lint-ci + +# build-amazonlinux-latest: +# if: github.repository_owner == 'valkey-io' +# strategy: +# # Run all jobs +# fail-fast: false +# matrix: +# go: +# - 1.22.0 +# runs-on: ubuntu-latest +# container: amazonlinux:latest +# timeout-minutes: 15 +# steps: +# - name: Install git +# run: | +# yum -y remove git +# yum -y remove git-* +# yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm +# yum update +# yum install -y git +# git --version +# +# - uses: actions/checkout@v4 +# +# - name: Checkout submodules +# run: | +# git config --global --add safe.directory "$GITHUB_WORKSPACE" +# git submodule update --init --recursive +# +# - name: Install shared software dependencies +# uses: ./.github/workflows/install-shared-dependencies +# with: +# os: "amazon-linux" +# target: "x86_64-unknown-linux-gnu" +# github-token: ${{ secrets.GITHUB_TOKEN }} +# engine-version: "7.2.5" +# +# - name: Install Go +# run: | +# yum -y install wget +# yum -y install tar +# wget https://go.dev/dl/go${{ matrix.go }}.linux-amd64.tar.gz +# tar -C /usr/local -xzf go${{ matrix.go }}.linux-amd64.tar.gz +# echo "/usr/local/go/bin" >> $GITHUB_PATH +# echo "$HOME/go/bin" >> $GITHUB_PATH +# +# - name: Install tools for Go ${{ matrix.go }} +# working-directory: ./go +# run: make install-tools-go${{ matrix.go }} +# +# - name: Set LD_LIBRARY_PATH +# run: echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/go/target/release/deps/" >> $GITHUB_ENV +# +# - name: Build client +# working-directory: ./go +# run: make build +# +# - name: Run linters +# working-directory: ./go +# run: make lint-ci +# +# - name: Run tests +# working-directory: ./go +# run: | +# make test +# +# - name: Upload cluster manager logs +# if: always() +# continue-on-error: true +# uses: actions/upload-artifact@v4 +# with: +# name: cluster-manager-logs-${{ matrix.go }}-redis-6-amazonlinux +# path: | +# utils/clusters/** diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 5ea0ab2d7a..51404bd19f 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -73,7 +73,6 @@ jobs: language-name: node # Run full test matrix if job started by cron or it was explictly specified by a person who triggered the workflow run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} -# containers: false test-node: runs-on: ${{ matrix.host.RUNNER }} diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 564aed0798..92ee34eb42 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -64,8 +64,6 @@ run-name: jobs: get-matrices: runs-on: ubuntu-latest - # Avoid running on schedule for forks - if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') || github.event_name == 'push' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' outputs: engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} @@ -105,133 +103,133 @@ jobs: python -m pip install --upgrade pip pip install mypy-protobuf -# - name: Build Python wrapper -# uses: ./.github/workflows/build-python-wrapper -# with: -# os: ${{ matrix.host.OS }} -# target: ${{ matrix.host.TARGET }} -# github-token: ${{ secrets.GITHUB_TOKEN }} -# engine-version: ${{ matrix.engine.version }} -# -# - name: Type check with mypy -# working-directory: ./python -# run: | -# # The type check should run inside the virtual env to get -# # all installed dependencies and build files -# source .env/bin/activate -# pip install mypy types-protobuf -# # Install the benchmark requirements -# pip install -r ../benchmarks/python/requirements.txt -# python -m mypy .. -# -# - name: Test with pytest -# working-directory: ./python -# run: | -# source .env/bin/activate -# cd python/tests/ -# pytest --asyncio-mode=auto --html=pytest_report.html --self-contained-html -# -# - uses: ./.github/workflows/test-benchmark -# with: -# language-flag: -python -# -# - name: Upload test reports -# if: always() -# continue-on-error: true -# uses: actions/upload-artifact@v4 -# with: -# name: test-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }} -# path: | -# python/python/tests/pytest_report.html -# utils/clusters/** -# benchmarks/results/** + - name: Build Python wrapper + uses: ./.github/workflows/build-python-wrapper + with: + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: ${{ matrix.engine.version }} + + - name: Type check with mypy + working-directory: ./python + run: | + # The type check should run inside the virtual env to get + # all installed dependencies and build files + source .env/bin/activate + pip install mypy types-protobuf + # Install the benchmark requirements + pip install -r ../benchmarks/python/requirements.txt + python -m mypy .. + + - name: Test with pytest + working-directory: ./python + run: | + source .env/bin/activate + cd python/tests/ + pytest --asyncio-mode=auto --html=pytest_report.html --self-contained-html + + - uses: ./.github/workflows/test-benchmark + with: + language-flag: -python + + - name: Upload test reports + if: always() + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: test-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }} + path: | + python/python/tests/pytest_report.html + utils/clusters/** + benchmarks/results/** # run pubsub tests in another job - they take too much time -# test-pubsub-python: -# runs-on: ${{ matrix.host.RUNNER }} -# needs: get-matrices -# timeout-minutes: 35 -# strategy: -# fail-fast: false -# matrix: -# python: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} -# engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} -# host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} -# steps: -# - uses: actions/checkout@v4 -# with: -# submodules: recursive -# -# - name: Set up Python -# uses: actions/setup-python@v5 -# with: -# python-version: ${{ matrix.python }} -# -# - name: Build Python wrapper -# uses: ./.github/workflows/build-python-wrapper -# with: -# os: ${{ matrix.host.OS }} -# target: ${{ matrix.host.TARGET }} -# github-token: ${{ secrets.GITHUB_TOKEN }} -# engine-version: ${{ matrix.engine.version }} -# -# - name: Test pubsub with pytest -# working-directory: ./python -# run: | -# source .env/bin/activate -# cd python/tests/ -# pytest --asyncio-mode=auto -k test_pubsub --html=pytest_report.html --self-contained-html -# -# - name: Upload test reports -# if: always() -# continue-on-error: true -# uses: actions/upload-artifact@v4 -# with: -# name: pubsub-test-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }} -# path: | -# python/python/tests/pytest_report.html - -# lint: -# runs-on: ubuntu-latest -# timeout-minutes: 15 -# steps: -# - uses: actions/checkout@v4 -# with: -# submodules: recursive -# -# - name: lint rust -# uses: ./.github/workflows/lint-rust -# with: -# cargo-toml-folder: python -# github-token: ${{ secrets.GITHUB_TOKEN }} -# -# - name: Install dependencies -# if: always() -# working-directory: ./python -# run: | -# python -m pip install --upgrade pip -# pip install flake8 isort black -# -# - name: Lint python with isort -# if: always() -# working-directory: ./python -# run: | -# isort . --profile black --check --diff -# -# - name: Lint python with flake8 -# if: always() -# working-directory: ./python -# run: | -# # stop the build if there are Python syntax errors or undefined names -# flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --extend-ignore=E230 --exclude=python/glide/protobuf,.env/* -# # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide -# flake8 . --count --exit-zero --max-complexity=12 --max-line-length=127 --statistics --extend-ignore=E230 --exclude=python/glide/protobuf,.env/* -# -# - name: Lint python with black -# if: always() -# working-directory: ./python -# run: | -# black --check --diff . + test-pubsub-python: + runs-on: ${{ matrix.host.RUNNER }} + needs: get-matrices + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + python: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Build Python wrapper + uses: ./.github/workflows/build-python-wrapper + with: + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: ${{ matrix.engine.version }} + + - name: Test pubsub with pytest + working-directory: ./python + run: | + source .env/bin/activate + cd python/tests/ + pytest --asyncio-mode=auto -k test_pubsub --html=pytest_report.html --self-contained-html + + - name: Upload test reports + if: always() + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: pubsub-test-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }} + path: | + python/python/tests/pytest_report.html + + lint: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: lint rust + uses: ./.github/workflows/lint-rust + with: + cargo-toml-folder: python + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install dependencies + if: always() + working-directory: ./python + run: | + python -m pip install --upgrade pip + pip install flake8 isort black + + - name: Lint python with isort + if: always() + working-directory: ./python + run: | + isort . --profile black --check --diff + + - name: Lint python with flake8 + if: always() + working-directory: ./python + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --extend-ignore=E230 --exclude=python/glide/protobuf,.env/* + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=12 --max-line-length=127 --statistics --extend-ignore=E230 --exclude=python/glide/protobuf,.env/* + + - name: Lint python with black + if: always() + working-directory: ./python + run: | + black --check --diff . get-containers: runs-on: ubuntu-latest