raydp.yml

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name: RayDP CI

on:
  push:
    branches: [ main, master ]
  pull_request:
    branches: [ main, master ]
  workflow_dispatch:

permissions:  # added using https://github.com/step-security/secure-repo
  contents: read

jobs:
  build-and-test:

    strategy:
      matrix:
        os: [ ubuntu-latest ]
        python-version: [3.8, 3.9]
        spark-version: [3.1.3, 3.2.4, 3.3.2, 3.4.0]

    runs-on: ${{ matrix.os }}

    steps:
      - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2.3.4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Set up JDK 1.8
        uses: actions/setup-java@b6e674f4b717d7b0ae3baee0fbe79f498905dfde # v1.4.4
        with:
          java-version: 1.8
      - name: Install extra dependencies for macOS
        if: matrix.os == 'macos-latest'
        run: |
          brew install pkg-config
          brew install libuv libomp mpich
      - name: Install extra dependencies for Ubuntu
        if: matrix.os == 'ubuntu-latest'
        run: |
          sudo apt-get install -y mpich
      - name: Cache pip - Ubuntu
        if: matrix.os == 'ubuntu-latest'
        uses: actions/cache@8492260343ad570701412c2f464a5877dc76bace # v2
        with:
          path: ~/.cache/pip
          key: ${{ matrix.os }}-${{ matrix.python-version }}-pip
      - name: Cache pip - MacOS
        if: matrix.os == 'macos-latest'
        uses: actions/cache@8492260343ad570701412c2f464a5877dc76bace # v2
        with:
          path: ~/Library/Caches/pip
          key: ${{ matrix.os }}-${{ matrix.python-version }}-pip
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install wheel
          pip install "numpy<1.24"
          pip install "pydantic<2.0"
          SUBVERSION=$(python -c 'import sys; print(sys.version_info[1])')
          if [ "$(uname -s)" == "Linux" ]
          then
            pip install torch --index-url https://download.pytorch.org/whl/cpu
          else
            pip install torch
          fi
          pip install pyarrow==6.0.1 ray[train] pytest koalas tensorflow==2.13.1 tabulate grpcio-tools wget
          pip install "xgboost_ray[default]<=0.1.13"
          pip install torchmetrics
      - name: Cache Maven
        uses: actions/cache@8492260343ad570701412c2f464a5877dc76bace # v2
        with:
          path: ~/.m2
          key: ${{ matrix.os }}-m2-${{ hashFiles('core/pom.xml') }}
      - name: Build and install
        env:
          GITHUB_CI: 1
        run: |
          pip install pyspark==${{ matrix.spark-version }}
          ./build.sh
          pip install dist/raydp-*.whl
      - name: Lint
        run: |
          pip install pylint==2.8.3
          pylint --rcfile=python/pylintrc python/raydp
          pylint --rcfile=python/pylintrc examples/*.py
      - name: Test with pytest
        run: |
          ray start --head --num-cpus 6
          pytest python/raydp/tests/ -v
          ray stop --force
      - name: Test Examples
        run: |
          ray start --head
          python examples/raydp-submit.py
          ray stop
          python examples/pytorch_nyctaxi.py
          python examples/tensorflow_nyctaxi.py
          python examples/xgboost_ray_nyctaxi.py
          # python examples/raytrain_nyctaxi.py
          python examples/data_process.py