From 79899213ebb74ceb152435b22ba38dbbe8603a05 Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Mon, 23 Oct 2023 13:31:08 +0200 Subject: [PATCH 1/7] [enhance] Reduce tests-mpi to the oldest and latest Python versions * Add GITHUB_OUTPUT to share the env var * If job_mode is `run`, all the checks will be skipped --- .github/workflows/tests-mpi.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests-mpi.yml b/.github/workflows/tests-mpi.yml index 3ea1f1f46f5..db6a66cfe83 100644 --- a/.github/workflows/tests-mpi.yml +++ b/.github/workflows/tests-mpi.yml @@ -21,16 +21,31 @@ jobs: matrix: python-version: ['3.8', '3.9', '3.10', '3.11'] - steps: + steps: + - name: Check for skip + id: check-for-skip + # The oldest and latest versions must be adapted. + run: | + if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "${{ matrix.python-version }}" = "3.8" ] || [ "${{ matrix.python-version }}" = "3.11" ] ; then + echo "job_mode=run" >> $GITHUB_OUTPUT + echo "Run the checks" + else + echo "job_mode=skip" >> $GITHUB_OUTPUT + echo "Skip the checks" + fi + - name: Checkout + if: steps.check-for-skip.outputs.job_mode == 'run' uses: actions/checkout@v3 - name: Setup Python${{ matrix.python-version }} + if: steps.check-for-skip.outputs.job_mode == 'run' uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Setup cache + if: steps.check-for-skip.outputs.job_mode == 'run' uses: actions/cache@v3 env: cache-name: test-mpi @@ -41,11 +56,13 @@ jobs: ${{ runner.os }}-${{ matrix.python-version }}-${{ env.cache-name }}-${{ hashFiles('**/pyproject.toml') }} - name: Setup environment + if: steps.check-for-skip.outputs.job_mode == 'run' run: | sudo apt-get update sudo apt-get -y install openmpi-bin libopenmpi-dev - name: Install + if: steps.check-for-skip.outputs.job_mode == 'run' run: | python -m pip install --upgrade pip @@ -63,14 +80,17 @@ jobs: pip install --progress-bar off .[integration] --extra-index-url https://download.pytorch.org/whl/cpu - name: Output installed packages + if: steps.check-for-skip.outputs.job_mode == 'run' run: | pip freeze --all - name: Output dependency tree + if: steps.check-for-skip.outputs.job_mode == 'run' run: | pip install pipdeptree pipdeptree - name: Tests + if: steps.check-for-skip.outputs.job_mode == 'run' run: | export OMPI_MCA_rmaps_base_oversubscribe=yes From 423dafd215205b7eb8b4b792b71b86aee0e45bd0 Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Mon, 30 Oct 2023 06:13:24 +0100 Subject: [PATCH 2/7] Try the dynamic matrix suggested by not522 --- .github/workflows/tests-mpi.yml | 40 +++++++++++++++------------------ 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/.github/workflows/tests-mpi.yml b/.github/workflows/tests-mpi.yml index db6a66cfe83..ce4d713e9e0 100644 --- a/.github/workflows/tests-mpi.yml +++ b/.github/workflows/tests-mpi.yml @@ -13,39 +13,40 @@ concurrency: cancel-in-progress: true jobs: + define-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - id: set-matrix + run: | + if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ] ; then + # Schedule --> run all the versions + echo "::set-output name=matrix::{\"python-version\":[\"3.8\", \"3.9\", \"3.10\", \"3.11\"]}" + else + # PR push --> run only the oldest and the latest versions + echo "::set-output name=matrix::{\"python-version\":[\"3.8\", \"3.11\"]}" + fi + tests-mpi: if: (github.event_name == 'schedule' && github.repository == 'optuna/optuna') || (github.event_name != 'schedule') + needs: define-matrix runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] - - steps: - - name: Check for skip - id: check-for-skip - # The oldest and latest versions must be adapted. - run: | - if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "${{ matrix.python-version }}" = "3.8" ] || [ "${{ matrix.python-version }}" = "3.11" ] ; then - echo "job_mode=run" >> $GITHUB_OUTPUT - echo "Run the checks" - else - echo "job_mode=skip" >> $GITHUB_OUTPUT - echo "Skip the checks" - fi + python-version: ${{fromJson(needs.define-matrix.outputs.matrix)}} + steps: - name: Checkout - if: steps.check-for-skip.outputs.job_mode == 'run' uses: actions/checkout@v3 - name: Setup Python${{ matrix.python-version }} - if: steps.check-for-skip.outputs.job_mode == 'run' uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Setup cache - if: steps.check-for-skip.outputs.job_mode == 'run' uses: actions/cache@v3 env: cache-name: test-mpi @@ -56,13 +57,11 @@ jobs: ${{ runner.os }}-${{ matrix.python-version }}-${{ env.cache-name }}-${{ hashFiles('**/pyproject.toml') }} - name: Setup environment - if: steps.check-for-skip.outputs.job_mode == 'run' run: | sudo apt-get update sudo apt-get -y install openmpi-bin libopenmpi-dev - name: Install - if: steps.check-for-skip.outputs.job_mode == 'run' run: | python -m pip install --upgrade pip @@ -80,17 +79,14 @@ jobs: pip install --progress-bar off .[integration] --extra-index-url https://download.pytorch.org/whl/cpu - name: Output installed packages - if: steps.check-for-skip.outputs.job_mode == 'run' run: | pip freeze --all - name: Output dependency tree - if: steps.check-for-skip.outputs.job_mode == 'run' run: | pip install pipdeptree pipdeptree - name: Tests - if: steps.check-for-skip.outputs.job_mode == 'run' run: | export OMPI_MCA_rmaps_base_oversubscribe=yes From fda2f9c0588342a6dac4afd43cf055f2b026cac1 Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Mon, 30 Oct 2023 06:26:53 +0100 Subject: [PATCH 3/7] [dev] Disable the other tests temporally for quick check --- .github/workflows/tests-integration.yml | 2 +- .github/workflows/tests-storage.yml | 2 +- .github/workflows/tests.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-integration.yml b/.github/workflows/tests-integration.yml index 015c01a8f9d..0129100c4c7 100644 --- a/.github/workflows/tests-integration.yml +++ b/.github/workflows/tests-integration.yml @@ -15,7 +15,7 @@ concurrency: jobs: tests-integration: - if: (github.event_name == 'schedule' && github.repository == 'optuna/optuna') || (github.event_name != 'schedule') + if: false && (github.event_name == 'schedule' && github.repository == 'optuna/optuna') || (github.event_name != 'schedule') runs-on: ubuntu-latest strategy: diff --git a/.github/workflows/tests-storage.yml b/.github/workflows/tests-storage.yml index bf45161d2ce..db9d3197281 100644 --- a/.github/workflows/tests-storage.yml +++ b/.github/workflows/tests-storage.yml @@ -17,7 +17,7 @@ jobs: # RDB. Since current name "tests-rdbstorage" is required in the Branch protection rules, you # need to modify the Branch protection rules as well. tests-rdbstorage: - if: (github.event_name == 'schedule' && github.repository == 'optuna/optuna') || (github.event_name != 'schedule') + if: false && (github.event_name == 'schedule' && github.repository == 'optuna/optuna') || (github.event_name != 'schedule') runs-on: ubuntu-latest strategy: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4c8d1721395..8e0f1dd4339 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ concurrency: jobs: tests: - if: (github.event_name == 'schedule' && github.repository == 'optuna/optuna') || (github.event_name != 'schedule') + if: false && (github.event_name == 'schedule' && github.repository == 'optuna/optuna') || (github.event_name != 'schedule') runs-on: ubuntu-latest strategy: From b82ae71b3201e6498d517ad4357886cb89d6250a Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Mon, 30 Oct 2023 06:41:41 +0100 Subject: [PATCH 4/7] Fix a mistake in matrix input --- .github/workflows/tests-mpi.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-mpi.yml b/.github/workflows/tests-mpi.yml index ce4d713e9e0..eab0371417e 100644 --- a/.github/workflows/tests-mpi.yml +++ b/.github/workflows/tests-mpi.yml @@ -18,7 +18,8 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - id: set-matrix + - name: Define workflow matrix + id: set-matrix run: | if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ] ; then # Schedule --> run all the versions @@ -34,8 +35,7 @@ jobs: runs-on: ubuntu-latest strategy: - matrix: - python-version: ${{fromJson(needs.define-matrix.outputs.matrix)}} + matrix: ${{fromJson(needs.define-matrix.outputs.matrix)}} steps: - name: Checkout From 2942ca843e3bb14faa115fdd15f26ee6379c3a80 Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Mon, 30 Oct 2023 06:44:47 +0100 Subject: [PATCH 5/7] Revert the temporally disabled tests --- .github/workflows/tests-integration.yml | 2 +- .github/workflows/tests-mpi.yml | 4 ++-- .github/workflows/tests-storage.yml | 2 +- .github/workflows/tests.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests-integration.yml b/.github/workflows/tests-integration.yml index 0129100c4c7..015c01a8f9d 100644 --- a/.github/workflows/tests-integration.yml +++ b/.github/workflows/tests-integration.yml @@ -15,7 +15,7 @@ concurrency: jobs: tests-integration: - if: false && (github.event_name == 'schedule' && github.repository == 'optuna/optuna') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'optuna/optuna') || (github.event_name != 'schedule') runs-on: ubuntu-latest strategy: diff --git a/.github/workflows/tests-mpi.yml b/.github/workflows/tests-mpi.yml index eab0371417e..24e178d7b02 100644 --- a/.github/workflows/tests-mpi.yml +++ b/.github/workflows/tests-mpi.yml @@ -22,10 +22,10 @@ jobs: id: set-matrix run: | if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ] ; then - # Schedule --> run all the versions + echo "Schedule tests (run all the versions)" echo "::set-output name=matrix::{\"python-version\":[\"3.8\", \"3.9\", \"3.10\", \"3.11\"]}" else - # PR push --> run only the oldest and the latest versions + echo "PR push tests (run only the oldest and the latest versions)" echo "::set-output name=matrix::{\"python-version\":[\"3.8\", \"3.11\"]}" fi diff --git a/.github/workflows/tests-storage.yml b/.github/workflows/tests-storage.yml index db9d3197281..bf45161d2ce 100644 --- a/.github/workflows/tests-storage.yml +++ b/.github/workflows/tests-storage.yml @@ -17,7 +17,7 @@ jobs: # RDB. Since current name "tests-rdbstorage" is required in the Branch protection rules, you # need to modify the Branch protection rules as well. tests-rdbstorage: - if: false && (github.event_name == 'schedule' && github.repository == 'optuna/optuna') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'optuna/optuna') || (github.event_name != 'schedule') runs-on: ubuntu-latest strategy: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8e0f1dd4339..4c8d1721395 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ concurrency: jobs: tests: - if: false && (github.event_name == 'schedule' && github.repository == 'optuna/optuna') || (github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository == 'optuna/optuna') || (github.event_name != 'schedule') runs-on: ubuntu-latest strategy: From 80df2e688bde0df7026e91de2bcd586937edbb50 Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Mon, 30 Oct 2023 09:55:53 +0100 Subject: [PATCH 6/7] Add workflow dispatch to tests-mpi --- .github/workflows/tests-mpi.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests-mpi.yml b/.github/workflows/tests-mpi.yml index 24e178d7b02..489eab7317d 100644 --- a/.github/workflows/tests-mpi.yml +++ b/.github/workflows/tests-mpi.yml @@ -1,6 +1,7 @@ name: Tests (MPI) on: + workflow_dispatch: push: branches: - master From 6a2675846c53361b2a3ebde6c51a19b090c41940 Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Wed, 1 Nov 2023 07:19:12 +0100 Subject: [PATCH 7/7] Fix the deprecated warning in set-output --- .github/workflows/tests-mpi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests-mpi.yml b/.github/workflows/tests-mpi.yml index 489eab7317d..fd6050bbc70 100644 --- a/.github/workflows/tests-mpi.yml +++ b/.github/workflows/tests-mpi.yml @@ -24,10 +24,10 @@ jobs: run: | if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ] ; then echo "Schedule tests (run all the versions)" - echo "::set-output name=matrix::{\"python-version\":[\"3.8\", \"3.9\", \"3.10\", \"3.11\"]}" + echo "matrix={\"python-version\":[\"3.8\", \"3.9\", \"3.10\", \"3.11\"]}" >> $GITHUB_OUTPUT else echo "PR push tests (run only the oldest and the latest versions)" - echo "::set-output name=matrix::{\"python-version\":[\"3.8\", \"3.11\"]}" + echo "matrix={\"python-version\":[\"3.8\", \"3.11\"]}" >> $GITHUB_OUTPUT fi tests-mpi: