From 0611a1a1ce49de1b330a7a95f99937e1293e9c71 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 31 Oct 2023 19:47:58 -0400 Subject: [PATCH] Onboard jenkins prod docker images to github actions (#1025) (#1028) * Onboard jenkins prod docker images to github actions * Add more * Add more --------- Signed-off-by: Peter Zhu --- .../workflows/multi-node-test-workflow.yml | 17 ++- .github/workflows/release-workflow.yml | 121 ------------------ .github/workflows/security-test-workflow.yml | 16 ++- .github/workflows/test-and-build-workflow.yml | 81 ++++++++++-- 4 files changed, 99 insertions(+), 136 deletions(-) delete mode 100644 .github/workflows/release-workflow.yml diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml index 78984c784..73881a6f1 100644 --- a/.github/workflows/multi-node-test-workflow.yml +++ b/.github/workflows/multi-node-test-workflow.yml @@ -9,7 +9,13 @@ on: - "**" jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + multi-node-test: + needs: Get-CI-Image-Tag env: TEST_FILTER: ${{ matrix.test_filter }} strategy: @@ -23,6 +29,13 @@ jobs: test_filter: -PexcludeTests="org.opensearch.indexmanagement.indexstatemanagement*" # This job runs on Linux runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + steps: # This step uses the setup-java Github action: https://github.com/actions/setup-java - name: Set Up JDK @@ -33,7 +46,9 @@ jobs: - name: Checkout Branch uses: actions/checkout@v2 - name: Run integration tests with multi node config - run: ./gradlew integTest -PnumNodes=3 ${{ env.TEST_FILTER }} + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew integTest -PnumNodes=3 ${{ env.TEST_FILTER }}" - name: Upload failed logs uses: actions/upload-artifact@v2 if: failure() diff --git a/.github/workflows/release-workflow.yml b/.github/workflows/release-workflow.yml deleted file mode 100644 index b6343f9f1..000000000 --- a/.github/workflows/release-workflow.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: Release workflow -# This workflow is triggered on creating tags to main or an opendistro release branch -on: - push: - tags: - - 'v*' - -jobs: - build: - strategy: - matrix: - java: [11] - # Job name - name: Build Index Management with JDK ${{ matrix.java }} - # This job runs on Linux - runs-on: ubuntu-latest - steps: - # This step uses the checkout Github action: https://github.com/actions/checkout - - name: Checkout Branch - uses: actions/checkout@v2 - # This step uses the setup-java Github action: https://github.com/actions/setup-java - - name: Set Up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - # Building zip, deb and rpm files - - name: Build with Gradle - run: ./gradlew build buildDeb buildRpm --no-daemon -Dbuild.snapshot=false - - - name: Create Artifact Path - run: | - mkdir -p index-management-artifacts - cp ./build/distributions/*.zip index-management-artifacts - cp ./build/distributions/*.zip index-management-artifacts_zip - cp ./build/distributions/*.deb index-management-artifacts - cp ./build/distributions/*.deb index-management-artifacts_deb - cp ./build/distributions/*.rpm index-management-artifacts - cp ./build/distributions/*.rpm index-management-artifacts_rpm - echo "TAG_VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV - - # AWS authentication - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact - - name: Upload Artifacts to S3 - run: | - zip=`ls index-management-artifacts/*.zip` - rpm=`ls index-management-artifacts/*.rpm` - deb=`ls index-management-artifacts/*.deb` - - # Inject the build number before the suffix - zip_outfile=`basename ${zip%.zip}-build-${GITHUB_RUN_NUMBER}.zip` - rpm_outfile=`basename ${rpm%.rpm}-build-${GITHUB_RUN_NUMBER}.rpm` - deb_outfile=`basename ${deb%.deb}-build-${GITHUB_RUN_NUMBER}.deb` - - s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-plugins/index-management/" - - echo "Copying ${zip} to ${s3_prefix}${zip_outfile}" - aws s3 cp --quiet $zip ${s3_prefix}${zip_outfile} - - echo "Copying ${rpm} to ${s3_prefix}${rpm_outfile}" - aws s3 cp --quiet $rpm ${s3_prefix}${rpm_outfile} - - echo "Copying ${deb} to ${s3_prefix}${deb_outfile}" - aws s3 cp --quiet $deb ${s3_prefix}${deb_outfile} - - - name: Create Github Draft Release - id: create_release - uses: actions/create-release@v1.0.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ env.TAG_VERSION }} - draft: true - prerelease: false - - # Upload the release with .zip as asset - - name: Upload Release Asset - uses: actions/upload-release-asset@v1.0.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_name: index-management-artifacts.zip - asset_path: index-management-artifacts_zip - asset_content_type: application/zip - - # Upload the release with .rpm as asset - - name: Upload Release Asset - uses: actions/upload-release-asset@v1.0.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_name: index-management-artifacts.rpm - asset_path: index-management-artifacts_rpm - asset_content_type: application/zip - - # Upload the release with .deb as asset - - name: Upload Release Asset - uses: actions/upload-release-asset@v1.0.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_name: index-management-artifacts.deb - asset_path: index-management-artifacts_deb - asset_content_type: application/zip - - - name: Upload Workflow Artifacts - uses: actions/upload-artifact@v1 - with: - name: index-management-plugin - path: index-management-artifacts diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml index 8f4dae924..855147d27 100644 --- a/.github/workflows/security-test-workflow.yml +++ b/.github/workflows/security-test-workflow.yml @@ -9,9 +9,22 @@ on: - "**" jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + security-test: + needs: Get-CI-Image-Tag # This job runs on Linux runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + steps: # This step uses the setup-java Github action: https://github.com/actions/setup-java - name: Set Up JDK @@ -23,7 +36,8 @@ jobs: uses: actions/checkout@v2 - name: Run integration tests run: | - ./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*SecurityBehaviorIT' + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*SecurityBehaviorIT'" - name: Upload failed logs uses: actions/upload-artifact@v2 if: failure() diff --git a/.github/workflows/test-and-build-workflow.yml b/.github/workflows/test-and-build-workflow.yml index 12d435d91..aba9c39d8 100644 --- a/.github/workflows/test-and-build-workflow.yml +++ b/.github/workflows/test-and-build-workflow.yml @@ -8,7 +8,73 @@ on: - "**" jobs: - test-and-build: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + test-and-build-linux: + needs: Get-CI-Image-Tag + env: + TEST_FILTER: ${{ matrix.test_filter }} + strategy: + # This setting says that all jobs should finish, even if one fails + fail-fast: false + # This starts three jobs, setting these environment variables uniquely for the different jobs + matrix: + java: [11, 17] + feature: [ism, non-ism] + include: + - feature: ism + test_filter: -PincludeTests="org.opensearch.indexmanagement.indexstatemanagement*" + - feature: non-ism + test_filter: -PexcludeTests="org.opensearch.indexmanagement.indexstatemanagement*" + runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + + steps: + # This step uses the setup-java Github action: https://github.com/actions/setup-java + - name: Set Up JDK ${{ matrix.java }} + uses: actions/setup-java@v2 + with: + distribution: temurin # Temurin is a distribution of adoptium + java-version: ${{ matrix.java }} + # build index management + - name: Checkout Branch + uses: actions/checkout@v2 + # This is a hack, but this step creates a link to the X: mounted drive, which makes the path + # short enough to work on Windows + - name: Build with Gradle + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew build ${{ env.TEST_FILTER }}" + - name: Upload failed logs + uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + name: logs + path: build/testclusters/integTest-*/logs/* + - name: Create Artifact Path + run: | + mkdir -p index-management-artifacts + cp ./build/distributions/*.zip index-management-artifacts + - name: Uploads coverage + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact + - name: Upload Artifacts + uses: actions/upload-artifact@v1 + with: + name: index-management-plugin-ubuntu-latest + path: index-management-artifacts + + test-and-build-windows-macos: env: BUILD_ARGS: ${{ matrix.os_build_args }} WORKING_DIR: ${{ matrix.working_directory }}. @@ -19,7 +85,7 @@ jobs: # This starts three jobs, setting these environment variables uniquely for the different jobs matrix: java: [11, 17] - os: [ubuntu-latest, windows-latest, macos-latest] + os: [windows-latest, macos-latest] feature: [ism, non-ism] include: - os: windows-latest @@ -52,21 +118,10 @@ jobs: run: ./gradlew build ${{ env.BUILD_ARGS }} ${{ env.TEST_FILTER }} env: _JAVA_OPTIONS: ${{ matrix.os_java_options }} - - name: Upload failed logs - uses: actions/upload-artifact@v2 - if: ${{ failure() && matrix.os == 'ubuntu-latest' }} - with: - name: logs - path: build/testclusters/integTest-*/logs/* - name: Create Artifact Path run: | mkdir -p index-management-artifacts cp ./build/distributions/*.zip index-management-artifacts - - name: Uploads coverage - if: ${{ matrix.os == 'ubuntu-latest' }} - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact - name: Upload Artifacts uses: actions/upload-artifact@v1