From 31c9b3269fcdfee1d742efac43e621c9e5a1f5f4 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 21 Feb 2024 14:05:47 -0600 Subject: [PATCH 1/7] Add workflow to publish python package --- .github/workflows/python-release.yml | 53 ++++++++++++++++++++++++++++ .github/workflows/python-tests.yml | 1 - bindings/python/.gitignore | 3 ++ bindings/python/setup.py | 9 +++-- 4 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/python-release.yml diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml new file mode 100644 index 000000000..f8d0dbd10 --- /dev/null +++ b/.github/workflows/python-release.yml @@ -0,0 +1,53 @@ +name: Upload Python Package + +on: + push: + branches: + - main + release: + branches: + - main + +jobs: + deploy: + runs-on: ${{ matrix.host }} + strategy: + matrix: + include: + # Linux + - host: ubuntu-latest + target: x86_64-unknown-linux-gnu + - host: ubuntu-latest + target: aarch64-unknown-linux-gnu + # MacOS + - host: macos-latest + target: x86_64-apple-darwin + - host: macos-latest + target: aarch64-apple-darwin + # Windows + - host: windows-latest + target: x86_64-pc-windows-msvc + + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build CKZG + run: | + cd src + make + - name: Build and publish + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + cd bindings/python + python setup.py bdist_wheel + twine upload dist/* \ No newline at end of file diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index a44474142..583d6577a 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -20,7 +20,6 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install py_ecc pip install PyYAML - name: Build BLST run: | diff --git a/bindings/python/.gitignore b/bindings/python/.gitignore index 567609b12..60446066a 100644 --- a/bindings/python/.gitignore +++ b/bindings/python/.gitignore @@ -1 +1,4 @@ build/ +dist/ +ckzg.egg-info/ +src/ diff --git a/bindings/python/setup.py b/bindings/python/setup.py index b072833e9..b703b192d 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -1,10 +1,12 @@ -from distutils.core import setup, Extension +from setuptools import setup, Extension + def main(): setup( name="ckzg", - version="1.0.0", - description="Python interface for C-KZG-4844", + version="1.4.2", + author="Ethereum Foundation", + description="Python bindings for C-KZG-4844", ext_modules=[ Extension( "ckzg", @@ -13,5 +15,6 @@ def main(): library_dirs=["../../lib"], libraries=["blst"])]) + if __name__ == "__main__": main() From ea41aaf9641aad9548c7d8df444d01652a31fecd Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 21 Feb 2024 21:14:42 -0600 Subject: [PATCH 2/7] Fix version number --- bindings/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/setup.py b/bindings/python/setup.py index b703b192d..907da41c4 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -4,7 +4,7 @@ def main(): setup( name="ckzg", - version="1.4.2", + version="0.4.2", author="Ethereum Foundation", description="Python bindings for C-KZG-4844", ext_modules=[ From 012e1b4e45a21ff5fda387080b617d55a8c8359c Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Mon, 26 Feb 2024 09:48:04 -0600 Subject: [PATCH 3/7] Use cibuildwheel --- .github/workflows/python-release.yml | 62 ++++++++++++---------------- 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index f8d0dbd10..94893c623 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -1,4 +1,4 @@ -name: Upload Python Package +name: Python Package on: push: @@ -9,45 +9,37 @@ on: - main jobs: - deploy: - runs-on: ${{ matrix.host }} + name: Build wheels for ${{ matrix.os }} + runs-on: ${{ matrix.os }} strategy: matrix: - include: - # Linux - - host: ubuntu-latest - target: x86_64-unknown-linux-gnu - - host: ubuntu-latest - target: aarch64-unknown-linux-gnu - # MacOS - - host: macos-latest - target: x86_64-apple-darwin - - host: macos-latest - target: aarch64-apple-darwin - # Windows - - host: windows-latest - target: x86_64-pc-windows-msvc + os: + - ubuntu-latest + - windows-latest + - macos-13 + - macos-14 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: recursive + + - name: Build CKZG + working-directory: src + run: make + - uses: actions/setup-python@v4 + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==2.16.5 + + - name: Build wheels + working-directory: bindings/python + run: python -m cibuildwheel --output-dir wheelhouse + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + working-directory: bindings/python with: - python-version: '3.10' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build CKZG - run: | - cd src - make - - name: Build and publish - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - cd bindings/python - python setup.py bdist_wheel - twine upload dist/* \ No newline at end of file + password: ${{ secrets.PYPI_PASSWORD }} + packages-dir: wheelhouse From 4f44fc67576bd0ad5d503bb94bbb4c313de07ed5 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Mon, 26 Feb 2024 10:18:37 -0600 Subject: [PATCH 4/7] Build with linux on multiple archs --- .github/workflows/python-release.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 94893c623..1cc49ae9b 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -9,6 +9,7 @@ on: - main jobs: + build-wheels: name: Build wheels for ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: @@ -24,11 +25,20 @@ jobs: with: submodules: recursive + - name: Setup QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v3 + with: + platforms: all + - name: Build CKZG working-directory: src run: make - - uses: actions/setup-python@v4 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' - name: Install cibuildwheel run: python -m pip install cibuildwheel==2.16.5 @@ -36,10 +46,11 @@ jobs: - name: Build wheels working-directory: bindings/python run: python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_ARCHS_LINUX: auto aarch64 - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - working-directory: bindings/python with: password: ${{ secrets.PYPI_PASSWORD }} - packages-dir: wheelhouse + packages-dir: bindings/python/wheelhouse From 1f2a749f39f486b0799df62f2ca4c5dcddd84f2a Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 27 Feb 2024 16:48:54 -0600 Subject: [PATCH 5/7] Fix some cibw things --- .github/workflows/python-release.yml | 25 +++++++++++++------------ src/Makefile | 2 +- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 1cc49ae9b..a26bb2071 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -1,9 +1,6 @@ name: Python Package on: - push: - branches: - - main release: branches: - main @@ -31,23 +28,27 @@ jobs: with: platforms: all - - name: Build CKZG - working-directory: src - run: make - - name: Setup Python uses: actions/setup-python@v4 - with: - python-version: '3.10' - name: Install cibuildwheel run: python -m pip install cibuildwheel==2.16.5 - name: Build wheels - working-directory: bindings/python - run: python -m cibuildwheel --output-dir wheelhouse + run: python -m cibuildwheel --output-dir wheelhouse bindings/python env: - CIBW_ARCHS_LINUX: auto aarch64 + CIBW_ARCHS_LINUX: x86_64 i686 aarch64 + # musllinux uses apk & manylinux uses yum + CIBW_BEFORE_BUILD_LINUX: | + if command -v apk > /dev/null; then + apk add --update clang && make -C src c_kzg_4844.o + elif command -v yum > /dev/null; then + yum install -y clang && make -C src c_kzg_4844.o + fi + CIBW_BEFORE_BUILD_WINDOWS: | + make -C src c_kzg_4844.o + CIBW_BEFORE_BUILD_MACOS: | + make -C src c_kzg_4844.o - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/src/Makefile b/src/Makefile index b6c75ad6c..7bc63298d 100644 --- a/src/Makefile +++ b/src/Makefile @@ -20,7 +20,7 @@ ifeq ($(PLATFORM),Darwin) endif # The base compiler flags. More can be added on command line. -CFLAGS += -I../inc -O2 -Wall -Wextra +CFLAGS += -I../inc -O2 -Wall -Wextra -Wno-missing-braces # Cross-platform compilation settings. ifeq ($(PLATFORM),Windows) From 5c92e043c95b3a0767fc529fd5d35df92c5c7f34 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Mon, 4 Mar 2024 10:01:25 -0600 Subject: [PATCH 6/7] Fix builds for Windows & macOS --- .github/workflows/python-release.yml | 32 ++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index a26bb2071..6180899fa 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -14,22 +14,31 @@ jobs: os: - ubuntu-latest - windows-latest - - macos-13 - - macos-14 + - macos-13 # x86_64 + - macos-14 # aarch64 steps: - uses: actions/checkout@v4 with: submodules: recursive + # On Linux, use QEMU to build multiple platforms. - name: Setup QEMU if: runner.os == 'Linux' uses: docker/setup-qemu-action@v3 with: platforms: all + # Need this for macos-14, which doesn't come with python for some reason. - name: Setup Python uses: actions/setup-python@v4 + with: + python-version: '3.10' + + # Need this to get cl.exe on the path. + - name: Set up Visual Studio shell + if: runner.os == 'Windows' + uses: egor-tensin/vs-shell@v2 - name: Install cibuildwheel run: python -m pip install cibuildwheel==2.16.5 @@ -37,18 +46,27 @@ jobs: - name: Build wheels run: python -m cibuildwheel --output-dir wheelhouse bindings/python env: + + # We have QEMU setup and can build everything. CIBW_ARCHS_LINUX: x86_64 i686 aarch64 - # musllinux uses apk & manylinux uses yum + # For some reason these don't use the same distro. + # musllinux uses apk & manylinux uses yum. CIBW_BEFORE_BUILD_LINUX: | if command -v apk > /dev/null; then - apk add --update clang && make -C src c_kzg_4844.o + apk add --update clang && make -C src blst elif command -v yum > /dev/null; then - yum install -y clang && make -C src c_kzg_4844.o + yum install -y clang && make -C src blst fi + + # Building x86 (32-bit) package is difficult. + # We're missing the 32-bit Python library. + CIBW_ARCHS_WINDOWS: AMD64 + # We need blst.lib (via MSVC) which our Makefile doesn't support. CIBW_BEFORE_BUILD_WINDOWS: | - make -C src c_kzg_4844.o + cd blst && build.bat && cp blst.lib ../lib && cp bindings/*.h ../inc + CIBW_BEFORE_BUILD_MACOS: | - make -C src c_kzg_4844.o + make -C src blst - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 From 9ae1bff42701264b48b541ec3404d6d83726a002 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Mon, 4 Mar 2024 10:19:23 -0600 Subject: [PATCH 7/7] Remove -Wno-missing-braces flag --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 7bc63298d..b6c75ad6c 100644 --- a/src/Makefile +++ b/src/Makefile @@ -20,7 +20,7 @@ ifeq ($(PLATFORM),Darwin) endif # The base compiler flags. More can be added on command line. -CFLAGS += -I../inc -O2 -Wall -Wextra -Wno-missing-braces +CFLAGS += -I../inc -O2 -Wall -Wextra # Cross-platform compilation settings. ifeq ($(PLATFORM),Windows)