From a388dbebeec2197112c9361aea842027718956c3 Mon Sep 17 00:00:00 2001 From: Philipp Hossner Date: Wed, 14 Feb 2024 23:53:12 +0100 Subject: [PATCH] Initial fork of PyDuckling built against the latest version of Meta's Duckling (#1) This is a fork of the original pyduckling-native library. There are differences to the original: * Build against the latest version of [Duckling](https://github.com/facebook/duckling) * Supported Python versions range from 3.8 to 3.12 * x86_64 Linux only, but contributions welcome if you dare to take on the challenge Internally some other things have changed as well: * Updated to latest Maturin v1.4.0 * Adapted Rust code to work with latest Maturin/Rust compiler * Removed `GHC_VERSION` constant since Duckling will always be build with the same GHC version as the corresponding Duckling version * Introduced Docker-based build pipeline * Publish packages as manylinux/musllinux wheels * Minor changes to various Markdown files --- .dockerignore | 2 + .github/scripts/build_ffi.sh | 8 - .github/scripts/build_linux_wheels.sh | 60 --- .github/scripts/build_mac_wheels.sh | 26 -- .github/scripts/build_pyduckling.sh | 10 - .github/scripts/run_tests.sh | 11 - .github/stack/stack.yaml | 3 - .github/workflows/build.yml | 44 +++ .github/workflows/linux-wheels.yml | 43 -- .github/workflows/mac-tests.yml | 54 --- .github/workflows/mac-wheels.yml | 58 --- .github/workflows/rust.yml | 57 --- .gitmodules | 3 - CHANGELOG.md | 7 +- Cargo.toml | 18 +- HISTORY.md | 26 ++ README.md | 158 +++++--- RELEASE.md | 32 +- build-vars.sh | 19 + build.rs | 41 +- build.sh | 111 ++++++ containers/duckling-ffi/Dockerfile | 189 +++++++++ containers/duckling-ffi/build.sh | 17 + .../cabal-0001-force-ld.gold.patch | 10 + containers/duckling-ffi/ghc-8.8.patch | 78 ++++ containers/pyduckling/build.sh | 28 ++ containers/pyduckling/glibc/Dockerfile | 14 + containers/pyduckling/musl/Dockerfile | 17 + duckling-ffi | 1 - duckling-ffi/.gitignore | 26 ++ duckling-ffi/DucklingFFI/FFI.hs | 366 ++++++++++++++++++ duckling-ffi/LICENSE | 21 + duckling-ffi/README.md | 2 + duckling-ffi/Setup.hs | 2 + duckling-ffi/duckling-ffi.cabal | 47 +++ duckling-ffi/stack.yaml | 15 + duckling-ffi/stack.yaml.lock | 44 +++ duckling/__init__.py | 3 +- ...0001-Allow-binaries-larger-than-32MB.patch | 25 -- packaging/build_wheels.py | 337 ---------------- pyproject.toml | 18 +- src/lib.rs | 42 +- 42 files changed, 1240 insertions(+), 853 deletions(-) create mode 100644 .dockerignore delete mode 100644 .github/scripts/build_ffi.sh delete mode 100755 .github/scripts/build_linux_wheels.sh delete mode 100644 .github/scripts/build_mac_wheels.sh delete mode 100644 .github/scripts/build_pyduckling.sh delete mode 100644 .github/scripts/run_tests.sh delete mode 100644 .github/stack/stack.yaml create mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/linux-wheels.yml delete mode 100644 .github/workflows/mac-tests.yml delete mode 100644 .github/workflows/mac-wheels.yml delete mode 100644 .github/workflows/rust.yml delete mode 100644 .gitmodules create mode 100644 HISTORY.md create mode 100644 build-vars.sh create mode 100755 build.sh create mode 100644 containers/duckling-ffi/Dockerfile create mode 100755 containers/duckling-ffi/build.sh create mode 100644 containers/duckling-ffi/cabal-0001-force-ld.gold.patch create mode 100644 containers/duckling-ffi/ghc-8.8.patch create mode 100755 containers/pyduckling/build.sh create mode 100644 containers/pyduckling/glibc/Dockerfile create mode 100644 containers/pyduckling/musl/Dockerfile delete mode 160000 duckling-ffi create mode 100644 duckling-ffi/.gitignore create mode 100644 duckling-ffi/DucklingFFI/FFI.hs create mode 100644 duckling-ffi/LICENSE create mode 100644 duckling-ffi/README.md create mode 100644 duckling-ffi/Setup.hs create mode 100644 duckling-ffi/duckling-ffi.cabal create mode 100644 duckling-ffi/stack.yaml create mode 100644 duckling-ffi/stack.yaml.lock delete mode 100644 packaging/0001-Allow-binaries-larger-than-32MB.patch delete mode 100644 packaging/build_wheels.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..09c575f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +Dockerfile +.cache \ No newline at end of file diff --git a/.github/scripts/build_ffi.sh b/.github/scripts/build_ffi.sh deleted file mode 100644 index 0d177cf..0000000 --- a/.github/scripts/build_ffi.sh +++ /dev/null @@ -1,8 +0,0 @@ - -# Install Stack -# curl -sSL https://get.haskellstack.org/ | sh -# export PATH="$HOME/.local/bin:$PATH" - -cd duckling-ffi -stack build -cp libducklingffi.so ../ext_lib diff --git a/.github/scripts/build_linux_wheels.sh b/.github/scripts/build_linux_wheels.sh deleted file mode 100755 index 90e981c..0000000 --- a/.github/scripts/build_linux_wheels.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -set -ex -shopt -s nullglob - -PYBIN=/opt/python/cp$(echo $PYTHON_VERSION | sed -e 's/\.//g')*/bin -echo $PYBIN -# PYBIN=$(echo $PYBIN) - -# Install ZLib and sudo -yum install -y zlib-devel sudo -export HOME="/root" - -# Install Rustup -curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly -y -export PATH="$HOME/.cargo/bin:$PATH" - -# Install Stack -curl -sSL https://get.haskellstack.org/ | sh -export PATH="$HOME/.local/bin:$PATH" - -# Set stack resolver to 8.6.5 -mkdir -p $HOME/.stack/global-project -cp .github/stack/stack.yaml $HOME/.stack/global-project -cp packaging/0001-Allow-binaries-larger-than-32MB.patch $HOME - -pushd $HOME -stack config set resolver ghc-8.6.5 -popd - -# Compile patchelf and apply 64MB patch -pushd /root -git clone https://github.com/NixOS/patchelf -cd patchelf -git apply $HOME/0001-Allow-binaries-larger-than-32MB.patch - -bash bootstrap.sh -./configure -make -make install -popd - -# Compile libducklingffi -pushd duckling-ffi - -stack build -cp libducklingffi.so ../ext_lib -popd - -# Produce wheels and patch binaries for redistribution -PYBIN=$(echo $PYBIN) -GHC_LIB=$(stack exec -- ghc --print-libdir) -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GHC_LIB/rts:$(pwd)/ext_lib -# for PYBIN in /opt/python/cp{35,36,37,38,39}*/bin; do -"${PYBIN}/pip" install -U setuptools wheel setuptools-rust auditwheel -"${PYBIN}/python" packaging/build_wheels.py -# done - -if [[ $PYTHON_VERSION == "3.9" ]]; then - "${PYBIN}/python" setup.py sdist -fi diff --git a/.github/scripts/build_mac_wheels.sh b/.github/scripts/build_mac_wheels.sh deleted file mode 100644 index c10b896..0000000 --- a/.github/scripts/build_mac_wheels.sh +++ /dev/null @@ -1,26 +0,0 @@ -set -ex - -mkdir -p $HOME/.stack/global-project -cp .github/stack/stack.yaml $HOME/.stack/global-project - -pushd $HOME -stack config set resolver ghc-8.6.5 -popd - -# conda activate test -# which python -# Adjust PATH in macOS because conda is not at front of it -export PATH=/usr/local/miniconda/envs/test/bin:/usr/local/miniconda/condabin:$PATH -GHC_LIB=$(stack exec -- ghc --print-libdir) -export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$GHC_LIB/rts:$(pwd)/ext_lib - -for dir in $GHC_LIB/*/; do - export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$dir -done - -# python setup.py bdist_wheel -python packaging/build_wheels.py - -if [[ $PYTHON_VERSION == "3.8" ]]; then - python setup.py sdist -fi diff --git a/.github/scripts/build_pyduckling.sh b/.github/scripts/build_pyduckling.sh deleted file mode 100644 index 1b716fd..0000000 --- a/.github/scripts/build_pyduckling.sh +++ /dev/null @@ -1,10 +0,0 @@ - -# export PATH="$HOME/.local/bin:$PATH" -mkdir -p $HOME/.stack/global-project -cp .github/stack/stack.yaml $HOME/.stack/global-project - -pushd $HOME -stack config set resolver ghc-8.6.5 -popd - -maturin develop diff --git a/.github/scripts/run_tests.sh b/.github/scripts/run_tests.sh deleted file mode 100644 index 9019baf..0000000 --- a/.github/scripts/run_tests.sh +++ /dev/null @@ -1,11 +0,0 @@ - -# Set LD_LIBRARY_PATH in order to load dynamic libraries -GHC_PATH=$(stack exec -- ghc --print-libdir) - - -if [[ "$(uname)" != Darwin ]]; then - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GHC_PATH/rts:$(pwd)/ext_lib -else - export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$GHC_PATH/rts:$(pwd)/ext_lib -fi -pytest -x -v --cov=duckling duckling/tests diff --git a/.github/stack/stack.yaml b/.github/stack/stack.yaml deleted file mode 100644 index 57198ec..0000000 --- a/.github/stack/stack.yaml +++ /dev/null @@ -1,3 +0,0 @@ -packages: [] -resolver: - compiler: ghc-8.6.5 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..6012dd8 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,44 @@ +name: Build +on: + push: {} + pull_request: {} + release: + types: [ published ] +jobs: + build-and-test: + runs-on: ubuntu-latest + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name + steps: + - name: Check out repository code + uses: actions/checkout@v4 + - name: Cache build dependencies + uses: actions/cache@v4 + env: + cache-name: cache-build-deps + with: + key: ${{ runner.os }}-build-${{ env.cache-name }} + path: | + .cache + duckling-ffi/.stack-work + - name: Run build script + run: | + ./build.sh + - name: Archive artifacts + uses: actions/upload-artifact@v4 + with: + name: wheels + path: | + target/wheels/*.tar.gz + target/wheels/*.whl + publish: + needs: build-and-test + if: github.event_name == 'release' + runs-on: ubuntu-latest + steps: + - name: Download wheels + uses: actions/download-artifact@v4 + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: target/wheels/ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/linux-wheels.yml b/.github/workflows/linux-wheels.yml deleted file mode 100644 index 2182480..0000000 --- a/.github/workflows/linux-wheels.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Linux Wheels - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -env: - CARGO_TERM_COLOR: always - -jobs: - linux: - name: Linux (CentOS 7) Py${{ matrix.PYTHON_VERSION }} - runs-on: ubuntu-latest - container: - image: quay.io/pypa/manylinux2014_x86_64:latest - volumes: - - my_docker_volume:/volume_mount - env: - PYTHON_VERSION: ${{ matrix.PYTHON_VERSION }} - RUNNER_OS: "ubuntu" - HOME: "/root" - # options: -u $(id -u):$(id -g) - strategy: - fail-fast: false - matrix: - PYTHON_VERSION: ["3.5", "3.6", "3.7", "3.8", "3.9"] - steps: - - name: Checkout branch - uses: actions/checkout@v2 - with: - submodules: true - - name: Build wheel - shell: bash -l {0} - run: bash -l .github/scripts/build_linux_wheels.sh - - name: Upload wheel artifact - uses: actions/upload-artifact@v2 - with: - name: linux_dist - path: | - dist/*.whl - dist/*.tar.gz diff --git a/.github/workflows/mac-tests.yml b/.github/workflows/mac-tests.yml deleted file mode 100644 index 0d4253d..0000000 --- a/.github/workflows/mac-tests.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Mac Tests - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -env: - CARGO_TERM_COLOR: always - -jobs: - macos: - name: MacOS Py${{ matrix.PYTHON_VERSION }} - runs-on: macos-latest - env: - PYTHON_VERSION: ${{ matrix.PYTHON_VERSION }} - RUNNER_OS: "macos" - strategy: - fail-fast: false - matrix: - PYTHON_VERSION: ["3.5", "3.6", "3.7", "3.8"] - steps: - - name: Checkout branch - uses: actions/checkout@v2 - with: - submodules: true - - name: Install latest Rust nightly - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly - override: true - components: rustfmt, clippy - # - name: Install Haskell Stack - # uses: mstksg/setup-stack@v1 - - name: Install Conda - uses: goanpeca/setup-miniconda@v1 - with: - activate-environment: test - auto-update-conda: true - auto-activate-base: false - python-version: ${{ matrix.PYTHON_VERSION }} - - name: Compile duckling-ffi - shell: bash -l {0} - run: bash -l .github/scripts/build_ffi.sh - - name: Install build/test dependencies - shell: bash -l {0} - run: pip install maturin toml pytest pytest-cov coverage pendulum - - name: Build pyduckling - shell: bash -l {0} - run: bash -l .github/scripts/build_pyduckling.sh - - name: Run tests - shell: bash -l {0} - run: bash -l .github/scripts/run_tests.sh diff --git a/.github/workflows/mac-wheels.yml b/.github/workflows/mac-wheels.yml deleted file mode 100644 index 0acc431..0000000 --- a/.github/workflows/mac-wheels.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: Mac Wheels - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -env: - CARGO_TERM_COLOR: always - -jobs: - macos: - name: MacOS Py${{ matrix.PYTHON_VERSION }} - runs-on: macos-latest - env: - PYTHON_VERSION: ${{ matrix.PYTHON_VERSION }} - RUNNER_OS: "macos" - strategy: - fail-fast: false - matrix: - PYTHON_VERSION: ["3.5", "3.6", "3.7", "3.8"] - steps: - - name: Checkout branch - uses: actions/checkout@v2 - with: - submodules: true - - name: Install latest Rust nightly - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly - override: true - components: rustfmt, clippy - # - name: Install Haskell Stack - # uses: mstksg/setup-stack@v1 - - name: Install Conda - uses: goanpeca/setup-miniconda@v1 - with: - activate-environment: test - auto-update-conda: true - auto-activate-base: false - python-version: ${{ matrix.PYTHON_VERSION }} - - name: Compile duckling-ffi - shell: bash -l {0} - run: bash -l .github/scripts/build_ffi.sh - - name: Install build dependencies - shell: bash -l {0} - run: pip install setuptools-rust wheel auditwheel delocate toml - - name: Build wheel - shell: bash -l {0} - run: bash -l .github/scripts/build_mac_wheels.sh - - name: Upload wheel artifact - uses: actions/upload-artifact@v2 - with: - name: mac_dist - path: | - dist/*.whl - dist/*.tar.gz diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index f646998..0000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Linux Tests - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -env: - CARGO_TERM_COLOR: always - -jobs: - linux: - name: Linux Py${{ matrix.PYTHON_VERSION }} - runs-on: ubuntu-latest - env: - PYTHON_VERSION: ${{ matrix.PYTHON_VERSION }} - RUNNER_OS: "ubuntu" - strategy: - fail-fast: false - matrix: - PYTHON_VERSION: ["3.5", "3.6", "3.7", "3.8"] - steps: - - name: Checkout branch - uses: actions/checkout@v2 - with: - submodules: true - - name: Install latest Rust nightly - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly - override: true - components: rustfmt, clippy - - name: Print Rust version - shell: bash -l {0} - run: rustc --version - - name: Install Conda - uses: goanpeca/setup-miniconda@v1 - with: - activate-environment: test - auto-update-conda: true - auto-activate-base: false - python-version: ${{ matrix.PYTHON_VERSION }} - - name: Install Haskell Stack - uses: mstksg/setup-stack@v1 - - name: Compile duckling-ffi - shell: bash -l {0} - run: bash -l .github/scripts/build_ffi.sh - - name: Install build/test dependencies - shell: bash -l {0} - run: pip install maturin toml pytest pytest-cov coverage pendulum - - name: Build pyduckling - shell: bash -l {0} - run: bash -l .github/scripts/build_pyduckling.sh - - name: Run tests - shell: bash -l {0} - run: bash -l .github/scripts/run_tests.sh diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index efcb25e..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "duckling-ffi"] - path = duckling-ffi - url = https://github.com/treble-ai/duckling-ffi diff --git a/CHANGELOG.md b/CHANGELOG.md index 1460d06..6e2d9eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ -## Version 0.1.0 (2020/09/03) +# Changelog + +## v0.1.0 (2020/09/03) ### Issues Closed @@ -24,3 +26,6 @@ In this release 6 issues were closed. * [PR 3](https://github.com/treble-ai/pyduckling/pull/3) - PR: Enable Github Actions, by [@andfoy](https://github.com/andfoy) ([1](https://github.com/treble-ai/pyduckling/issues/1)) In this release 9 pull requests were closed. + + +\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* diff --git a/Cargo.toml b/Cargo.toml index fe663d8..9f9a77c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "pyduckling-native" -version = "0.1.1-dev0" +name = "pyduckling-native-phihos" +version = "0.1.1-dev2" authors = ["Edgar Andrés Margffoy Tuay "] description = "Rust-based Python wrapper for duckling library in Haskell." repository = "https://github.com/treble-ai/pyduckling" @@ -19,17 +19,5 @@ crate-type = ["cdylib"] libc = "0.2" [dependencies.pyo3] -version = "0.10.1" +version = "0.20.0" features = ["extension-module"] - -[package.metadata.maturin] -requires-dist = ["pendulum"] -classifier = [ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8" -] diff --git a/HISTORY.md b/HISTORY.md new file mode 100644 index 0000000..b300c94 --- /dev/null +++ b/HISTORY.md @@ -0,0 +1,26 @@ +## v0.1.0 (2020/09/03) + +### Issues Closed + +* [Issue 10](https://github.com/treble-ai/pyduckling/issues/10) - Release v0.1.0 +* [Issue 9](https://github.com/treble-ai/pyduckling/issues/9) - Add RELEASE instructions ([PR 15](https://github.com/treble-ai/pyduckling/pull/15) by [@andfoy](https://github.com/andfoy)) +* [Issue 8](https://github.com/treble-ai/pyduckling/issues/8) - Improve README and add compilling instructions ([PR 12](https://github.com/treble-ai/pyduckling/pull/12) by [@andfoy](https://github.com/andfoy)) +* [Issue 4](https://github.com/treble-ai/pyduckling/issues/4) - Migrate all functions ([PR 5](https://github.com/treble-ai/pyduckling/pull/5) by [@andfoy](https://github.com/andfoy)) +* [Issue 2](https://github.com/treble-ai/pyduckling/issues/2) - Add tests ([PR 7](https://github.com/treble-ai/pyduckling/pull/7) by [@andfoy](https://github.com/andfoy)) +* [Issue 1](https://github.com/treble-ai/pyduckling/issues/1) - Setup CI using Github Actions ([PR 3](https://github.com/treble-ai/pyduckling/pull/3) by [@andfoy](https://github.com/andfoy)) + +In this release 6 issues were closed. + +### Pull Requests Merged + +* [PR 15](https://github.com/treble-ai/pyduckling/pull/15) - PR: Add release instructions, by [@andfoy](https://github.com/andfoy) ([9](https://github.com/treble-ai/pyduckling/issues/9)) +* [PR 14](https://github.com/treble-ai/pyduckling/pull/14) - PR: Rename PyPi distribution to pyduckling-native, by [@andfoy](https://github.com/andfoy) +* [PR 13](https://github.com/treble-ai/pyduckling/pull/13) - PR: Patch and relocate wheels manually, add macOS support, by [@andfoy](https://github.com/andfoy) +* [PR 12](https://github.com/treble-ai/pyduckling/pull/12) - PR: Improve README and add compilation instructions, by [@andfoy](https://github.com/andfoy) ([8](https://github.com/treble-ai/pyduckling/issues/8)) +* [PR 11](https://github.com/treble-ai/pyduckling/pull/11) - PR: Add typing stubs, by [@andfoy](https://github.com/andfoy) +* [PR 7](https://github.com/treble-ai/pyduckling/pull/7) - PR: Add pyduckling tests, by [@andfoy](https://github.com/andfoy) ([2](https://github.com/treble-ai/pyduckling/issues/2)) +* [PR 6](https://github.com/treble-ai/pyduckling/pull/6) - Fix CIs, by [@andfoy](https://github.com/andfoy) +* [PR 5](https://github.com/treble-ai/pyduckling/pull/5) - PR: Migrate all functions, by [@andfoy](https://github.com/andfoy) ([4](https://github.com/treble-ai/pyduckling/issues/4)) +* [PR 3](https://github.com/treble-ai/pyduckling/pull/3) - PR: Enable Github Actions, by [@andfoy](https://github.com/andfoy) ([1](https://github.com/treble-ai/pyduckling/issues/1)) + +In this release 9 pull requests were closed. diff --git a/README.md b/README.md index ced6487..11792d6 100644 --- a/README.md +++ b/README.md @@ -1,81 +1,42 @@ # PyDuckling -[![Project License - MIT](https://img.shields.io/pypi/l/pyduckling-native.svg)](https://raw.githubusercontent.com/treble-ai/pyduckling-native/master/LICENSE) -[![pypi version](https://img.shields.io/pypi/v/pyduckling-native.svg)](https://pypi.org/project/pyduckling-native/) -[![conda version](https://img.shields.io/conda/vn/treble-ai/pyduckling.svg)](https://www.anaconda.com/download/) -[![download count](https://img.shields.io/conda/dn/treble-ai/pyduckling.svg)](https://www.anaconda.com/download/) -[![Downloads](https://pepy.tech/badge/pyduckling-native)](https://pepy.tech/project/pyduckling-native) -[![PyPI status](https://img.shields.io/pypi/status/pyduckling-native.svg)](https://github.com/treble-ai/pyduckling-native) -![Linux Tests](https://github.com/treble-ai/pyduckling/workflows/Linux%20Tests/badge.svg?branch=master) -![Mac Tests](https://github.com/treble-ai/pyduckling/workflows/Mac%20Tests/badge.svg?branch=master) +[![Project License - MIT](https://img.shields.io/pypi/l/pyduckling-native.svg)](https://raw.githubusercontent.com/phihos/pyduckling-native/master/LICENSE) +[![pypi version](https://img.shields.io/pypi/v/pyduckling-native-phihos.svg)](https://pypi.org/project/pyduckling-native-phihos/) +[![Downloads](https://pepy.tech/badge/pyduckling-native-phihos)](https://pepy.tech/project/pyduckling-native-phihos) +[![PyPI status](https://img.shields.io/pypi/status/pyduckling-native-phihos.svg)](https://github.com/phihos/pyduckling-native) +![Linux Tests](https://github.com/phihos/pyduckling/workflows/build/badge.svg?branch=master) *Copyright © 2020– Treble.ai* +> ℹ️ This is a fork of the original pyduckling-native library. There are differences to the original: +> * Build against the latest version of [Duckling](https://github.com/facebook/duckling) +> * Supported Python versions range from 3.8 to 3.12 +> * x86_64 Linux only, but contributions welcome if you dare to take on the challenge + ## Overview This package provides native bindings for Facebook's [Duckling](https://github.com/facebook/duckling) in Python. This package supports all dimensions and languages available on the original library, and it does not require to spawn a Haskell server and does not use HTTP to call the Duckling API. -**Note:** This package is completely Haskell-less +> ℹ️ This package is completely Haskell-less ## Installing To install pyduckling, you can use both conda and pip package managers: ```bash # Using pip -pip install pyduckling-native - -# Using conda -conda install pyduckling -c treble-ai -``` - -**Notes:** Right now, we only provide package distributions for Linux (x86_64). We will provide Windows and Mac distributions on the next release - - -## Dependencies -To compile pyduckling, you will require the latest nightly release of [Rust](https://rustup.rs/), alongside [Cargo](https://crates.io/). Also, it requires a Python distribution with its corresponding development headers. Finally, this project depends on the following Cargo crates: - -* [PyO3](https://github.com/PyO3/pyo3): Library used to produce Python bindings from Rust code. -* [Maturin](https://github.com/PyO3/maturin): Build system to build and publish Rust-based Python packages - -Additionally, this package depends on [Duckling-FFI](https://github.com/treble-ai/duckling-ffi), used to compile the native interface to Duckling on Haskell. In order to compile Duckling-FFI, you will require the [Stack](https://haskell-lang.org/get-started) Haskell manager. - - -## Installing locally -Besides Rust and Stack, you will require the latest version of maturin installed to compile this project locally: - -```bash -pip install maturin toml -``` - -First, you will need to compile Duckling-FFI in order to produce the shared library ``libducklingffi``, to do so, you can use the git submodule found at the root of this repository: - -```bash -cd duckling-ffi -stack build +pip install pyduckling-native-phihos ``` -Then, you will need to move the resulting binary ``libducklingffi.so`` to the ``ext_lib`` folder: +> ℹ️ Right now, we only provide package distributions for Linux (x86_64). -```bash -cp duckling-ffi/libducklingffi.so ext_lib -``` -After completing this procedure, it is possible to execute the following command to compile pyduckling: +## Version Matrix -```bash -maturin develop -``` - -In order to produce wheels, ``maturin build`` can be used instead. This project supports [PEP517](https://www.python.org/dev/peps/pep-0517/), thus pip can be used to install this package as well: +The following table shows which PyDuckling version corresponds to which Duckling version -```bash -pip install -U . -``` - -## Running tests -We use pytest to run tests as it follows (after calling ``maturin develop``): - -```bash -pytest -v duckling/tests -``` +| PyDuckling | Duckling | +|--------------------|-------------------------------------------------------------------| +| 0.2.0 (unreleased) | v0.2.0.0 (commit [7520daa](https://github.com/facebook/duckling)) | +| | | +| | | ## Package usage PyDuckling provides access to the parsing capabilities of Duckling used to extract structured data from text. @@ -85,6 +46,8 @@ PyDuckling provides access to the parsing capabilities of Duckling used to extra from duckling import (load_time_zones, parse_ref_time, parse_lang, default_locale_lang, parse_locale, parse_dimensions, parse, Context) +# Install with pip install pendulum +import pendulum # Load reference time for time parsing time_zones = load_time_zones("/usr/share/zoneinfo") @@ -110,7 +73,7 @@ valid_dimensions = ["amount-of-money", "credit-card-number", "distance", output_dims = parse_dimensions(valid_dimensions) # Parse a phrase -result = parse('En dos semanas', context, dims, False) +result = parse('En dos semanas', context, output_dims, False) ``` This wrapper allows access to all the dimensions and languages available on Duckling: @@ -131,9 +94,82 @@ This wrapper allows access to all the dimensions and languages available on Duck | `url` | "https://api.wit.ai/message?q=hi" | `{"value":"https://api.wit.ai/message?q=hi","domain":"api.wit.ai"}` | | `volume` | "4 gallons" | `{"value":4,"type":"value","unit":"gallon"}` | +## Dependencies +To compile pyduckling, you will require the latest nightly release of [Rust](https://rustup.rs/), alongside [Cargo](https://crates.io/). Also, it requires a Python distribution with its corresponding development headers. Finally, this project depends on the following Cargo crates: + +* [PyO3](https://github.com/PyO3/pyo3): Library used to produce Python bindings from Rust code. +* [Maturin](https://github.com/PyO3/maturin): Build system to build and publish Rust-based Python packages + +Additionally, this package depends on [Duckling-FFI](https://github.com/treble-ai/duckling-ffi), used to compile the native interface to Duckling on Haskell. In order to compile Duckling-FFI, you will require the [Stack](https://haskell-lang.org/get-started) Haskell manager. + + +## Installing locally + +### Via Docker + +The only thing you need is a running [Docker](https://docs.docker.com/engine/install/) daemon +and permission to run `docker build` and `docker run`. +Then just run +```shell +./build.sh +``` + +All build dependencies are already installed in container images. The build script will get them and start +containers for building the Haskell lib and the Python lib. The directories `.cache`, `duckling-ffi/.stack-work` +and `target` will appear. The first two are for accelerating future builds and the last one will contain your final +build result. + +After the `build.sh` completed successfully, you can find wheel files (binary distributions of the library) inside `target/wheels`. +The Python version (e.g. Python 3.11 = `cp311`), the libc variant +(`manylinux` or `musllinux`, if you are unsure you probably need manylinux) and the CPU architecture +(currently only `x86_64`) are encoded in the file name. Just pick the file matching to your system and install it with: + +```shell +pip install -U target/wheels/.whl +``` + +### Manually + +Besides Rust and Stack, you will require the latest version of maturin installed to compile this project locally: + +```bash +pip install maturin toml +``` + +First, you will need to compile Duckling-FFI in order to produce the shared library ``libducklingffi``, to do so, you can use the git submodule found at the root of this repository: + +```bash +cd duckling-ffi +stack build +``` + +Then, you will need to move the resulting binary ``libducklingffi.a`` to the ``ext_lib`` folder: + +```bash +cp duckling-ffi/libducklingffi.a ext_lib +``` + +After completing this procedure, it is possible to execute the following command to compile pyduckling: + +```bash +maturin develop +``` + +In order to produce wheels, ``maturin build`` can be used instead. This project supports [PEP517](https://www.python.org/dev/peps/pep-0517/), thus pip can be used to install this package as well: + +```bash +pip install -U . +``` + +## Running tests +We use pytest to run tests as it follows (after calling ``maturin develop``): + +```bash +pytest -v duckling/tests +``` ## Changelog -Please see our [CHANGELOG](https://github.com/treble-ai/pyduckling/blob/master/CHANGELOG.md) file to learn more about our new features and improvements. +Please see our [CHANGELOG](https://github.com/phihos/pyduckling/blob/master/CHANGELOG.md) file to learn more about our new features and improvements. ## Contribution guidelines diff --git a/RELEASE.md b/RELEASE.md index 3f4ff2e..cf08960 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,20 +1,24 @@ To release a new version of pyduckling: 1. git fetch upstream && git checkout upstream/master -2. Close milestone on GitHub -3. git clean -xfdi -4. Update CHANGELOG.md with loghub -5. git add -A && git commit -m "Update Changelog" -6. Update release version in ``Cargo.toml`` (set release version, remove 'dev0') +2. git clean -xfdi +3. Update CHANGELOG.md with + ```shell + docker run -it --rm \ + -v "$(pwd)":/usr/local/src/your-app \ + githubchangeloggenerator/github-changelog-generator \ + -u phihos -p pyduckling --future-release + ``` +4. git add -A && git commit -m "Update Changelog" +5. Update release version in ``Cargo.toml`` (set release version, remove 'dev0') +6. Update the [version matrix](./README.md#version-matrix) in `README.md` by removing the `(unreleased)` behind the release version. 7. git add -A && git commit -m "Release vX.X.X" -8. git tag -a vX.X.X -m "Release vX.X.X" -9. git push upstream master -10. git push upstream --tags -11. Wait for GitHub Actions to produce the wheels -12. Download the wheels locally for Linux and Mac -13. twine upload dist/* -14. Update development version in ``Cargo.toml`` (add '-dev0' and increment minor, see [1](#explanation)) -15. git add -A && git commit -m "Back to work" -16. git push upstream master +8. git push upstream master +9. Make release on Github and use tag `v` +10. Wait for GitHub Actions to upload the wheels to PyPI +11. Update development version in ``Cargo.toml`` (add '-dev0' and increment minor, see [1](#explanation)) +12. Update the [version matrix](./README.md#version-matrix) in `README.md` by adding ` (unreleased)`. +13. git add -A && git commit -m "Back to work" +14. git push upstream master [1] We need to append '-dev0', as Cargo does not support the '.dev0' diff --git a/build-vars.sh b/build-vars.sh new file mode 100644 index 0000000..a551a46 --- /dev/null +++ b/build-vars.sh @@ -0,0 +1,19 @@ +# common source for all build variables +export PYTHON3_VERSION_MIN=8 +export PYTHON3_VERSION_MAX=12 +export PYTHON3_VERSION_RANGE=$(seq "${PYTHON3_VERSION_MIN}" 1 "${PYTHON3_VERSION_MAX}") +export GHC_VERSION=8.8.4 +export CABAL_VERSION=3.2.0.0 +export ALPINE_VERSION_DUCKLING_FFI=3.12 +export ALPINE_VERSION_PYDUCKLING=3.19 +export MATURIN_VERSION=1.4.0 +export IMAGE_PREFIX="ghcr.io/phihos/pyduckling" +export BUILD_IMAGE_DUCKLING_FFI=$IMAGE_PREFIX/duckling-ffi-build:alpine-${ALPINE_VERSION_DUCKLING_FFI}-ghc-${GHC_VERSION}-cabal-${CABAL_VERSION} +export BUILD_IMAGE_PYDUCKLING=$IMAGE_PREFIX/pyduckling-build:alpine-${ALPINE_VERSION_PYDUCKLING}-maturin-${MATURIN_VERSION} + +function build_image_pyduckling_for_python3_version() { + python3_version=$1 + libc_version=$2 + echo "$IMAGE_PREFIX/pyduckling-build:${libc_version}-python-3.${python3_version}-maturin-${MATURIN_VERSION}" +} +export -f build_image_pyduckling_for_python3_version diff --git a/build.rs b/build.rs index c0cddd0..7aa996a 100644 --- a/build.rs +++ b/build.rs @@ -1,37 +1,10 @@ -use std::path::Path; -use std::process::Command; -use std::{env, str}; - -fn command_ok(cmd: &mut Command) -> bool { - cmd.status().ok().map_or(false, |s| s.success()) -} - -fn command_output(cmd: &mut Command) -> String { - str::from_utf8(&cmd.output().unwrap().stdout) - .unwrap() - .trim() - .to_string() -} +use std::env; fn main() { - if command_ok(Command::new("stack").arg("--version")) { - let ghc_lib = - command_output(Command::new("stack").args(&["exec", "--", "ghc", "--print-libdir"])); - let ghc_version = - command_output(Command::new("stack").args(&["exec", "--", "ghc", "--numeric-version"])); - let dir_path = env::current_dir().unwrap(); - let path = dir_path.to_str().unwrap(); - let ghc_lib_path = Path::new(&ghc_lib); - let rts_path = ghc_lib_path.join("rts"); - println!("cargo:rustc-link-search=native={}/ext_lib/", path); - println!( - "cargo:rustc-link-search=native={}", - rts_path.to_str().unwrap() - ); - println!("cargo:rustc-link-lib=dylib=ducklingffi"); - println!("cargo:rustc-link-lib=dylib=HSrts-ghc{}", ghc_version); - println!("cargo:rustc-env=GHC_VERSION={}", ghc_version); - } else { - panic!("Stack was not found in the PATH") - } + let dir_path = env::current_dir().unwrap(); + let path = dir_path.to_str().unwrap(); + println!("cargo:rustc-link-lib=static=ducklingffi"); + println!("cargo:rustc-link-search=native={}/ext_lib/", path); + println!("cargo:rustc-link-lib=dylib=pcre"); + println!("cargo:rustc-link-lib=dylib=gmp"); } diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..0266652 --- /dev/null +++ b/build.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +set -eo pipefail + +cd "${0%/*}" || exit # go to script dir + +# load common vars +source build-vars.sh + +export USER_HOME_CACHE_PATH="$(pwd)"/.cache/userhome +export STATIC_LIB_CONTAINER_NAME="pyduckling-build-static-lib" +export PYDUCKLING_CONTAINER_NAME="pyduckling-build-wheel" + +if ! command -v docker &> /dev/null +then + echo "docker could not be found" + exit 1 +fi + +function cleanup { + echo "Cleaning up..." + docker rm -f "${STATIC_LIB_CONTAINER_NAME}" > /dev/null 2>&1 || true + for python3_version in $PYTHON3_VERSION_RANGE; do + CONTAINER_NAME="${PYDUCKLING_CONTAINER_NAME}-${python3_version}-glibc" + docker rm -f "${CONTAINER_NAME}" > /dev/null 2>&1 || true + done + for python3_version in $PYTHON3_VERSION_RANGE; do + CONTAINER_NAME="${PYDUCKLING_CONTAINER_NAME}-${python3_version}-musl" + docker rm -f "${CONTAINER_NAME}" > /dev/null 2>&1 || true + done +} +trap cleanup EXIT + +function build_pyduckling_build_image { + python3_version=$1 + libc_version=$2 + BUILD_IMAGE=$(build_image_pyduckling_for_python3_version "$python3_version" "$libc_version") + if docker manifest inspect "${BUILD_IMAGE}" > /dev/null 2>&1; then + echo "Image \"${BUILD_IMAGE}\" already available. Skipping build..." + else + echo "Building \"${BUILD_IMAGE}\"..." + export PYTHON3_VERSION=$python3_version + export LIBC_VERSION=$libc_version + containers/pyduckling/build.sh + fi +} +export -f build_pyduckling_build_image + +function build_python_package() { + python3_version=$1 + libc_version=$2 + BUILD_IMAGE=$(build_image_pyduckling_for_python3_version "$python3_version" "$libc_version") + CONTAINER_NAME="${PYDUCKLING_CONTAINER_NAME}-${python3_version}-${libc_version}" + VENV_NAME="venv-3.${python3_version}-${libc_version}" + echo "Building wheel for Python 3.${python3_version} in ${BUILD_IMAGE}..." + docker run \ + --mount type=bind,source="$(pwd)",target=/repo \ + --mount type=bind,source="${USER_HOME_CACHE_PATH}",target=/userhome \ + --user "$(id -u):$(id -g)" \ + --name "${CONTAINER_NAME}" \ + -e HOME=/userhome \ + -e VENV="/userhome/${VENV_NAME}" \ + --detach \ + "${BUILD_IMAGE}" \ + sleep infinity + docker exec "${CONTAINER_NAME}" bash -c 'if [[ ! -d "$VENV" ]]; then python -m venv "$VENV"; fi' + docker exec "${CONTAINER_NAME}" bash -c 'source "$VENV/bin/activate" && pip install pytest pytest-cov coverage pendulum' + docker exec "${CONTAINER_NAME}" bash -c 'source "$VENV/bin/activate" && cd /repo && maturin develop' + docker exec "${CONTAINER_NAME}" bash -c 'source "$VENV/bin/activate" && cd /repo && pytest -x -v --cov=duckling duckling/tests' + docker exec "${CONTAINER_NAME}" bash -c 'source "$VENV/bin/activate" && cd /repo && maturin build -r' + if [[ "$PUBLISH" == "1" ]]; then + docker exec -e MATURIN_USERNAME -e MATURIN_PASSWORD "${CONTAINER_NAME}" bash -c 'source "$VENV/bin/activate" && cd /repo && maturin publish --skip-existing --no-sdist' + fi +} +export -f build_python_package + +# --- build necessary container images +if docker manifest inspect "${BUILD_IMAGE_DUCKLING_FFI}" > /dev/null 2>&1; then + echo "Image \"${BUILD_IMAGE_DUCKLING_FFI}\" already available. Skipping build..." +else + echo "Building \"${BUILD_IMAGE_DUCKLING_FFI}\"..." + containers/duckling-ffi/build.sh +fi +echo -n $PYTHON3_VERSION_RANGE | parallel -j0 --halt now,fail=1 -d ' ' 'build_pyduckling_build_image {} glibc' +echo -n $PYTHON3_VERSION_RANGE | parallel -j0 --halt now,fail=1 -d ' ' 'build_pyduckling_build_image {} musl' + +# --- create cache directory for faster repeated execution +mkdir -p "${USER_HOME_CACHE_PATH}" + +# --- build the statically linked library file +echo "---" +echo "Building statically linked \"libducklingffi.a\"..." +docker run \ + --mount type=bind,source="$(pwd)"/duckling-ffi,target=/duckling-ffi \ + --mount type=bind,source="${USER_HOME_CACHE_PATH}",target=/userhome \ + --user "$(id -u):$(id -g)" \ + --name "${STATIC_LIB_CONTAINER_NAME}" \ + -e HOME=/userhome \ + --detach \ + "${BUILD_IMAGE_DUCKLING_FFI}" \ + sleep infinity +docker exec "${STATIC_LIB_CONTAINER_NAME}" bash -c 'cd /duckling-ffi && stack build --no-install-ghc --system-ghc --allow-different-user --force-dirty' +docker cp "${STATIC_LIB_CONTAINER_NAME}:/duckling-ffi/libducklingffi.a" ext_lib/libducklingffi.a + +# --- build binary distributions +echo "---" +echo "Building GLIBC wheels for all python versions..." +echo -n $PYTHON3_VERSION_RANGE | parallel -j0 --halt now,fail=1 -d ' ' 'build_python_package {} glibc' +echo "---" +echo "Building MUSL wheels for all python versions..." +echo -n $PYTHON3_VERSION_RANGE | parallel -j0 --halt now,fail=1 -d ' ' 'build_python_package {} musl' diff --git a/containers/duckling-ffi/Dockerfile b/containers/duckling-ffi/Dockerfile new file mode 100644 index 0000000..0a5cb0f --- /dev/null +++ b/containers/duckling-ffi/Dockerfile @@ -0,0 +1,189 @@ +ARG GHC_VERSION +ARG CABAL_VERSION +ARG ALPINE_VERSION + +ARG GHC_VERSION_BUILD=${GHC_VERSION} +ARG CABAL_VERSION_BUILD=${CABAL_VERSION} + +FROM alpine:${ALPINE_VERSION} as cabal + +ARG CABAL_VERSION_BUILD + +ENV CABAL_VERSION=${CABAL_VERSION_BUILD} + +COPY ghc-8.8.patch /tmp/ +COPY cabal-0001-force-ld.gold.patch /tmp/ + +RUN apk add --no-cache \ + autoconf \ + automake \ + binutils-gold \ + build-base \ + coreutils \ + cpio \ + curl \ + ghc \ + gnupg \ + linux-headers \ + libffi-dev \ + ncurses-dev \ + perl \ + python3 \ + xz \ + zlib-dev + +RUN cd /tmp/ \ + && curl -sSLO https://downloads.haskell.org/~cabal/cabal-install-$CABAL_VERSION/cabal-install-$CABAL_VERSION.tar.gz \ + && tar zxf cabal-install-$CABAL_VERSION.tar.gz \ + && cd /tmp/cabal-install-$CABAL_VERSION/ \ + && patch < /tmp/ghc-8.8.patch \ + && patch < /tmp/cabal-0001-force-ld.gold.patch \ + && EXTRA_CONFIGURE_OPTS="" ./bootstrap.sh --jobs "$(($(nproc)+1))" --no-doc + +FROM alpine:3.12 as prepare + +ARG GHC_VERSION_BUILD +ARG CABAL_VERSION_BUILD + +ENV GHC_VERSION=${GHC_VERSION_BUILD} +ENV CABAL_VERSION=${CABAL_VERSION_BUILD} + +RUN apk add --no-cache \ + bash \ + build-base \ + bzip2 \ + bzip2-dev \ + #bzip2-static \ + curl \ + curl-static \ + fakeroot \ + ghc \ + git \ + gmp-dev \ + libcurl \ + libffi \ + libffi-dev \ + ncurses-dev \ + ncurses-static \ + openssl-dev \ + pcre \ + pcre-dev \ + pcre2 \ + pcre2-dev \ + perl \ + wget \ + xz \ + xz-dev \ + zlib \ + zlib-dev + +COPY --from=cabal /root/.cabal/bin/cabal /usr/bin/cabal + + +RUN apk upgrade --no-cache \ + && apk add --no-cache \ + autoconf \ + automake \ + binutils-gold \ + build-base \ + coreutils \ + cpio \ + curl \ + gnupg \ + linux-headers \ + libffi-dev \ + ncurses-dev \ + perl \ + python3 \ + xz \ + zlib-dev + +RUN cd /tmp \ + && curl -sSLO https://downloads.haskell.org/~ghc/$GHC_VERSION/ghc-$GHC_VERSION-src.tar.xz \ + && curl -sSLO https://downloads.haskell.org/~ghc/$GHC_VERSION/ghc-$GHC_VERSION-src.tar.xz.sig \ + && gpg --keyserver hkps://keyserver.ubuntu.com:443 \ + --receive-keys FFEB7CE81E16A36B3E2DED6F2DE04D4E97DB64AD || \ + gpg --keyserver hkp://keyserver.ubuntu.com:80 \ + --receive-keys FFEB7CE81E16A36B3E2DED6F2DE04D4E97DB64AD \ + && gpg --verify ghc-$GHC_VERSION-src.tar.xz.sig ghc-$GHC_VERSION-src.tar.xz \ + && tar xf ghc-$GHC_VERSION-src.tar.xz \ + && cd ghc-$GHC_VERSION \ + # Set llvm version to 10 + && sed -i -e 's/LlvmVersion=7/LlvmVersion=10/g' configure.ac \ + && cp mk/build.mk.sample mk/build.mk \ + && echo 'BuildFlavour=perf-llvm' >> mk/build.mk \ + && echo 'BeConservative=YES' >> mk/build.mk \ + && echo 'SplitSections=YES' >> mk/build.mk \ + && echo 'HADDOCK_DOCS=YES' >> mk/build.mk \ + && echo 'HSCOLOUR_SRCS=NO' >> mk/build.mk \ + && echo 'BUILD_SPHINX_HTML=NO' >> mk/build.mk \ + && echo 'BUILD_SPHINX_PS=NO' >> mk/build.mk \ + && echo 'BUILD_SPHINX_PDF=NO' >> mk/build.mk \ + && echo 'GhcLibHcOpts += -fPIC' >> mk/build.mk \ + && echo 'GhcRtsHcOpts += -fPIC' >> mk/build.mk \ + && echo 'GhcRtsCcOpts += -fPIC' >> mk/build.mk \ + && autoreconf \ + && ./configure --disable-ld-override LD=ld.gold \ + # Switch llvm-targets from unknown-linux-gnueabihf->alpine-linux + # so we can match the llvm vendor string alpine uses + && sed -i -e 's/unknown-linux-gnueabihf/alpine-linux/g' llvm-targets \ + && sed -i -e 's/unknown-linux-gnueabi/alpine-linux/g' llvm-targets \ + && sed -i -e 's/unknown-linux-gnu/alpine-linux/g' llvm-targets \ + # See https://unix.stackexchange.com/questions/519092/what-is-the-logic-of-using-nproc-1-in-make-command + && make -j"$(($(nproc)+1))" \ + && make binary-dist \ + && cabal update \ + # See https://gitlab.haskell.org/ghc/ghc/-/wikis/commentary/libraries/version-history + && cabal install --constraint 'Cabal-syntax<3.3' cabal-install-$CABAL_VERSION + +FROM alpine:3.12 as ghc + +ARG GHC_VERSION_BUILD +ARG CABAL_VERSION_BUILD + +ENV GHC_VERSION=${GHC_VERSION_BUILD} +ENV CABAL_VERSION=${CABAL_VERSION_BUILD} + +RUN apk add --no-cache \ + bash \ + build-base \ + bzip2 \ + bzip2-dev \ + bzip2-static \ + curl \ + curl-static \ + dpkg \ + fakeroot \ + git \ + gmp-dev \ + libcurl \ + libffi \ + libffi-dev \ + llvm10 \ + ncurses-dev \ + ncurses-static \ + openssl-dev \ + openssl-libs-static \ + pcre \ + pcre-dev \ + pcre2 \ + pcre2-dev \ + perl \ + wget \ + xz \ + xz-dev \ + zlib \ + zlib-dev \ + zlib-static + +COPY --from=prepare /tmp/ghc-$GHC_VERSION/ghc-$GHC_VERSION-*-alpine-linux.tar.xz /tmp/ +COPY --from=prepare /root/.cabal/bin/cabal /usr/bin/cabal + +RUN cd /tmp \ + && tar -xJf ghc-$GHC_VERSION-*-alpine-linux.tar.xz \ + && cd ghc-$GHC_VERSION \ + && ./configure --disable-ld-override --prefix=/usr \ + && make install \ + && rm -rf /tmp/* + +RUN curl -sSL https://get.haskellstack.org/ | sh diff --git a/containers/duckling-ffi/build.sh b/containers/duckling-ffi/build.sh new file mode 100755 index 0000000..908e521 --- /dev/null +++ b/containers/duckling-ffi/build.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cd "${0%/*}" || exit # go to script dir + +source ../../build-vars.sh + +docker buildx build \ + --build-arg="GHC_VERSION=${GHC_VERSION}" \ + --build-arg="CABAL_VERSION=${CABAL_VERSION}" \ + --build-arg="ALPINE_VERSION=${ALPINE_VERSION_DUCKLING_FFI}" \ + -t "${BUILD_IMAGE_DUCKLING_FFI}" . + +if [[ ${PUSH_IMAGES} ]]; then + docker push "${BUILD_IMAGE_DUCKLING_FFI}" +fi diff --git a/containers/duckling-ffi/cabal-0001-force-ld.gold.patch b/containers/duckling-ffi/cabal-0001-force-ld.gold.patch new file mode 100644 index 0000000..b45f266 --- /dev/null +++ b/containers/duckling-ffi/cabal-0001-force-ld.gold.patch @@ -0,0 +1,10 @@ +--- a/bootstrap.sh ++++ a/bootstrap.sh +@@ -74,6 +74,7 @@ + + # Fall back to "ld"... might work. + [ -$LINK- = -""- ] && LINK=ld ++LINK="ld.gold" + + # And finally, see if we can compile and link something. + echo 'int main(){}' | $CC -xc - -o /dev/null || \ No newline at end of file diff --git a/containers/duckling-ffi/ghc-8.8.patch b/containers/duckling-ffi/ghc-8.8.patch new file mode 100644 index 0000000..258f60a --- /dev/null +++ b/containers/duckling-ffi/ghc-8.8.patch @@ -0,0 +1,78 @@ +--- a/bootstrap.sh ++++ b/bootstrap.sh +@@ -213,37 +213,37 @@ + + # Versions of the packages to install. + # The version regex says what existing installed versions are ok. +-PARSEC_VER="3.1.13.0"; PARSEC_VER_REGEXP="[3]\.[1]\." ++PARSEC_VER="3.1.14.0"; PARSEC_VER_REGEXP="[3]\.[1]\." + # >= 3.1 && < 3.2 +-DEEPSEQ_VER="1.4.3.0"; DEEPSEQ_VER_REGEXP="1\.[1-9]\." ++DEEPSEQ_VER="1.4.4.0"; DEEPSEQ_VER_REGEXP="1\.[1-9]\." + # >= 1.1 && < 2 +-BINARY_VER="0.8.5.1"; BINARY_VER_REGEXP="[0]\.[78]\." ++BINARY_VER="0.8.8.0"; BINARY_VER_REGEXP="[0]\.[78]\." + # >= 0.7 && < 0.9 +-TEXT_VER="1.2.3.0"; TEXT_VER_REGEXP="[1]\.[2]\." ++TEXT_VER="1.2.4.0"; TEXT_VER_REGEXP="[1]\.[2]\." + # >= 1.2 && < 1.3 +-NETWORK_URI_VER="2.6.1.0"; NETWORK_URI_VER_REGEXP="2\.6\.(0\.[2-9]|[1-9])" ++NETWORK_URI_VER="2.6.3.0"; NETWORK_URI_VER_REGEXP="2\.6\.(0\.[2-9]|[1-9])" + # >= 2.6.0.2 && < 2.7 +-NETWORK_VER="2.7.0.0"; NETWORK_VER_REGEXP="2\.[0-7]\." +- # >= 2.0 && < 2.7 ++NETWORK_VER="2.8.0.0"; NETWORK_VER_REGEXP="2\.[0-8]\." ++ # >= 2.0 && < 2.8 + CABAL_VER="3.2.0.0"; CABAL_VER_REGEXP="3\.2\.[0-9]" + # >= 3.2 && < 3.3 + TRANS_VER="0.5.5.0"; TRANS_VER_REGEXP="0\.[45]\." + # >= 0.2.* && < 0.6 + MTL_VER="2.2.2"; MTL_VER_REGEXP="[2]\." + # >= 2.0 && < 3 +-HTTP_VER="4000.3.12"; HTTP_VER_REGEXP="4000\.(2\.([5-9]|1[0-9]|2[0-9])|3\.?)" ++HTTP_VER="4000.3.14"; HTTP_VER_REGEXP="4000\.(2\.([5-9]|1[0-9]|2[0-9])|3\.?)" + # >= 4000.2.5 < 4000.4 +-ZLIB_VER="0.6.2"; ZLIB_VER_REGEXP="(0\.5\.([3-9]|1[0-9])|0\.6)" ++ZLIB_VER="0.6.2.1"; ZLIB_VER_REGEXP="(0\.5\.([3-9]|1[0-9])|0\.6)" + # >= 0.5.3 && <= 0.7 + TIME_VER="1.9.1" TIME_VER_REGEXP="1\.[1-9]\.?" + # >= 1.1 && < 1.10 + RANDOM_VER="1.1" RANDOM_VER_REGEXP="1\.[01]\.?" + # >= 1 && < 1.2 +-STM_VER="2.4.5.0"; STM_VER_REGEXP="2\." ++STM_VER="2.4.5.1"; STM_VER_REGEXP="2\." + # == 2.* +-HASHABLE_VER="1.2.7.0"; HASHABLE_VER_REGEXP="1\." ++HASHABLE_VER="1.3.0.0"; HASHABLE_VER_REGEXP="1\." + # 1.* +-ASYNC_VER="2.2.1"; ASYNC_VER_REGEXP="2\." ++ASYNC_VER="2.2.2"; ASYNC_VER_REGEXP="2\." + # 2.* + BASE16_BYTESTRING_VER="0.1.1.6"; BASE16_BYTESTRING_VER_REGEXP="0\.1" + # 0.1.* +@@ -251,7 +251,7 @@ + # >=1.0 + CRYPTOHASH_SHA256_VER="0.11.101.0"; CRYPTOHASH_SHA256_VER_REGEXP="0\.11\.?" + # 0.11.* +-RESOLV_VER="0.1.1.1"; RESOLV_VER_REGEXP="0\.1\.[1-9]" ++RESOLV_VER="0.1.2.0"; RESOLV_VER_REGEXP="0\.1\.[1-9]" + # >= 0.1.1 && < 0.2 + MINTTY_VER="0.1.2"; MINTTY_VER_REGEXP="0\.1\.?" + # 0.1.* +@@ -261,13 +261,13 @@ + # 0.2.2.* + ED25519_VER="0.0.5.0"; ED25519_VER_REGEXP="0\.0\.?" + # 0.0.* +-HACKAGE_SECURITY_VER="0.6.0.0"; HACKAGE_SECURITY_VER_REGEXP="0\.6\." ++HACKAGE_SECURITY_VER="0.6.0.1"; HACKAGE_SECURITY_VER_REGEXP="0\.6\." + # >= 0.7.0.0 && < 0.7 +-TAR_VER="0.5.1.0"; TAR_VER_REGEXP="0\.5\.([1-9]|1[0-9]|0\.[3-9]|0\.1[0-9])\.?" ++TAR_VER="0.5.1.1"; TAR_VER_REGEXP="0\.5\.([1-9]|1[0-9]|0\.[3-9]|0\.1[0-9])\.?" + # >= 0.5.0.3 && < 0.6 + DIGEST_VER="0.0.1.2"; DIGEST_REGEXP="0\.0\.(1\.[2-9]|[2-9]\.?)" + # >= 0.0.1.2 && < 0.1 +-LUKKO_VER="0.1.1"; LUKKO_VER_REGEXP="0\.1\.[1-9]" ++LUKKO_VER="0.1.1.2"; LUKKO_VER_REGEXP="0\.1\.[1-9]" + # >= 0.1.1 && <0.2 + + HACKAGE_URL="https://hackage.haskell.org/package" \ No newline at end of file diff --git a/containers/pyduckling/build.sh b/containers/pyduckling/build.sh new file mode 100755 index 0000000..d08d848 --- /dev/null +++ b/containers/pyduckling/build.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -eo pipefail + +cd "${0%/*}" || exit # go to script dir + +source ../../build-vars.sh + +case "$LIBC_VERSION" in + musl) + cd musl + ;; + glibc) + cd glibc + ;; + *) + echo "Unsupported lib $2. Exiting..." + exit 1 +esac + +docker buildx build \ + --build-arg="MATURIN_VERSION=${MATURIN_VERSION}" \ + --build-arg="PYTHON_VERSION=3.${PYTHON3_VERSION}" \ + -t "$(build_image_pyduckling_for_python3_version "$PYTHON3_VERSION" "$LIBC_VERSION")" . + +if [[ "${PUSH_IMAGES}" ]]; then + docker push "$(build_image_pyduckling_for_python3_version "$PYTHON3_VERSION" "$LIBC_VERSION")" +fi diff --git a/containers/pyduckling/glibc/Dockerfile b/containers/pyduckling/glibc/Dockerfile new file mode 100644 index 0000000..04c13fd --- /dev/null +++ b/containers/pyduckling/glibc/Dockerfile @@ -0,0 +1,14 @@ +FROM quay.io/pypa/manylinux_2_28_x86_64 +ARG MATURIN_VERSION +ARG PYTHON_VERSION + +RUN ln -s /opt/python/cp${PYTHON_VERSION//.}-cp${PYTHON_VERSION//.}/bin/python /usr/bin/python \ + && ln -s /opt/python/cp${PYTHON_VERSION//.}-cp${PYTHON_VERSION//.}/bin/pip /usr/bin/pip +RUN yum install -y \ + tzdata \ + rust \ + cargo \ + pcre-devel \ + gmp-devel + +RUN pipx install "maturin[patchelf]==${MATURIN_VERSION}" diff --git a/containers/pyduckling/musl/Dockerfile b/containers/pyduckling/musl/Dockerfile new file mode 100644 index 0000000..0fd2bc0 --- /dev/null +++ b/containers/pyduckling/musl/Dockerfile @@ -0,0 +1,17 @@ +ARG PYTHON_VERSION +FROM python:${PYTHON_VERSION}-alpine +ARG MATURIN_VERSION + +RUN apk add --no-cache \ + bash \ + tzdata \ + rust \ + cargo \ + cmake \ + automake \ + autoconf \ + patchelf \ + build-base \ + pcre-dev gmp-dev \ + tar +RUN pip install --break-system-packages "maturin[patchelf]==${MATURIN_VERSION}" diff --git a/duckling-ffi b/duckling-ffi deleted file mode 160000 index 2dfdb40..0000000 --- a/duckling-ffi +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2dfdb400f947973a54a700acd95b34e2c6f0094e diff --git a/duckling-ffi/.gitignore b/duckling-ffi/.gitignore new file mode 100644 index 0000000..722dd67 --- /dev/null +++ b/duckling-ffi/.gitignore @@ -0,0 +1,26 @@ +dist +dist-* +cabal-dev +*.o +*.so +*.a +*.hi +*.hie +*.chi +*.so +*.chs.h +*.dyn_o +*.dyn_hi +.hpc +.hsenv +.cabal-sandbox/ +cabal.sandbox.config +*.prof +*.aux +*.hp +*.eventlog +.stack-work/ +cabal.project.local +cabal.project.local~ +.HTF/ +.ghc.environment.* diff --git a/duckling-ffi/DucklingFFI/FFI.hs b/duckling-ffi/DucklingFFI/FFI.hs new file mode 100644 index 0000000..76759c2 --- /dev/null +++ b/duckling-ffi/DucklingFFI/FFI.hs @@ -0,0 +1,366 @@ +module FFI + ( someFunc + , wcurrentReftime + , wloadTimeZoneSeries + ) +where + +import Control.Applicative hiding (empty) +import Data.Text ( Text ) +import Data.ByteString ( ByteString + , empty + ) +import Duckling.Core +-- import Duckling.Data.TimeZone +import Duckling.Resolve ( DucklingTime(..) ) +import Data.Aeson +import Data.Maybe +import Data.Tuple +import qualified Data.Text as Text +import qualified Data.Text.Encoding as Text +import qualified Data.ByteString.Lazy.Char8 as C8 + +import qualified Control.Exception as E +import Control.Monad.Extra +import Control.Monad.IO.Class +import Data.Coerce +import Data.Either +import Data.HashMap.Strict ( HashMap ) +import qualified Data.HashMap.Strict as HashMap +import qualified Data.HashSet as HashSet +import Data.String +import Data.String.Conversions ( cs ) +import Data.Time ( TimeZone(..) ) +import Data.Time.LocalTime.TimeZone.Olson +import Data.Time.LocalTime.TimeZone.Series +import Data.Time.Format +import Data.Time.Clock.POSIX ( posixSecondsToUTCTime ) +import System.Directory +import System.FilePath +import System.IO.Unsafe +import Text.Read + +import Foreign.StablePtr +import Foreign.Ptr +import Foreign.C +import Foreign.C.Types +import Foreign.Marshal.Array + +import Prelude +-- import Curryrs.Types + +-- TimeZoneSeries wrapper +newtype WrappedTimeZoneSeries = WrappedTimeZoneSeries { timeSeries :: HashMap Text TimeZoneSeries } +foreign export ccall tzdbDestroy :: Ptr () -> IO () + +tzdbCreate :: HashMap Text TimeZoneSeries -> IO (Ptr ()) +tzdbCreate x = castStablePtrToPtr <$> newStablePtr (WrappedTimeZoneSeries x) + +tzdbDestroy :: Ptr () -> IO () +tzdbDestroy p = freeStablePtr sp + where + sp :: StablePtr WrappedTimeZoneSeries + sp = castPtrToStablePtr p + +tzdbGet :: Ptr () -> IO (HashMap Text TimeZoneSeries) +tzdbGet p = timeSeries <$> deRefStablePtr (castPtrToStablePtr p) + +-- DucklingTime wrapper +newtype DucklingTimeWrapper = DucklingTimeWrapper { time :: DucklingTime } +foreign export ccall duckTimeDestroy :: Ptr () -> IO () + +duckTimeCreate :: DucklingTime -> IO (Ptr ()) +duckTimeCreate t = castStablePtrToPtr <$> newStablePtr (DucklingTimeWrapper t) + +duckTimeDestroy :: Ptr () -> IO () +duckTimeDestroy p = freeStablePtr sp + where + sp :: StablePtr DucklingTimeWrapper + sp = castPtrToStablePtr p + +duckTimeGet :: Ptr () -> IO DucklingTime +duckTimeGet p = time <$> deRefStablePtr (castPtrToStablePtr p) + +duckTimeToZST :: DucklingTime -> ZoneSeriesTime +duckTimeToZST (DucklingTime zst) = zst + +foreign export ccall duckTimeRepr :: Ptr() -> IO(CString) +duckTimeRepr p = do + duckTime <- duckTimeGet p + let zt = duckTimeToZST duckTime + let dateRepr = formatTime defaultTimeLocale "%FT%T%Q%z" zt + cRepr <- newCString dateRepr + return cRepr + +-- Lang wrapper +newtype LangWrapper = LangWrapper { lang :: Lang } +foreign export ccall langDestroy :: Ptr () -> IO () + +langCreate :: Lang -> IO (Ptr ()) +langCreate l = castStablePtrToPtr <$> newStablePtr (LangWrapper l) + +langDestroy :: Ptr () -> IO () +langDestroy p = freeStablePtr sp + where + sp :: StablePtr LangWrapper + sp = castPtrToStablePtr p + +langGet :: Ptr () -> IO Lang +langGet p = lang <$> deRefStablePtr (castPtrToStablePtr p) + +foreign export ccall langRepr :: Ptr() -> IO(CString) +langRepr p = do + langName <- langGet p + let langR = show langName + cRepr <- newCString langR + return cRepr + +-- Locale wrapper +newtype LocaleWrapper = LocaleWrapper { loc :: Locale } +foreign export ccall localeDestroy :: Ptr () -> IO () + +localeCreate :: Locale -> IO (Ptr ()) +localeCreate l = castStablePtrToPtr <$> newStablePtr (LocaleWrapper l) + +localeDestroy :: Ptr () -> IO () +localeDestroy p = freeStablePtr sp + where + sp :: StablePtr LocaleWrapper + sp = castPtrToStablePtr p + +localeGet :: Ptr () -> IO Locale +localeGet p = loc <$> deRefStablePtr (castPtrToStablePtr p) + +foreign export ccall localeRepr :: Ptr() -> IO(CString) +localeRepr p = do + localeF <- localeGet p + let localeR = show localeF + cRepr <- newCString localeR + return cRepr + +-- Dimension wrapper +newtype DimensionWrapper = DimensionWrapper { dimen :: Seal Dimension } +foreign export ccall dimensionDestroy :: Ptr () -> IO () + +dimensionCreate :: Seal Dimension -> IO (Ptr ()) +dimensionCreate d = castStablePtrToPtr <$> newStablePtr (DimensionWrapper d) + +dimensionDestroy :: Ptr() -> IO () +dimensionDestroy p = freeStablePtr sp + where + sp :: StablePtr DimensionWrapper + sp = castPtrToStablePtr p + +dimensionGet :: Ptr () -> IO (Seal Dimension) +dimensionGet p = dimen <$> deRefStablePtr (castPtrToStablePtr p) + +-- DimensionList wrapper +newtype DimensionListWrapper = DimensionListWrapper { + listDescriptor :: (Ptr(Ptr()), CInt) +} + +foreign export ccall dimensionListCreate :: Ptr(Ptr()) -> CInt -> IO(Ptr()) +foreign export ccall dimensionListLength :: Ptr() -> IO(CInt) +foreign export ccall dimensionListPtrs :: Ptr() -> IO(Ptr(Ptr())) +foreign export ccall dimensionListDestroy :: Ptr() -> IO () + +dimensionListCreate :: Ptr(Ptr ()) -> CInt -> IO(Ptr()) +dimensionListCreate ptrs numElements = castStablePtrToPtr <$> newStablePtr (DimensionListWrapper (ptrs, numElements)) + +dimensionListDescr :: Ptr () -> IO (Ptr(Ptr()), CInt) +dimensionListDescr p = listDescriptor <$> deRefStablePtr (castPtrToStablePtr p) + +dimensionListLength :: Ptr() -> IO(CInt) +dimensionListLength p = do + descr <- dimensionListDescr p + let numElements = snd descr + return numElements + +dimensionListPtrs :: Ptr() -> IO(Ptr(Ptr())) +dimensionListPtrs p = do + descr <- dimensionListDescr p + let dimensions = fst descr + return dimensions + +dimensionListDestroy :: Ptr() -> IO() +dimensionListDestroy p = freeStablePtr sp + where + sp :: StablePtr DimensionListWrapper + sp = castPtrToStablePtr p + + +-- | Reference implementation for pulling TimeZoneSeries data from local +-- Olson files. +-- Many linux distros have Olson data in "/usr/share/zoneinfo/" +loadTimeZoneSeries :: FilePath -> IO (HashMap Text TimeZoneSeries) +loadTimeZoneSeries base = do + files <- getFiles base + tzSeries <- mapM parseOlsonFile files + -- This data is large, will live a long time, and essentially be constant, + -- so it's a perfect candidate for compact regions + return $ HashMap.fromList $ rights tzSeries + where + -- Multiple versions of the data can exist. We intentionally ignore the + -- posix and right formats + ignored_dirs = HashSet.fromList $ map (base ) ["posix", "right"] + + -- Recursively crawls a directory to list every file underneath it, + -- ignoring certain directories as needed + getFiles :: FilePath -> IO [FilePath] + getFiles dir = do + fsAll <- getDirectoryContents dir + let fs = filter notDotFile fsAll + full_fs = map (dir ) fs + (dirs, files) <- partitionM doesDirectoryExist full_fs + + subdirs <- concatMapM + getFiles + [ d | d <- dirs, not $ HashSet.member d ignored_dirs ] + + return $ files ++ subdirs + + -- Attempts to read a file in Olson format and returns its + -- canonical name (file path relative to the base) and the data + parseOlsonFile :: FilePath -> IO (Either E.ErrorCall (Text, TimeZoneSeries)) + parseOlsonFile f = E.try $ do + r <- getTimeZoneSeriesFromOlsonFile f + return (Text.pack $ makeRelative base f, r) + + notDotFile s = not $ elem s [".", ".."] + + +parseTimeZone :: Text -> Maybe ByteString -> Text +parseTimeZone defaultTimeZone = maybe defaultTimeZone Text.decodeUtf8 + +someFunc :: IO () +someFunc = putStrLn "someFunc" + +foreign export ccall wcurrentReftime :: Ptr() -> CString -> IO(Ptr()) + +wcurrentReftime :: Ptr () -> CString -> IO (Ptr ()) +wcurrentReftime tzdb tzStr = do + timeSeries <- tzdbGet tzdb + -- wrapString <- stringGet strPtr + unwrappedStr <- peekCString $ tzStr + let hsStr = cs (unwrappedStr) + timeOut <- Duckling.Core.currentReftime timeSeries hsStr + convertedTime <- duckTimeCreate timeOut + return convertedTime + +foreign export ccall wloadTimeZoneSeries :: CString -> IO(Ptr ()) + +wloadTimeZoneSeries :: CString -> IO (Ptr ()) +wloadTimeZoneSeries pathCStr = do + unwrapString <- peekCString $ pathCStr + tzmap <- loadTimeZoneSeries unwrapString + wrappedDB <- tzdbCreate tzmap + return wrappedDB + + +foreign export ccall wparseRefTime :: Ptr() -> CString -> CSUSeconds -> IO(Ptr ()) + +wparseRefTime :: Ptr () -> CString -> CSUSeconds -> IO (Ptr ()) +wparseRefTime tzdb tzCStr refTimeC = do + timeSeries <- tzdbGet tzdb + unwrappedStr <- peekCString $ tzCStr + let utcTime = posixSecondsToUTCTime (realToFrac refTimeC) + let refTime = makeReftime timeSeries (cs (unwrappedStr)) utcTime + wrappedRefTime <- duckTimeCreate refTime + return wrappedRefTime + +parseLang :: Text -> Lang +parseLang l = fromMaybe EN (readMaybe (Text.unpack (Text.toUpper l))) + +foreign export ccall wparseLang :: CString -> IO(Ptr ()) + +wparseLang :: CString -> IO (Ptr ()) +wparseLang langCStr = do + langStr <- peekCString $ langCStr + let mappedLang = parseLang (cs (langStr)) + wrappedLang <- langCreate (mappedLang) + return wrappedLang + +foreign export ccall wmakeDefaultLocale :: Ptr() -> IO(Ptr ()) + +wmakeDefaultLocale :: Ptr () -> IO (Ptr ()) +wmakeDefaultLocale langPtr = do + lang <- langGet langPtr + let loc = makeLocale lang Nothing + wrappedLocale <- localeCreate (loc) + return wrappedLocale + +parseLocale :: Text -> Locale -> Locale +parseLocale x defaultLocale = maybe defaultLocale (`makeLocale` mregion) mlang + where + (mlang, mregion) = case chunks of + [a, b] -> (readMaybe a :: Maybe Lang, readMaybe b :: Maybe Region) + _ -> (Nothing, Nothing) + chunks = map Text.unpack . Text.split (== '_') . Text.toUpper $ x + +foreign export ccall wparseLocale :: CString -> Ptr() -> IO(Ptr ()) +wparseLocale :: CString -> Ptr() -> IO(Ptr ()) +wparseLocale localeCStr defaultLocalePtr = do + unwrappedLocale <- peekCString $ localeCStr + defaultLocale <- localeGet defaultLocalePtr + let loc = parseLocale (cs (unwrappedLocale)) defaultLocale + wrappedLocale <- localeCreate (loc) + return wrappedLocale + + +parseDimension :: Text -> Maybe (Seal Dimension) +parseDimension x = fromName x <|> fromCustomName x + where + fromCustomName :: Text -> Maybe (Seal Dimension) + fromCustomName name = HashMap.lookup name m + m = HashMap.fromList + [ -- ("my-dimension", This (CustomDimension MyDimension)) + ] + +convertDimension :: CString -> Maybe(Seal Dimension) +convertDimension x = parseDimension ( cs(unsafePerformIO(peekCString (x))) ) + + +wrapDimension :: Seal Dimension -> IO(Ptr ()) +wrapDimension dim = dimensionCreate (dim) + +convertString :: CString -> Text +convertString x = cs(unsafePerformIO(peekCString (x))) + +foreign export ccall wparseDimensions :: CInt -> Ptr(CString) -> IO(Ptr ()) + +wparseDimensions :: CInt -> Ptr(CString) -> IO(Ptr ()) +wparseDimensions n dimensionsPtr = do + cDimensions <- peekArray (fromInteger(toInteger n)) dimensionsPtr + let strings = map convertString cDimensions + let dimensions = mapMaybe convertDimension cDimensions + wrappedDimensions <- newArray =<< traverse wrapDimension dimensions + let cLength = toEnum(length(dimensions)) + wrappedDimList <- dimensionListCreate wrappedDimensions cLength + return wrappedDimList + +unwrapDimension :: Ptr() -> Seal Dimension +unwrapDimension p = unsafePerformIO(dimensionGet p) + + +foreign export ccall wparseText :: CString -> Ptr() -> Ptr() -> Ptr() -> CBool -> IO(CString) +wparseText :: CString -> Ptr() -> Ptr() -> Ptr() -> CBool -> IO(CString) +wparseText cText refTimePtr localePtr dimListPtr cWithLatent = do + refTime <- duckTimeGet refTimePtr + locale <- localeGet localePtr + dimLength <- dimensionListLength dimListPtr + dimRefs <- dimensionListPtrs dimListPtr + dimPtrs <- peekArray (fromInteger(toInteger dimLength)) dimRefs + let dimensions = map unwrapDimension dimPtrs + -- dimensions <- newArray =<< traverse unwrapDimension dimPtrs + let context = Context { + referenceTime = refTime, + locale = locale + } + let options = Options { + withLatent = (cWithLatent == 1) + } + let textT = convertString(cText) + let entities = parse textT context options dimensions + let entityStr = C8.unpack(encode entities) + cEntities <- newCString entityStr + return cEntities diff --git a/duckling-ffi/LICENSE b/duckling-ffi/LICENSE new file mode 100644 index 0000000..d6988a1 --- /dev/null +++ b/duckling-ffi/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 treble.ai + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/duckling-ffi/README.md b/duckling-ffi/README.md new file mode 100644 index 0000000..07b489f --- /dev/null +++ b/duckling-ffi/README.md @@ -0,0 +1,2 @@ +# duckling-ffi +Foreign Function Interface for Facebook's duckling library diff --git a/duckling-ffi/Setup.hs b/duckling-ffi/Setup.hs new file mode 100644 index 0000000..9a994af --- /dev/null +++ b/duckling-ffi/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/duckling-ffi/duckling-ffi.cabal b/duckling-ffi/duckling-ffi.cabal new file mode 100644 index 0000000..d74aa71 --- /dev/null +++ b/duckling-ffi/duckling-ffi.cabal @@ -0,0 +1,47 @@ +name: duckling-ffi +version: 0.1.0.0 +-- synopsis: +-- description: +homepage: https://github.com/treble-ai/duckling-ffi +license: MIT +license-file: LICENSE +author: Edgar Andrés Margffoy Tuay +maintainer: edgar@treble.ai +copyright: 2020 Edgar Andrés Margffoy Tuay +category: Systems +build-type: Simple +extra-source-files: README.md +cabal-version: >=1.10 + + +library + hs-source-dirs: DucklingFFI + exposed-modules: FFI + build-depends: base + , duckling + , aeson < 2 + -- , curryrs >= 0.2.0 && < 0.3.0 + , text + , bytestring + , string-conversions + , extra + , unordered-containers + , directory + , time + , timezone-olson + , timezone-series + , filepath + default-language: Haskell2010 + -- Static linking libraries + extra-libraries: HSrts + -- Static compilation flags + ghc-options: -static -no-hs-main -staticlib -optl-static -fPIC -o libducklingffi.a + cc-options: -static + ld-options: -static + -- Dynamic compilation library + -- ghc-options: -dynamic -fPIC -no-hs-main -shared -o libducklingffi.so + other-extensions: ForeignFunctionInterface + +source-repository head + type: git + location: https://github.com/treble-ai/duckling-ffi diff --git a/duckling-ffi/stack.yaml b/duckling-ffi/stack.yaml new file mode 100644 index 0000000..3ae2e02 --- /dev/null +++ b/duckling-ffi/stack.yaml @@ -0,0 +1,15 @@ +packages: + - . +extra-deps: + - git: https://github.com/facebook/duckling.git + commit: 7520daaeba28691cda8e1b5c3d946028a28fb64b + - regex-posix-clib-2.7 + - dependent-sum-0.7.1.0@sha256:0e419237f5b86da3659772afff9cab355c0f8d5b3fdb15a5b30e673d8dc83941,2147 + - constraints-extras-0.3.1.0@sha256:c70fcf437e1d640cfd50a8eda1db47a64e49c96857e08fd0d8c438327d908ac1,1846 +resolver: lts-16.31 +allow-newer: false +install-ghc: false +system-ghc: true +skip-ghc-check: false +ghc-options: + "$everything": -fPIC diff --git a/duckling-ffi/stack.yaml.lock b/duckling-ffi/stack.yaml.lock new file mode 100644 index 0000000..faf1f2d --- /dev/null +++ b/duckling-ffi/stack.yaml.lock @@ -0,0 +1,44 @@ +# This file was autogenerated by Stack. +# You should not edit this file by hand. +# For more information, please see the documentation at: +# https://docs.haskellstack.org/en/stable/lock_files + +packages: +- completed: + commit: 7520daaeba28691cda8e1b5c3d946028a28fb64b + git: https://github.com/facebook/duckling.git + name: duckling + pantry-tree: + sha256: 5e1e7c97425977cf9d7811b845cb80484dac2b420906923949603fd996f2fb18 + size: 77724 + version: 0.2.0.0 + original: + commit: 7520daaeba28691cda8e1b5c3d946028a28fb64b + git: https://github.com/facebook/duckling.git +- completed: + hackage: regex-posix-clib-2.7@sha256:998fca06da3d719818f0691ef0b8b9b7375afeea228cb08bd9e2db53f96b0cd7,1232 + pantry-tree: + sha256: 5bf20952472d930323766d250f5fbbe3fe6f2cd3d931e5cf7b62a238ab2099ff + size: 524 + original: + hackage: regex-posix-clib-2.7 +- completed: + hackage: dependent-sum-0.7.1.0@sha256:0e419237f5b86da3659772afff9cab355c0f8d5b3fdb15a5b30e673d8dc83941,2147 + pantry-tree: + sha256: 009601e5d33e835c7a384e240c5c6ad12643bce9c54b2a0b07201daee3380e51 + size: 290 + original: + hackage: dependent-sum-0.7.1.0@sha256:0e419237f5b86da3659772afff9cab355c0f8d5b3fdb15a5b30e673d8dc83941,2147 +- completed: + hackage: constraints-extras-0.3.1.0@sha256:c70fcf437e1d640cfd50a8eda1db47a64e49c96857e08fd0d8c438327d908ac1,1846 + pantry-tree: + sha256: f29920355b21ef7a7218573591aa740b01b3d1fb3b0bbd66e2f726d091d56fb2 + size: 595 + original: + hackage: constraints-extras-0.3.1.0@sha256:c70fcf437e1d640cfd50a8eda1db47a64e49c96857e08fd0d8c438327d908ac1,1846 +snapshots: +- completed: + sha256: 637fb77049b25560622a224845b7acfe81a09fdb6a96a3c75997a10b651667f6 + size: 534126 + url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/16/31.yaml + original: lts-16.31 diff --git a/duckling/__init__.py b/duckling/__init__.py index 46dadda..11e591d 100644 --- a/duckling/__init__.py +++ b/duckling/__init__.py @@ -18,10 +18,9 @@ get_current_ref_time, parse_ref_time, parse_lang, default_locale_lang, parse_locale, parse_dimensions, parse_text, Context, Dimension, - Locale, __version__, GHC_VERSION) + Locale, __version__) __version__ -GHC_VERSION init stop load_time_zones diff --git a/packaging/0001-Allow-binaries-larger-than-32MB.patch b/packaging/0001-Allow-binaries-larger-than-32MB.patch deleted file mode 100644 index 0e37212..0000000 --- a/packaging/0001-Allow-binaries-larger-than-32MB.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 5b84342f57009f8cc1be801825a0a5925f0fcebc Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= -Date: Mon, 31 Aug 2020 11:37:56 -0500 -Subject: [PATCH] Allow binaries larger than 32MB - ---- - src/patchelf.cc | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/patchelf.cc b/src/patchelf.cc -index d0063f9..19d1483 100644 ---- a/src/patchelf.cc -+++ b/src/patchelf.cc -@@ -328,7 +328,7 @@ static FileContents readFile(std::string fileName, - size_t size = std::min(cutOff, (size_t) st.st_size); - - FileContents contents = std::make_shared>(); -- contents->reserve(size + 32 * 1024 * 1024); -+ contents->reserve(size + 64 * 1024 * 1024); - contents->resize(size, 0); - - int fd = open(fileName.c_str(), O_RDONLY); --- -2.20.1 - diff --git a/packaging/build_wheels.py b/packaging/build_wheels.py deleted file mode 100644 index 9f4fc91..0000000 --- a/packaging/build_wheels.py +++ /dev/null @@ -1,337 +0,0 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- -# Copyright (c) Treble.ai -# -# Licensed under the terms of the MIT License -# (see LICENSE.txt for details) -# ----------------------------------------------------------------------------- - -"""Helper script to package wheels and relocate binaries.""" - -# Standard library imports -import os -import io -import sys -import glob -import shutil -import zipfile -import hashlib -import platform -import subprocess -import os.path as osp -from base64 import urlsafe_b64encode - -# Third party imports -import toml -from auditwheel.lddtree import lddtree -from wheel.bdist_wheel import get_abi_tag - - -HERE = osp.dirname(osp.abspath(__file__)) -PACKAGE_ROOT = osp.dirname(HERE) -PLATFORM_ARCH = platform.machine() -PYTHON_VERSION = sys.version_info - - -def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): - """Yield pieces of data from a file-like object until EOF.""" - while True: - chunk = file.read(size) - if not chunk: - break - yield chunk - - -def rehash(path, blocksize=1 << 20): - """Return (hash, length) for path using hashlib.sha256()""" - h = hashlib.sha256() - length = 0 - with open(path, 'rb') as f: - for block in read_chunks(f, size=blocksize): - length += len(block) - h.update(block) - digest = 'sha256=' + urlsafe_b64encode( - h.digest() - ).decode('latin1').rstrip('=') - # unicode/str python2 issues - return (digest, str(length)) # type: ignore - - -def unzip_file(file, dest): - """Decompress zip `file` into directory `dest`.""" - with zipfile.ZipFile(file, 'r') as zip_ref: - zip_ref.extractall(dest) - - -def get_metadata(): - """Get version from text file and avoids importing the module.""" - with open(os.path.join(PACKAGE_ROOT, 'Cargo.toml'), 'r') as f: - data = toml.load(f) - # version = data['package']['version'] - return data['package'] - - -def is_program_installed(basename): - """ - Return program absolute path if installed in PATH. - Otherwise, return None - On macOS systems, a .app is considered installed if - it exists. - """ - if (sys.platform == 'darwin' and basename.endswith('.app') and - osp.exists(basename)): - return basename - - for path in os.environ["PATH"].split(os.pathsep): - abspath = osp.join(path, basename) - if osp.isfile(abspath): - return abspath - - -def find_program(basename): - """ - Find program in PATH and return absolute path - Try adding .exe or .bat to basename on Windows platforms - (return None if not found) - """ - names = [basename] - if os.name == 'nt': - # Windows platforms - extensions = ('.exe', '.bat', '.cmd') - if not basename.endswith(extensions): - names = [basename+ext for ext in extensions]+[basename] - for name in names: - path = is_program_installed(name) - if path: - return path - - -def patch_new_path(library_path, new_dir): - library = osp.basename(library_path) - name, *rest = library.split('.') - rest = '.'.join(rest) - hash_id = hashlib.sha256(library_path.encode('utf-8')).hexdigest()[:8] - new_name = '.'.join([name, hash_id, rest]) - return osp.join(new_dir, new_name) - - -def patch_mac(): - # Find delocate location - delocate_wheel = find_program('delocate-wheel') - delocate_list = find_program('delocate-listdeps') - if delocate_wheel is None: - raise FileNotFoundError('Delocate was not found in the system, ' - 'please install it via pip') - # Produce wheel - print('Producing wheel...') - subprocess.check_output( - [ - sys.executable, - 'setup.py', - 'bdist_wheel' - ], - cwd=PACKAGE_ROOT - ) - - package_info = get_metadata() - version = package_info['version'].replace('-', '.') - wheel_name = 'pyduckling_native-{0}-cp{1}{2}-{3}-macosx_10_15_{4}.whl'.format( - version, PYTHON_VERSION.major, PYTHON_VERSION.minor, - get_abi_tag(), PLATFORM_ARCH) - dist = osp.join(PACKAGE_ROOT, 'dist', wheel_name) - - print('Calling delocate...') - subprocess.check_output( - [ - delocate_wheel, - '-v', - dist - ], - cwd=PACKAGE_ROOT - ) - - print('Resulting libraries') - subprocess.check_output( - [ - delocate_list, - '--all', - dist - ], - cwd=PACKAGE_ROOT - ) - - -def patch_linux(): - # Get patchelf location - patchelf = find_program('patchelf') - if patchelf is None: - raise FileNotFoundError('Patchelf was not found in the system, please' - ' make sure that is available on the PATH.') - - # Produce wheel - print('Producing wheel...') - subprocess.check_output( - [ - sys.executable, - 'setup.py', - 'bdist_wheel' - ], - cwd=PACKAGE_ROOT - ) - - package_info = get_metadata() - version = package_info['version'].replace('-', '.') - wheel_name = 'pyduckling_native-{0}-cp{1}{2}-{3}-linux_{4}.whl'.format( - version, PYTHON_VERSION.major, PYTHON_VERSION.minor, - get_abi_tag(), PLATFORM_ARCH) - dist = osp.join(PACKAGE_ROOT, 'dist', wheel_name) - output_dir = osp.join(PACKAGE_ROOT, '.wheel-process') - - print(glob.glob(osp.join(PACKAGE_ROOT, 'dist', '*.whl'))) - - if osp.exists(output_dir): - shutil.rmtree(output_dir) - - os.makedirs(output_dir) - - print('Unzipping wheel...') - unzip_file(dist, output_dir) - - print('Finding ELF dependencies...') - main_binary = 'duckling.cpython-{0}-{1}-linux-gnu.so'.format( - get_abi_tag().replace('cp', ''), PLATFORM_ARCH) - output_library = osp.join(output_dir, 'duckling') - binary_path = osp.join(output_library, main_binary) - - ld_tree = lddtree(binary_path) - tree_libs = ld_tree['libs'] - - binary_queue = [(n, main_binary) for n in ld_tree['needed']] - binary_paths = {main_binary: binary_path} - binary_dependencies = {} - - while binary_queue != []: - library, parent = binary_queue.pop(0) - library_info = tree_libs[library] - print(library) - print(library_info) - if (library_info['path'].startswith('/lib') and - not library.startswith('libpcre')): - # Omit glibc/gcc/system libraries - continue - - parent_dependencies = binary_dependencies.get(parent, []) - parent_dependencies.append(library) - binary_dependencies[parent] = parent_dependencies - - if library in binary_paths: - continue - - binary_paths[library] = library_info['path'] - binary_queue += [(n, library) for n in library_info['needed']] - - print('Copying dependencies to wheel directory') - new_libraries_path = osp.join(output_dir, 'duckling.libs') - os.makedirs(new_libraries_path) - new_names = {main_binary: binary_path} - - for library in binary_paths: - if library != main_binary: - library_path = binary_paths[library] - new_library_path = patch_new_path(library_path, new_libraries_path) - print('{0} -> {1}'.format(library, new_library_path)) - shutil.copyfile(library_path, new_library_path) - new_names[library] = new_library_path - - print('Updating dependency names by new files') - for library in binary_paths: - if library != main_binary: - if library not in binary_dependencies: - continue - library_dependencies = binary_dependencies[library] - new_library_name = new_names[library] - for dep in library_dependencies: - new_dep = osp.basename(new_names[dep]) - print('{0}: {1} -> {2}'.format(library, dep, new_dep)) - subprocess.check_output( - [ - patchelf, - '--replace-needed', - dep, - new_dep, - new_library_name - ], - cwd=new_libraries_path) - - print('Updating library rpath') - subprocess.check_output( - [ - patchelf, - '--set-rpath', - "$ORIGIN", - new_library_name - ], - cwd=new_libraries_path) - - subprocess.check_output( - [ - patchelf, - '--print-rpath', - new_library_name - ], - cwd=new_libraries_path) - - print("Update main library dependencies") - library_dependencies = binary_dependencies[main_binary] - for dep in library_dependencies: - new_dep = osp.basename(new_names[dep]) - print('{0}: {1} -> {2}'.format(main_binary, dep, new_dep)) - subprocess.check_output( - [ - patchelf, - '--replace-needed', - dep, - new_dep, - main_binary - ], - cwd=output_library) - - print('Update main library rpath') - subprocess.check_output( - [ - patchelf, - '--set-rpath', - "$ORIGIN:$ORIGIN/../duckling.libs", - binary_path - ], - cwd=output_library - ) - - print('Update RECORD file in wheel') - dist_info = osp.join( - output_dir, 'pyduckling_native-{0}.dist-info'.format(version)) - record_file = osp.join(dist_info, 'RECORD') - - with open(record_file, 'w') as f: - for root, _, files in os.walk(output_dir): - for this_file in files: - full_file = osp.join(root, this_file) - rel_file = osp.relpath(full_file, output_dir) - if full_file == record_file: - f.write('{0},,\n'.format(rel_file)) - else: - digest, size = rehash(full_file) - f.write('{0},{1},{2}\n'.format(rel_file, digest, size)) - - print('Compressing wheel') - shutil.make_archive(dist, 'zip', output_dir) - os.remove(dist) - shutil.move('{0}.zip'.format(dist), dist) - shutil.rmtree(output_dir) - - -if __name__ == '__main__': - if sys.platform == 'linux': - patch_linux() - elif sys.platform == 'darwin': - patch_mac() diff --git a/pyproject.toml b/pyproject.toml index 5e3502f..f1a0053 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,16 @@ [build-system] -requires = ["maturin"] +requires = ["maturin>=1.0,<2.0"] build-backend = "maturin" [tool.maturin] -manylinux = "off" -sdist-include = ["duckling-ffi/*"] - -# [build-system] -# requires = ["setuptools", "wheel", "setuptools-rust"] +requires-dist = ["pendulum"] +classifier = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] diff --git a/src/lib.rs b/src/lib.rs index 1554469..deb73ae 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,7 +10,7 @@ // use pyo3::class::PyMappingProtocol; use pyo3::create_exception; use pyo3::exceptions; -use pyo3::gc::{PyGCProtocol, PyVisit}; +use pyo3::gc::PyVisit; use pyo3::prelude::*; use pyo3::wrap_pyfunction; use pyo3::PyTraverseError; @@ -24,7 +24,6 @@ use std::sync::Once; // Package version const VERSION: &'static str = env!("CARGO_PKG_VERSION"); -const GHC_VERSION: &'static str = env!("GHC_VERSION"); // Haskell runtime status static START_ONCE: Once = Once::new(); @@ -83,7 +82,7 @@ extern "C" { pub fn hs_exit(); } -create_exception!(pyduckling, RuntimeStoppedError, exceptions::Exception); +create_exception!(pyduckling, RuntimeStoppedError, exceptions::PyException); /// Initialize the Haskell runtime. This function is safe to call more than once, and /// will do nothing on subsequent calls. @@ -116,7 +115,7 @@ pub fn stop() -> PyResult<()> { let err = "Haskell: The GHC runtime may only be stopped once. See \ https://downloads.haskell.org/%7Eghc/latest/docs/html/users_guide\ /ffi-chap.html#id1"; - let exc = RuntimeStoppedError::py_err(err.to_string()); + let exc = RuntimeStoppedError::new_err(err.to_string()); return Err(exc); } stop_hs(); @@ -136,15 +135,16 @@ extern "C" fn stop_hs() { } /// Handle to the time zone database stored by Duckling -#[pyclass(name=TimeZoneDatabase)] +#[pyclass(name="TimeZoneDatabase")] #[derive(Debug, Clone)] pub struct TimeZoneDatabaseWrapper { ptr: *mut HaskellValue, } +unsafe impl Send for TimeZoneDatabaseWrapper {} -#[pyproto] -impl PyGCProtocol for TimeZoneDatabaseWrapper { - fn __traverse__(&self, _visit: PyVisit) -> Result<(), PyTraverseError> { +#[pymethods] +impl TimeZoneDatabaseWrapper { + fn __traverse__(&self, _visit: PyVisit<'_>) -> Result<(), PyTraverseError> { Ok(()) } @@ -163,11 +163,12 @@ impl PyGCProtocol for TimeZoneDatabaseWrapper { // } /// Handle to the time zone database stored by Duckling -#[pyclass(name=DucklingTime)] +#[pyclass(name="DucklingTime")] #[derive(Debug, Clone)] pub struct DucklingTimeWrapper { ptr: *mut HaskellValue, } +unsafe impl Send for DucklingTimeWrapper {} #[pymethods] impl DucklingTimeWrapper { @@ -182,10 +183,7 @@ impl DucklingTimeWrapper { }; Ok(string_result) } -} -#[pyproto] -impl PyGCProtocol for DucklingTimeWrapper { fn __traverse__(&self, _visit: PyVisit) -> Result<(), PyTraverseError> { Ok(()) } @@ -196,11 +194,12 @@ impl PyGCProtocol for DucklingTimeWrapper { } /// Handle to a language code stored by Duckling -#[pyclass(name=Language)] +#[pyclass(name="Language")] #[derive(Debug, Clone)] pub struct LanguageWrapper { ptr: *mut HaskellValue, } +unsafe impl Send for LanguageWrapper {} #[pymethods] impl LanguageWrapper { @@ -215,10 +214,7 @@ impl LanguageWrapper { }; Ok(string_result) } -} -#[pyproto] -impl PyGCProtocol for LanguageWrapper { fn __traverse__(&self, _visit: PyVisit) -> Result<(), PyTraverseError> { Ok(()) } @@ -229,11 +225,12 @@ impl PyGCProtocol for LanguageWrapper { } /// Handle to a locale code stored by Duckling -#[pyclass(name=Locale)] +#[pyclass(name="Locale")] #[derive(Debug, Clone)] pub struct LocaleWrapper { ptr: *mut HaskellValue, } +unsafe impl Send for LocaleWrapper {} #[pymethods] impl LocaleWrapper { @@ -248,10 +245,7 @@ impl LocaleWrapper { }; Ok(string_result) } -} -#[pyproto] -impl PyGCProtocol for LocaleWrapper { fn __traverse__(&self, _visit: PyVisit) -> Result<(), PyTraverseError> { Ok(()) } @@ -262,14 +256,15 @@ impl PyGCProtocol for LocaleWrapper { } /// Handle to a parsing dimension identifier -#[pyclass(name=Dimension)] +#[pyclass(name="Dimension")] #[derive(Debug, Clone)] pub struct DimensionWrapper { ptr: *mut HaskellValue, } +unsafe impl Send for DimensionWrapper {} -#[pyproto] -impl PyGCProtocol for DimensionWrapper { +#[pymethods] +impl DimensionWrapper { fn __traverse__(&self, _visit: PyVisit) -> Result<(), PyTraverseError> { Ok(()) } @@ -520,7 +515,6 @@ fn parse_text( #[pymodule] fn duckling(_py: Python, m: &PyModule) -> PyResult<()> { m.add("__version__", VERSION)?; - m.add("GHC_VERSION", GHC_VERSION)?; m.add_wrapped(wrap_pyfunction!(load_time_zones))?; m.add_wrapped(wrap_pyfunction!(get_current_ref_time))?; m.add_wrapped(wrap_pyfunction!(parse_ref_time))?;