From dafaf19ca73b922da2bf1032e2942b54ddc3001b Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Wed, 8 Nov 2023 01:49:51 -0500 Subject: [PATCH 01/47] Install pgvector on CI --- .github/scripts/setup_postgres.sh | 17 +++++++++++++++++ .github/workflows/test.yml | 21 ++++++++------------- 2 files changed, 25 insertions(+), 13 deletions(-) create mode 100644 .github/scripts/setup_postgres.sh diff --git a/.github/scripts/setup_postgres.sh b/.github/scripts/setup_postgres.sh new file mode 100644 index 00000000..4aa0f781 --- /dev/null +++ b/.github/scripts/setup_postgres.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -o nounset -o xtrace -o errexit -o pipefail + +PG_MAJOR_VERSION=$(pg_config --version | grep -oE '[0-9]+' | head -n 1) + +export DEBIAN_FRONTEND=something +apt-get install -y \ + postgresql-"$PG_MAJOR_VERSION"-plpython3 \ + postgresql-"$PG_MAJOR_VERSION"-pgvector \ + python3-pip + +psql "$POSTGRES_DB" -c "CREATE EXTENSION plpython3u; CREATE EXTENSION vector;" + +POSTGRES_USER_SITE=$(su --login postgres --session-command "python3 -m site --user-site") +mkdir -p "$POSTGRES_USER_SITE" +chown -R postgres "$POSTGRES_USER_SITE" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 68ab5d0e..64117fcc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,10 @@ jobs: fail-fast: false matrix: python-version: ["3.9", "3.10"] - postgres-version: ['12', '13'] + # To test different Python versions: + # - in 12-bulleye, python3 is python3.9 + # - in 12-bookworm, python3 is python3.11 + postgres-version: ['12-bullseye', '12-bookworm'] include: - tox_test: "test_py39" python-version: "3.9" @@ -34,10 +37,7 @@ jobs: services: postgres: - # we need postgres with plpython extension postgres base docker hub image did not have that - # I use this one - # image: yihong0618/postgres-plpython:${{ matrix.postgres-version }} - image: thorinschiffer/postgres-plpython:${{ matrix.postgres-version }} + image: postgres:${{ matrix.postgres-version }} env: POSTGRES_USER: ${{ env.PGUSER }} POSTGRES_PASSWORD: ${{ env.PGPASSWORD }} @@ -53,20 +53,15 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: pip + - name: Setup postgres service container + run: | + ls -la . - name: Create plpython3u language and environment run: | psql postgres://${{ env.PGUSER }}:${{ env.PGPASSWORD }}@localhost/${{ env.TESTDB }} -c "CREATE LANGUAGE plpython3u;" - name: Install Dependencies run: | pip install tox==3.25.0 - - - name: List all containers - run: docker ps -a - - - name: Install Python in PostgreSQL server container - run: | - docker exec --user 0 postgres sh -c 'apt-get update && apt-get install -y python3-pip && mkdir -p $(su --login postgres --session-command "python3 -m site --user-site") && chown -R postgres /var/lib/postgresql' - - name: Run Tests run: | export POSTGRES_PASSWORD=${{ env.PGPASSWORD }} From ffeec29bf63e026e70e086273455a606d7414fdd Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 05:24:13 -0500 Subject: [PATCH 02/47] Build custom images for server --- .github/workflows/test.yml | 29 ++++++------------- server/build.sh | 16 +++++++++++ server/postgres12-python311.Dockerfile | 4 +++ server/postgres12-python39.Dockerfile | 4 +++ tests/__init__.py | 25 +++++++++++++---- tox.ini | 39 +++++++++++++++++++++----- 6 files changed, 84 insertions(+), 33 deletions(-) create mode 100644 server/build.sh create mode 100644 server/postgres12-python311.Dockerfile create mode 100644 server/postgres12-python39.Dockerfile diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 64117fcc..430f00e1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,13 +18,11 @@ jobs: name: Test Python${{ matrix.python-version }}, PSQL${{matrix.postgres-version}} runs-on: ubuntu-latest strategy: - fail-fast: false + fail-fast: true matrix: python-version: ["3.9", "3.10"] - # To test different Python versions: - # - in 12-bulleye, python3 is python3.9 - # - in 12-bookworm, python3 is python3.11 - postgres-version: ['12-bullseye', '12-bookworm'] + python-server-version: ["39", "311"] + postgres-version: ["12-bullseye", "12-bookworm"] include: - tox_test: "test_py39" python-version: "3.9" @@ -38,13 +36,7 @@ jobs: services: postgres: image: postgres:${{ matrix.postgres-version }} - env: - POSTGRES_USER: ${{ env.PGUSER }} - POSTGRES_PASSWORD: ${{ env.PGPASSWORD }} - POSTGRES_DB: ${{ env.TESTDB }} - ports: - - 5432:5432 - options: --name postgres + options: --name postgres-image-puller steps: - uses: actions/checkout@v2 @@ -53,15 +45,12 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: pip - - name: Setup postgres service container - run: | - ls -la . - - name: Create plpython3u language and environment - run: | - psql postgres://${{ env.PGUSER }}:${{ env.PGPASSWORD }}@localhost/${{ env.TESTDB }} -c "CREATE LANGUAGE plpython3u;" - - name: Install Dependencies + + - name: Setup database server container run: | - pip install tox==3.25.0 + python3 -m pip install tox + tox -e server + - name: Run Tests run: | export POSTGRES_PASSWORD=${{ env.PGPASSWORD }} diff --git a/server/build.sh b/server/build.sh new file mode 100644 index 00000000..830eed96 --- /dev/null +++ b/server/build.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -o nounset -o xtrace -o errexit -o pipefail + +PG_MAJOR_VERSION=$(pg_config --version | grep --only-matching --extended-regexp '[0-9]+' | head -n 1) +export DEBIAN_FRONTEND=nointeractive +apt-get update +apt-get install --no-install-recommends -y \ + postgresql-plpython3-"$PG_MAJOR_VERSION" \ + postgresql-"$PG_MAJOR_VERSION"-pgvector \ + python3-pip +apt-get autoclean + +POSTGRES_USER_SITE=$(su --login postgres --session-command "python3 -m site --user-site") +mkdir --parents "$POSTGRES_USER_SITE" +chown --recursive postgres "$POSTGRES_USER_SITE" diff --git a/server/postgres12-python311.Dockerfile b/server/postgres12-python311.Dockerfile new file mode 100644 index 00000000..4089f331 --- /dev/null +++ b/server/postgres12-python311.Dockerfile @@ -0,0 +1,4 @@ +FROM postgres:12-bookworm + +COPY build.sh /tmp/build.sh +RUN bash /tmp/build.sh && rm -rf /tmp/build.sh diff --git a/server/postgres12-python39.Dockerfile b/server/postgres12-python39.Dockerfile new file mode 100644 index 00000000..b4aa66d6 --- /dev/null +++ b/server/postgres12-python39.Dockerfile @@ -0,0 +1,4 @@ +FROM postgres:12-bullseye + +COPY build.sh /tmp/build.sh +RUN bash /tmp/build.sh && rm -rf /tmp/build.sh diff --git a/tests/__init__.py b/tests/__init__.py index eac8cb50..60cbd3dd 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -10,13 +10,26 @@ def db(): # for the connection both work for GitHub Actions and concourse db = gp.database( params={ - "host": environ.get("POSTGRES_HOST", "localhost"), - "dbname": environ.get("TESTDB", "gpadmin"), - "user": environ.get("PGUSER"), - "password": environ.get("PGPASSWORD"), + "host": environ["PGHOST"], + "dbname": environ["TESTDB"], + "user": environ["PGUSER"], + "password": environ["PGPASSWORD"], } ) - db._execute("DROP SCHEMA IF EXISTS test CASCADE; CREATE SCHEMA test;", has_results=False) + db._execute( + """ + CREATE EXTENSION IF NOT EXISTS plpython3u; + CREATE EXTENSION IF NOT EXISTS vector; + """, + has_results=False + ) + db._execute( + """ + DROP SCHEMA IF EXISTS test CASCADE; + CREATE SCHEMA test; + """, + has_results=False + ) yield db db.close() @@ -24,7 +37,7 @@ def db(): @pytest.fixture() def con(): host = "localhost" - dbname = environ.get("TESTDB", "gpadmin") + dbname = environ["TESTDB"] con = f"postgresql://{host}/{dbname}" yield con diff --git a/tox.ini b/tox.ini index ff456517..d57d66bb 100644 --- a/tox.ini +++ b/tox.ini @@ -12,16 +12,41 @@ deps = -r requirements-dev.txt -r requirements-doc.txt +[server] +postgres_version = postgres12 +python_version = python39 + +postgres_user = {env:USER} +postgres_testdb = {env:USER} +postgres_password = UNUSED +postgres_host = localhost +postgres_port = 5432 + +[testenv:server] +skip_install = true +allowlist_externals = docker +commands = + docker build \ + -t pygp-server:{[server]postgres_version}-{[server]python_version} \ + -f server/{[server]postgres_version}-{[server]python_version}.Dockerfile \ + server + docker run -d \ + --name pygp-server \ + --env POSTGRES_PASSWORD={[server]postgres_password} \ + --env POSTGRES_DB={[server]postgres_testdb} \ + --publish 5432:{[server]postgres_port} \ + pygp-server:{[server]postgres_version}-{[server]python_version} + [testenv:test] deps = {[base]deps} commands = pytest {posargs} -passenv= - POSTGRES_HOST - PGPORT - PGPASSWORD - PGUSER - TESTDB +setenv = + PGHOST={env:PGHOST:{[server]postgres_host}} + PGPORT={env:PGPORT:{[server]postgres_port}} + PGPASSWORD={env:PGPASSWORD:{[server]postgres_password}} + PGUSER={env:PGUSER:{[server]postgres_user}} + TESTDB={env:TESTDB:{[server]postgres_testdb}} skip_install=true [testenv:test_py{39,310}] @@ -30,7 +55,7 @@ basepython = test_py310: python3.10 deps = {[testenv:test]deps} commands = {[testenv:test]commands} -passenv= {[testenv:test]passenv} +setenv= {[testenv:test]setenv} skip_install=true [testenv:lint] From e87bf84656211d0a85855fbbf826bf4a2f67ef64 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 06:09:02 -0500 Subject: [PATCH 03/47] Clean code --- .github/scripts/setup_postgres.sh | 17 ---------------- .github/workflows/test.yml | 33 ++++++------------------------- tox.ini | 22 ++++++++------------- 3 files changed, 14 insertions(+), 58 deletions(-) delete mode 100644 .github/scripts/setup_postgres.sh diff --git a/.github/scripts/setup_postgres.sh b/.github/scripts/setup_postgres.sh deleted file mode 100644 index 4aa0f781..00000000 --- a/.github/scripts/setup_postgres.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -o nounset -o xtrace -o errexit -o pipefail - -PG_MAJOR_VERSION=$(pg_config --version | grep -oE '[0-9]+' | head -n 1) - -export DEBIAN_FRONTEND=something -apt-get install -y \ - postgresql-"$PG_MAJOR_VERSION"-plpython3 \ - postgresql-"$PG_MAJOR_VERSION"-pgvector \ - python3-pip - -psql "$POSTGRES_DB" -c "CREATE EXTENSION plpython3u; CREATE EXTENSION vector;" - -POSTGRES_USER_SITE=$(su --login postgres --session-command "python3 -m site --user-site") -mkdir -p "$POSTGRES_USER_SITE" -chown -R postgres "$POSTGRES_USER_SITE" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 430f00e1..62993fab 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,21 +8,15 @@ on: paths-ignore: - "*.md" -env: - PGPASSWORD: postgres - TESTDB: greenplum_python_test - PGUSER: postgres - jobs: build: - name: Test Python${{ matrix.python-version }}, PSQL${{matrix.postgres-version}} + name: Test client:python${{ matrix.python-version }}, server:postgres runs-on: ubuntu-latest strategy: fail-fast: true matrix: python-version: ["3.9", "3.10"] - python-server-version: ["39", "311"] - postgres-version: ["12-bullseye", "12-bookworm"] + postgres-image: ["12-bullseye", "12-bookworm"] include: - tox_test: "test_py39" python-version: "3.9" @@ -35,38 +29,23 @@ jobs: services: postgres: - image: postgres:${{ matrix.postgres-version }} + image: postgres:${{ matrix.postgres-image }} options: --name postgres-image-puller steps: - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} + - name: Setup Python ${{ matrix.python-version }} on client uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} cache: pip - - name: Setup database server container run: | python3 -m pip install tox tox -e server - - - name: Run Tests + - name: Run tests without pickler run: | - export POSTGRES_PASSWORD=${{ env.PGPASSWORD }} - export POSTGRES_USER=${{ env.PGUSER }} - export POSTGRES_DB=${{ env.TESTDB }} tox -e ${{ matrix.tox_test }} - - - name: Install Dependencies for greenplumpython dill - # plpytyhon for postgres install to pg services - run: | - pip install pip install --target=. dill - docker cp dill $(docker ps -q):/usr/lib/${{ matrix.dill_version }}/ - rm -rf dill pip - - name: Run Tests with dill + - name: Run tests with pickler run: | - export POSTGRES_PASSWORD=${{ env.PGPASSWORD }} - export POSTGRES_USER=${{ env.PGUSER }} - export POSTGRES_DB=${{ env.TESTDB }} tox -e ${{ matrix.tox_test }} diff --git a/tox.ini b/tox.ini index d57d66bb..a108b4d6 100644 --- a/tox.ini +++ b/tox.ini @@ -4,13 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = lint, test_py{39,310}, docs, tag_ref - -[base] -deps = - -r requirements.txt - -r requirements-dev.txt - -r requirements-doc.txt +envlist = test [server] postgres_version = postgres12 @@ -27,8 +21,8 @@ skip_install = true allowlist_externals = docker commands = docker build \ - -t pygp-server:{[server]postgres_version}-{[server]python_version} \ - -f server/{[server]postgres_version}-{[server]python_version}.Dockerfile \ + --tag pygp-server:{[server]postgres_version}-{[server]python_version} \ + --file server/{[server]postgres_version}-{[server]python_version}.Dockerfile \ server docker run -d \ --name pygp-server \ @@ -38,7 +32,7 @@ commands = pygp-server:{[server]postgres_version}-{[server]python_version} [testenv:test] -deps = {[base]deps} +deps = -r requirements-dev.txt commands = pytest {posargs} setenv = @@ -60,16 +54,16 @@ skip_install=true [testenv:lint] basepython = python3.9 -deps = {[base]deps} +deps = {[testenv:test]deps} skip_install = true commands = black . --check --diff isort --profile black . -c --diff pyright -[testenv:docs] +[testenv:doc] basepython = python3.9 -deps = {[base]deps} +deps = -r requirements-doc.txt skip_install = true passenv= TAG_REF @@ -80,7 +74,7 @@ commands = [testenv:docstyle] basepython = python3.9 -deps = {[base]deps} +deps = {[testenv:docs]deps} skip_install = true commands = pydocstyle {posargs:greenplumpython} From 7f36dd9377f2ff942db5ed4bd1aefc928fd83f21 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 06:17:20 -0500 Subject: [PATCH 04/47] Fix dependencies --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index a108b4d6..575b0dee 100644 --- a/tox.ini +++ b/tox.ini @@ -32,7 +32,7 @@ commands = pygp-server:{[server]postgres_version}-{[server]python_version} [testenv:test] -deps = -r requirements-dev.txt +deps = -r requirements.txt commands = pytest {posargs} setenv = @@ -54,7 +54,7 @@ skip_install=true [testenv:lint] basepython = python3.9 -deps = {[testenv:test]deps} +deps = -r requirements-dev.txt skip_install = true commands = black . --check --diff From bc7a67eba6f265bfe3e43ba413f99a1c1d3b74c2 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 06:26:39 -0500 Subject: [PATCH 05/47] Clean again --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 62993fab..350a4aa0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: Test Postgres +name: Test on: push: @@ -10,7 +10,7 @@ on: jobs: build: - name: Test client:python${{ matrix.python-version }}, server:postgres + name: client:python${{ matrix.python-version }}, server:postgres${{ matrix.postgres-image }} runs-on: ubuntu-latest strategy: fail-fast: true @@ -34,12 +34,12 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Setup Python ${{ matrix.python-version }} on client + - name: Setup client uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} cache: pip - - name: Setup database server container + - name: Setup database server run: | python3 -m pip install tox tox -e server From 1333f38d24a7f5bdaca78597b0186f4b53a72e74 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 06:30:12 -0500 Subject: [PATCH 06/47] Print debug message --- tests/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 60cbd3dd..b17e8dc3 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,6 +7,7 @@ @pytest.fixture() def db(): + print("connecting ...") # for the connection both work for GitHub Actions and concourse db = gp.database( params={ @@ -20,15 +21,15 @@ def db(): """ CREATE EXTENSION IF NOT EXISTS plpython3u; CREATE EXTENSION IF NOT EXISTS vector; - """, - has_results=False + """, + has_results=False, ) db._execute( """ DROP SCHEMA IF EXISTS test CASCADE; CREATE SCHEMA test; - """, - has_results=False + """, + has_results=False, ) yield db db.close() From 51280b2629fb074244cae2c7ac7629006768d332 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 06:37:50 -0500 Subject: [PATCH 07/47] Fix dependencies again --- tests/__init__.py | 1 - tox.ini | 12 +++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index b17e8dc3..f52b147f 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,7 +7,6 @@ @pytest.fixture() def db(): - print("connecting ...") # for the connection both work for GitHub Actions and concourse db = gp.database( params={ diff --git a/tox.ini b/tox.ini index 575b0dee..fd288fe4 100644 --- a/tox.ini +++ b/tox.ini @@ -32,7 +32,7 @@ commands = pygp-server:{[server]postgres_version}-{[server]python_version} [testenv:test] -deps = -r requirements.txt +deps = -r requirements-dev.txt commands = pytest {posargs} setenv = @@ -41,7 +41,6 @@ setenv = PGPASSWORD={env:PGPASSWORD:{[server]postgres_password}} PGUSER={env:PGUSER:{[server]postgres_user}} TESTDB={env:TESTDB:{[server]postgres_testdb}} -skip_install=true [testenv:test_py{39,310}] basepython = @@ -50,12 +49,10 @@ basepython = deps = {[testenv:test]deps} commands = {[testenv:test]commands} setenv= {[testenv:test]setenv} -skip_install=true [testenv:lint] basepython = python3.9 -deps = -r requirements-dev.txt -skip_install = true +deps = {[testenv:test]deps} commands = black . --check --diff isort --profile black . -c --diff @@ -64,12 +61,9 @@ commands = [testenv:doc] basepython = python3.9 deps = -r requirements-doc.txt -skip_install = true passenv= - TAG_REF + TAG_REF commands = - # for doc generate we need to install this moudle that can `import greenplumpython` - pip install --force-reinstall . sphinx-build -W --keep-going -b html -j 2 -c {toxinidir}/doc/ {toxinidir}/doc/source {toxinidir}/build/doc/{env:TAG_REF:} [testenv:docstyle] From 8f935a5533280c2635e9c07182090f40efa332af Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 06:42:04 -0500 Subject: [PATCH 08/47] Specify postgres user --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index fd288fe4..57acf77c 100644 --- a/tox.ini +++ b/tox.ini @@ -26,6 +26,7 @@ commands = server docker run -d \ --name pygp-server \ + --env POSTGRES_USER=[[server]postgres_user] --env POSTGRES_PASSWORD={[server]postgres_password} \ --env POSTGRES_DB={[server]postgres_testdb} \ --publish 5432:{[server]postgres_port} \ @@ -34,7 +35,7 @@ commands = [testenv:test] deps = -r requirements-dev.txt commands = - pytest {posargs} + pytest --exitfirst {posargs} setenv = PGHOST={env:PGHOST:{[server]postgres_host}} PGPORT={env:PGPORT:{[server]postgres_port}} From 44f92e0ba3bd6e5d46ed39588669f0085ab60f4a Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 06:43:53 -0500 Subject: [PATCH 09/47] Fix bracket --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 57acf77c..70791d39 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,7 @@ commands = server docker run -d \ --name pygp-server \ - --env POSTGRES_USER=[[server]postgres_user] + --env POSTGRES_USER={[server]postgres_user} \ --env POSTGRES_PASSWORD={[server]postgres_password} \ --env POSTGRES_DB={[server]postgres_testdb} \ --publish 5432:{[server]postgres_port} \ From 52e5f0298d710c41c9bc33091ee96c280f9aa639 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 06:54:20 -0500 Subject: [PATCH 10/47] Fix permission issue --- server/build.sh | 3 ++- tox.ini | 16 +++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/server/build.sh b/server/build.sh index 830eed96..d3c35056 100644 --- a/server/build.sh +++ b/server/build.sh @@ -12,5 +12,6 @@ apt-get install --no-install-recommends -y \ apt-get autoclean POSTGRES_USER_SITE=$(su --login postgres --session-command "python3 -m site --user-site") +POSTGRES_USER_BASE=$(su --login postgres --session-command "python3 -m site --user-base") mkdir --parents "$POSTGRES_USER_SITE" -chown --recursive postgres "$POSTGRES_USER_SITE" +chown --recursive postgres "$POSTGRES_USER_BASE" diff --git a/tox.ini b/tox.ini index 70791d39..1f531c33 100644 --- a/tox.ini +++ b/tox.ini @@ -6,15 +6,17 @@ [tox] envlist = test +[client] +python_version=python3.9 + [server] postgres_version = postgres12 -python_version = python39 - postgres_user = {env:USER} postgres_testdb = {env:USER} postgres_password = UNUSED postgres_host = localhost postgres_port = 5432 +python_version = python39 [testenv:server] skip_install = true @@ -45,14 +47,14 @@ setenv = [testenv:test_py{39,310}] basepython = - test_py39: python3.9 + test_py39: {[client]python_version} test_py310: python3.10 deps = {[testenv:test]deps} commands = {[testenv:test]commands} setenv= {[testenv:test]setenv} [testenv:lint] -basepython = python3.9 +basepython = {[client]python_version} deps = {[testenv:test]deps} commands = black . --check --diff @@ -60,7 +62,7 @@ commands = pyright [testenv:doc] -basepython = python3.9 +basepython = {[client]python_version} deps = -r requirements-doc.txt passenv= TAG_REF @@ -68,8 +70,8 @@ commands = sphinx-build -W --keep-going -b html -j 2 -c {toxinidir}/doc/ {toxinidir}/doc/source {toxinidir}/build/doc/{env:TAG_REF:} [testenv:docstyle] -basepython = python3.9 -deps = {[testenv:docs]deps} +basepython = {[client]python_version} +deps = {[testenv:doc]deps} skip_install = true commands = pydocstyle {posargs:greenplumpython} From d32d2293d1eadfcfc3e32509e78b7662bb367399 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 06:58:38 -0500 Subject: [PATCH 11/47] Remove services --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 350a4aa0..a840f0cb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,14 +27,14 @@ jobs: - dill_version: python3.9 postgres-version: '13' - services: - postgres: - image: postgres:${{ matrix.postgres-image }} - options: --name postgres-image-puller + # services: + # postgres: + # image: postgres:${{ matrix.postgres-image }} + # options: --name postgres-image-puller steps: - uses: actions/checkout@v2 - - name: Setup client + - name: Setup python on client uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} From a548b1f4c0d60b3741cc74d7a2166ef71e72dacc Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 20:39:55 -0500 Subject: [PATCH 12/47] Fix errors in lint and doc --- .github/workflows/build_docs.yml | 2 +- .github/workflows/test.yml | 7 +------ tox.ini | 1 + 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml index 5fc244e6..a923eb32 100644 --- a/.github/workflows/build_docs.yml +++ b/.github/workflows/build_docs.yml @@ -37,7 +37,7 @@ jobs: - name: Build the docs run: | export TAG_REF=${{ github.ref }} - tox -e docs + tox -e doc - name: Save ref run: echo "${{ github.ref }}" >> build/doc/ref.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a840f0cb..6816a7fd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,11 +26,6 @@ jobs: postgres-version: '12' - dill_version: python3.9 postgres-version: '13' - - # services: - # postgres: - # image: postgres:${{ matrix.postgres-image }} - # options: --name postgres-image-puller steps: - uses: actions/checkout@v2 @@ -48,4 +43,4 @@ jobs: tox -e ${{ matrix.tox_test }} - name: Run tests with pickler run: | - tox -e ${{ matrix.tox_test }} + tox -e ${{ matrix.tox_test }} diff --git a/tox.ini b/tox.ini index 1f531c33..c738243d 100644 --- a/tox.ini +++ b/tox.ini @@ -56,6 +56,7 @@ setenv= {[testenv:test]setenv} [testenv:lint] basepython = {[client]python_version} deps = {[testenv:test]deps} +skip_install = true commands = black . --check --diff isort --profile black . -c --diff From db1461f4567c37b434be1f7628d27c020c723b37 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 20:52:03 -0500 Subject: [PATCH 13/47] Skip install for doc --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c738243d..866b1f56 100644 --- a/tox.ini +++ b/tox.ini @@ -65,8 +65,9 @@ commands = [testenv:doc] basepython = {[client]python_version} deps = -r requirements-doc.txt +skip_install = true passenv= - TAG_REF + TAG_REF commands = sphinx-build -W --keep-going -b html -j 2 -c {toxinidir}/doc/ {toxinidir}/doc/source {toxinidir}/build/doc/{env:TAG_REF:} From ee155c16a4474a1845571107ac9bc22cef2f7570 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 21:53:35 -0500 Subject: [PATCH 14/47] Remove skip install for doc again --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 866b1f56..e852b219 100644 --- a/tox.ini +++ b/tox.ini @@ -65,7 +65,6 @@ commands = [testenv:doc] basepython = {[client]python_version} deps = -r requirements-doc.txt -skip_install = true passenv= TAG_REF commands = From 239491f019185523e20c60e65018cb97bc31bb29 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 22:06:19 -0500 Subject: [PATCH 15/47] Add VERSION to MANIFEST.in --- MANIFEST.in | 3 +-- tox.ini | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 1d46e283..af99c68a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1 @@ -include versioneer.py -include greenplumpython/_version.py +include greenplumpython/VERSION diff --git a/tox.ini b/tox.ini index e852b219..5438ec33 100644 --- a/tox.ini +++ b/tox.ini @@ -79,7 +79,6 @@ commands = [pytest] addopts = --doctest-modules - doctest_optionflags = NORMALIZE_WHITESPACE # TODO: Enable all examples under greenplumpython testpaths = From 5d3f11e93f88e1157dfca17d2cb6329e776659ca Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 22:28:15 -0500 Subject: [PATCH 16/47] Cusomize env with tox --override --- .github/workflows/test.yml | 24 +++++++++++------------- requirements-doc.txt | 1 - 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6816a7fd..0e15cd52 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,22 +10,20 @@ on: jobs: build: - name: client:python${{ matrix.python-version }}, server:postgres${{ matrix.postgres-image }} + name: py${{ matrix.python-version }}; pg${{ matrix.postgres-version }}-py${{ matrix.postgres-python-version }} runs-on: ubuntu-latest strategy: fail-fast: true matrix: - python-version: ["3.9", "3.10"] + python-version: ["3.9", "3.11"] postgres-image: ["12-bullseye", "12-bookworm"] include: - - tox_test: "test_py39" - python-version: "3.9" - - tox_test: "test_py310" - python-version: "3.10" - - dill_version: python3.9 - postgres-version: '12' - - dill_version: python3.9 - postgres-version: '13' + - postgres-image: "12-bullseye" + postgres-version: 12 + postgres-python-version: 39 + - postgres-image: "12-bookworm" + postgres-version: 12 + postgres-python-version: 311 steps: - uses: actions/checkout@v2 @@ -37,10 +35,10 @@ jobs: - name: Setup database server run: | python3 -m pip install tox - tox -e server + tox --override server.postgres_version=${{ matrix.postgres-version }} server.python_version=${{ matrix.postgres-python-version }} -e server - name: Run tests without pickler run: | - tox -e ${{ matrix.tox_test }} + tox --override client.python_version=${{ matrix.python-version }} -e test - name: Run tests with pickler run: | - tox -e ${{ matrix.tox_test }} + tox --override client.python_version=${{ matrix.python-version }} -e test diff --git a/requirements-doc.txt b/requirements-doc.txt index 0e955023..a3e8b5e3 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -1,4 +1,3 @@ --r requirements.txt sphinx==5.1.1 sphinx_rtd_theme==1.0.0 nbsphinx From bf0af2e74438f683d22e1d7c6a3bb35c3890b9f0 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 22:31:09 -0500 Subject: [PATCH 17/47] Fix tox errors --- .github/workflows/test.yml | 2 +- tox.ini | 36 ++++++++++++++++++------------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0e15cd52..bc6f40ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: - name: Setup database server run: | python3 -m pip install tox - tox --override server.postgres_version=${{ matrix.postgres-version }} server.python_version=${{ matrix.postgres-python-version }} -e server + tox --override server.version=${{ matrix.postgres-version }} --override server.python_version=${{ matrix.postgres-python-version }} -e server - name: Run tests without pickler run: | tox --override client.python_version=${{ matrix.python-version }} -e test diff --git a/tox.ini b/tox.ini index 5438ec33..c2c2a533 100644 --- a/tox.ini +++ b/tox.ini @@ -10,12 +10,12 @@ envlist = test python_version=python3.9 [server] -postgres_version = postgres12 -postgres_user = {env:USER} -postgres_testdb = {env:USER} -postgres_password = UNUSED -postgres_host = localhost -postgres_port = 5432 +version = postgres12 +user = {env:USER} +testdb = {env:USER} +password = UNUSED +host = localhost +port = 5432 python_version = python39 [testenv:server] @@ -23,27 +23,27 @@ skip_install = true allowlist_externals = docker commands = docker build \ - --tag pygp-server:{[server]postgres_version}-{[server]python_version} \ - --file server/{[server]postgres_version}-{[server]python_version}.Dockerfile \ + --tag pygp-server:{[server]version}-{[server]python_version} \ + --file server/{[server]version}-{[server]python_version}.Dockerfile \ server docker run -d \ --name pygp-server \ - --env POSTGRES_USER={[server]postgres_user} \ - --env POSTGRES_PASSWORD={[server]postgres_password} \ - --env POSTGRES_DB={[server]postgres_testdb} \ - --publish 5432:{[server]postgres_port} \ - pygp-server:{[server]postgres_version}-{[server]python_version} + --env USER={[server]user} \ + --env PASSWORD={[server]password} \ + --env DB={[server]testdb} \ + --publish 5432:{[server]port} \ + pygp-server:{[server]version}-{[server]python_version} [testenv:test] deps = -r requirements-dev.txt commands = pytest --exitfirst {posargs} setenv = - PGHOST={env:PGHOST:{[server]postgres_host}} - PGPORT={env:PGPORT:{[server]postgres_port}} - PGPASSWORD={env:PGPASSWORD:{[server]postgres_password}} - PGUSER={env:PGUSER:{[server]postgres_user}} - TESTDB={env:TESTDB:{[server]postgres_testdb}} + PGHOST={env:PGHOST:{[server]host}} + PGPORT={env:PGPORT:{[server]port}} + PGPASSWORD={env:PGPASSWORD:{[server]password}} + PGUSER={env:PGUSER:{[server]user}} + TESTDB={env:TESTDB:{[server]testdb}} [testenv:test_py{39,310}] basepython = From dcb95950711f501cde0d36a84955aa2cbcae8eba Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 22:35:28 -0500 Subject: [PATCH 18/47] Fix variable errors --- .github/workflows/test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bc6f40ed..779897ce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ on: jobs: build: - name: py${{ matrix.python-version }}; pg${{ matrix.postgres-version }}-py${{ matrix.postgres-python-version }} + name: python${{ matrix.python-version }}; ${{ matrix.server-version }}-${{ matrix.server-python-version }} runs-on: ubuntu-latest strategy: fail-fast: true @@ -19,11 +19,11 @@ jobs: postgres-image: ["12-bullseye", "12-bookworm"] include: - postgres-image: "12-bullseye" - postgres-version: 12 - postgres-python-version: 39 + server-version: postgres12 + server-python-version: python39 - postgres-image: "12-bookworm" - postgres-version: 12 - postgres-python-version: 311 + server-version: postgres12 + server-python-version: python311 steps: - uses: actions/checkout@v2 @@ -35,7 +35,7 @@ jobs: - name: Setup database server run: | python3 -m pip install tox - tox --override server.version=${{ matrix.postgres-version }} --override server.python_version=${{ matrix.postgres-python-version }} -e server + tox --override server.version=${{ matrix.server-version }} --override server.python_version=${{ matrix.server-python-version }} -e server - name: Run tests without pickler run: | tox --override client.python_version=${{ matrix.python-version }} -e test From c6866b907c0b3a9035268ee5958288ade19392de Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 23:14:30 -0500 Subject: [PATCH 19/47] Fix error on docker env --- tox.ini | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index c2c2a533..1c138932 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ python_version=python3.9 version = postgres12 user = {env:USER} testdb = {env:USER} -password = UNUSED +password = password host = localhost port = 5432 python_version = python39 @@ -28,13 +28,14 @@ commands = server docker run -d \ --name pygp-server \ - --env USER={[server]user} \ - --env PASSWORD={[server]password} \ - --env DB={[server]testdb} \ + --env POSTFRES_USER={[server]user} \ + --env POSTFRES_PASSWORD={[server]password} \ + --env POSTFRES_DB={[server]testdb} \ --publish 5432:{[server]port} \ pygp-server:{[server]version}-{[server]python_version} [testenv:test] +basepython = {[client]python_version} deps = -r requirements-dev.txt commands = pytest --exitfirst {posargs} From 1bd2eed491a022176fc356e7c243ee1d4b435dd2 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 23:20:47 -0500 Subject: [PATCH 20/47] Fix tox variable issue again --- .github/workflows/test.yml | 10 +++++----- requirements-dev.txt | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 779897ce..b4b69887 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,12 +16,12 @@ jobs: fail-fast: true matrix: python-version: ["3.9", "3.11"] - postgres-image: ["12-bullseye", "12-bookworm"] + server-base-image: ["postgres:12-bullseye", "postgres:12-bookworm"] include: - - postgres-image: "12-bullseye" + - server-base-image: "postgres:12-bullseye" server-version: postgres12 server-python-version: python39 - - postgres-image: "12-bookworm" + - server-base-image: "postgres:12-bookworm" server-version: postgres12 server-python-version: python311 @@ -38,7 +38,7 @@ jobs: tox --override server.version=${{ matrix.server-version }} --override server.python_version=${{ matrix.server-python-version }} -e server - name: Run tests without pickler run: | - tox --override client.python_version=${{ matrix.python-version }} -e test + tox --override client.python_version=python${{ matrix.python-version }} -e test - name: Run tests with pickler run: | - tox --override client.python_version=${{ matrix.python-version }} -e test + tox --override client.python_version=python${{ matrix.python-version }} -e test diff --git a/requirements-dev.txt b/requirements-dev.txt index e69460b5..172deb32 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,5 @@ black==22.3.0 isort==5.10.1 pytest==7.0.1 -pglast==3.10 pyright==1.1.250 pandas From d0c315c3478caa35a3ab8add79c9ecbf25cd9648 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 23:26:23 -0500 Subject: [PATCH 21/47] Remove basepython --- .github/workflows/test.yml | 6 +++--- tox.ini | 12 ------------ 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b4b69887..16bbbfda 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: true matrix: - python-version: ["3.9", "3.11"] + python-version: ["3.9", "3.12"] server-base-image: ["postgres:12-bullseye", "postgres:12-bookworm"] include: - server-base-image: "postgres:12-bullseye" @@ -38,7 +38,7 @@ jobs: tox --override server.version=${{ matrix.server-version }} --override server.python_version=${{ matrix.server-python-version }} -e server - name: Run tests without pickler run: | - tox --override client.python_version=python${{ matrix.python-version }} -e test + tox -e test - name: Run tests with pickler run: | - tox --override client.python_version=python${{ matrix.python-version }} -e test + tox -e test diff --git a/tox.ini b/tox.ini index 1c138932..a0340dbc 100644 --- a/tox.ini +++ b/tox.ini @@ -6,9 +6,6 @@ [tox] envlist = test -[client] -python_version=python3.9 - [server] version = postgres12 user = {env:USER} @@ -35,7 +32,6 @@ commands = pygp-server:{[server]version}-{[server]python_version} [testenv:test] -basepython = {[client]python_version} deps = -r requirements-dev.txt commands = pytest --exitfirst {posargs} @@ -46,14 +42,6 @@ setenv = PGUSER={env:PGUSER:{[server]user}} TESTDB={env:TESTDB:{[server]testdb}} -[testenv:test_py{39,310}] -basepython = - test_py39: {[client]python_version} - test_py310: python3.10 -deps = {[testenv:test]deps} -commands = {[testenv:test]commands} -setenv= {[testenv:test]setenv} - [testenv:lint] basepython = {[client]python_version} deps = {[testenv:test]deps} From b1a58334b7ae983fb5f78abb40b3012c1a8f8c84 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 23:28:50 -0500 Subject: [PATCH 22/47] Remove all basepython --- .github/workflows/test.yml | 2 +- tox.ini | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 16bbbfda..e05da7b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,7 +34,7 @@ jobs: cache: pip - name: Setup database server run: | - python3 -m pip install tox + python3 -m pip install tox~=4.11 tox --override server.version=${{ matrix.server-version }} --override server.python_version=${{ matrix.server-python-version }} -e server - name: Run tests without pickler run: | diff --git a/tox.ini b/tox.ini index a0340dbc..a72998cb 100644 --- a/tox.ini +++ b/tox.ini @@ -43,7 +43,6 @@ setenv = TESTDB={env:TESTDB:{[server]testdb}} [testenv:lint] -basepython = {[client]python_version} deps = {[testenv:test]deps} skip_install = true commands = @@ -52,7 +51,6 @@ commands = pyright [testenv:doc] -basepython = {[client]python_version} deps = -r requirements-doc.txt passenv= TAG_REF @@ -60,7 +58,6 @@ commands = sphinx-build -W --keep-going -b html -j 2 -c {toxinidir}/doc/ {toxinidir}/doc/source {toxinidir}/build/doc/{env:TAG_REF:} [testenv:docstyle] -basepython = {[client]python_version} deps = {[testenv:doc]deps} skip_install = true commands = From 54ddd1bcb0e770581301e2aa9e10c75b48af338b Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 23:30:06 -0500 Subject: [PATCH 23/47] Fix typo --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index a72998cb..e37973cf 100644 --- a/tox.ini +++ b/tox.ini @@ -25,9 +25,9 @@ commands = server docker run -d \ --name pygp-server \ - --env POSTFRES_USER={[server]user} \ - --env POSTFRES_PASSWORD={[server]password} \ - --env POSTFRES_DB={[server]testdb} \ + --env POSTGRES_USER={[server]user} \ + --env POSTGRES_PASSWORD={[server]password} \ + --env POSTGRES_DB={[server]testdb} \ --publish 5432:{[server]port} \ pygp-server:{[server]version}-{[server]python_version} From f49617819d1b3537793217d8893ec868c0e9ed36 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Thu, 9 Nov 2023 23:34:44 -0500 Subject: [PATCH 24/47] Chnage back to 3.11 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e05da7b0..00de29d0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: true matrix: - python-version: ["3.9", "3.12"] + python-version: ["3.9", "3.11"] server-base-image: ["postgres:12-bullseye", "postgres:12-bookworm"] include: - server-base-image: "postgres:12-bullseye" From 78eb7d33fdb0380230f5263d50e79100a6e1a25c Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Fri, 10 Nov 2023 03:12:34 -0500 Subject: [PATCH 25/47] Fix bug on get_type_hints() --- greenplumpython/experimental/file.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/greenplumpython/experimental/file.py b/greenplumpython/experimental/file.py index ac8d49bd..f291dffd 100644 --- a/greenplumpython/experimental/file.py +++ b/greenplumpython/experimental/file.py @@ -79,11 +79,9 @@ def _archive_and_upload(tmp_archive_name: str, files: list[str], db: gp.Database def _from_files(_, files: list[str], parser: NormalFunction, db: gp.Database) -> gp.DataFrame: tmp_archive_name = f"tar_{uuid.uuid4().hex}" _archive_and_upload(tmp_archive_name, files, db) - func_sig = inspect.signature(parser.unwrap()) - result_members = get_type_hints(func_sig.return_annotation) return db.apply( lambda: parser(_extract_files(tmp_archive_name, "files")), - expand=len(result_members) == 0, + expand=True, ) From a059f2133a6771af2eda4ae83ea0d4ec1f02d884 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sat, 11 Nov 2023 06:58:13 -0500 Subject: [PATCH 26/47] Add option to use dill --- .github/workflows/test.yml | 4 ++-- BUILD.md | 10 ---------- README.dev.md | 10 ++-------- concourse/test.sh | 3 +-- tests/conftest.py | 16 ++++++++++++++++ tests/test_use_pickler.py | 7 +++++++ tox.ini | 4 +++- 7 files changed, 31 insertions(+), 23 deletions(-) delete mode 100644 BUILD.md create mode 100644 tests/conftest.py create mode 100644 tests/test_use_pickler.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 00de29d0..d66ddb1d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,7 +38,7 @@ jobs: tox --override server.version=${{ matrix.server-version }} --override server.python_version=${{ matrix.server-python-version }} -e server - name: Run tests without pickler run: | - tox -e test + tox --override server.use_pickler=false -e test - name: Run tests with pickler run: | - tox -e test + tox --override server.use_pickler=true -e test diff --git a/BUILD.md b/BUILD.md deleted file mode 100644 index 67c9e6f2..00000000 --- a/BUILD.md +++ /dev/null @@ -1,10 +0,0 @@ -# Install dependency -Make sure you have the latest versions of setuptools and wheel installed. - -python3 -m pip install --user --upgrade setuptools wheel - -# Generating distribution archives - -python3 setup.py sdist bdist_wheel - - diff --git a/README.dev.md b/README.dev.md index 23eb5229..f5828b03 100644 --- a/README.dev.md +++ b/README.dev.md @@ -4,10 +4,10 @@ ### [tox](https://tox.wiki) -Install with `pip`: +We use tox as the task runner. Tox can be installed with ``` -pip install tox +python3 -m pip install tox ``` Install with `brew` on macOS: @@ -42,12 +42,6 @@ Test with the default python version: tox -e test ``` -Test with specified officially supported version: - -``` -tox -e test_py39 -``` - Run a specified test case(s): ``` diff --git a/concourse/test.sh b/concourse/test.sh index 3c43ad6f..cdc39d62 100755 --- a/concourse/test.sh +++ b/concourse/test.sh @@ -47,8 +47,7 @@ _main() { unset PYTHONPATH unset PYTHONHOME python3.9 -m pip install tox - python3.9 -m pip install . - tox -e test_py39 + tox -e test popd } diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..62c92346 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,16 @@ +import pytest + + +def pytest_addoption(parser: pytest.Parser): + parser.addini( + "server_use_pickler", + type="bool", + default=False, + help="Use pickler to deserialize UDFs on server.", + ) + + +@pytest.fixture(scope="session") +def server_use_pickler(pytestconfig: pytest.Config) -> bool: + val: bool = pytestconfig.getini("server_use_pickler") + return val diff --git a/tests/test_use_pickler.py b/tests/test_use_pickler.py new file mode 100644 index 00000000..84af0e90 --- /dev/null +++ b/tests/test_use_pickler.py @@ -0,0 +1,7 @@ +import pytest + +import greenplumpython as gp + + +def test_option(server_use_pickler: bool): + assert server_use_pickler == True diff --git a/tox.ini b/tox.ini index e37973cf..7cd70178 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,7 @@ password = password host = localhost port = 5432 python_version = python39 +use_pickler = true [testenv:server] skip_install = true @@ -34,7 +35,7 @@ commands = [testenv:test] deps = -r requirements-dev.txt commands = - pytest --exitfirst {posargs} + pytest --exitfirst --override-ini=server_use_pickler={[server]use_pickler} {posargs} setenv = PGHOST={env:PGHOST:{[server]host}} PGPORT={env:PGPORT:{[server]port}} @@ -63,6 +64,7 @@ skip_install = true commands = pydocstyle {posargs:greenplumpython} +# Not handled by Tox [pytest] addopts = --doctest-modules doctest_optionflags = NORMALIZE_WHITESPACE From db28bb9847199b3abf19a8b9f7ed42f5debc11fd Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 03:43:25 -0500 Subject: [PATCH 27/47] Use tox-docker to run server --- server/build.sh | 3 ++ server/initdb.sh | 9 +++++ server/postgres12-python311.Dockerfile | 4 +-- server/postgres12-python39.Dockerfile | 4 +-- tox.ini | 47 ++++++++++++++------------ 5 files changed, 41 insertions(+), 26 deletions(-) create mode 100644 server/initdb.sh diff --git a/server/build.sh b/server/build.sh index d3c35056..e66d4da7 100644 --- a/server/build.sh +++ b/server/build.sh @@ -15,3 +15,6 @@ POSTGRES_USER_SITE=$(su --login postgres --session-command "python3 -m site --us POSTGRES_USER_BASE=$(su --login postgres --session-command "python3 -m site --user-base") mkdir --parents "$POSTGRES_USER_SITE" chown --recursive postgres "$POSTGRES_USER_BASE" + +cp /tmp/initdb.sh /docker-entrypoint-initdb.d +chown postgres /docker-entrypoint-initdb.d/* diff --git a/server/initdb.sh b/server/initdb.sh new file mode 100644 index 00000000..1d4275c7 --- /dev/null +++ b/server/initdb.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -o nounset -o xtrace -o errexit -o pipefail + +{ + echo "logging_collector = ON" + echo "log_statement = 'all'" + echo "log_destination = 'csvlog'" +} >>"$PGDATA"/postgresql.conf \ No newline at end of file diff --git a/server/postgres12-python311.Dockerfile b/server/postgres12-python311.Dockerfile index 4089f331..baab0b21 100644 --- a/server/postgres12-python311.Dockerfile +++ b/server/postgres12-python311.Dockerfile @@ -1,4 +1,4 @@ FROM postgres:12-bookworm -COPY build.sh /tmp/build.sh -RUN bash /tmp/build.sh && rm -rf /tmp/build.sh +COPY build.sh initdb.sh /tmp/ +RUN bash /tmp/build.sh diff --git a/server/postgres12-python39.Dockerfile b/server/postgres12-python39.Dockerfile index b4aa66d6..642128f3 100644 --- a/server/postgres12-python39.Dockerfile +++ b/server/postgres12-python39.Dockerfile @@ -1,4 +1,4 @@ FROM postgres:12-bullseye -COPY build.sh /tmp/build.sh -RUN bash /tmp/build.sh && rm -rf /tmp/build.sh +COPY build.sh initdb.sh /tmp/ +RUN bash /tmp/build.sh diff --git a/tox.ini b/tox.ini index 7cd70178..65448a74 100644 --- a/tox.ini +++ b/tox.ini @@ -12,36 +12,39 @@ user = {env:USER} testdb = {env:USER} password = password host = localhost -port = 5432 +port = 65432 python_version = python39 use_pickler = true -[testenv:server] -skip_install = true -allowlist_externals = docker -commands = - docker build \ - --tag pygp-server:{[server]version}-{[server]python_version} \ - --file server/{[server]version}-{[server]python_version}.Dockerfile \ - server - docker run -d \ - --name pygp-server \ - --env POSTGRES_USER={[server]user} \ - --env POSTGRES_PASSWORD={[server]password} \ - --env POSTGRES_DB={[server]testdb} \ - --publish 5432:{[server]port} \ - pygp-server:{[server]version}-{[server]python_version} +[docker:server] +dockerfile = server/{[server]version}-{[server]python_version}.Dockerfile +environment = + POSTGRES_USER={[server]user} + POSTGRES_PASSWORD={[server]password} + POSTGRES_DB={[server]testdb} +ports = + {[server]port}:5432/tcp +healthcheck_cmd = + PGPASSWORD=$POSTGRES_PASSWORD psql \ + --user=$POSTGRES_USER --dbname=$POSTGRES_DB \ + --host=localhost --single-transaction \ + --command="SELECT version();" +healthcheck_timeout = 1 +healthcheck_retries = 30 +healthcheck_interval = 1 +healthcheck_start_period = 1 -[testenv:test] +[testenv:remote] +docker = server deps = -r requirements-dev.txt commands = pytest --exitfirst --override-ini=server_use_pickler={[server]use_pickler} {posargs} setenv = - PGHOST={env:PGHOST:{[server]host}} - PGPORT={env:PGPORT:{[server]port}} - PGPASSWORD={env:PGPASSWORD:{[server]password}} - PGUSER={env:PGUSER:{[server]user}} - TESTDB={env:TESTDB:{[server]testdb}} + PGHOST={[server]host} + PGPORT={[server]port} + PGPASSWORD={[server]password} + PGUSER={[server]user} + TESTDB={[server]testdb} [testenv:lint] deps = {[testenv:test]deps} From 5a9cfbf73bca08e20aff65e30511ce195880cce5 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 04:02:45 -0500 Subject: [PATCH 28/47] Remove password --- .github/workflows/test.yml | 7 ++----- tests/__init__.py | 10 +++++----- tox.ini | 21 +++++++++++++++------ 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d66ddb1d..29d29298 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,13 +32,10 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: pip - - name: Setup database server - run: | - python3 -m pip install tox~=4.11 - tox --override server.version=${{ matrix.server-version }} --override server.python_version=${{ matrix.server-python-version }} -e server - name: Run tests without pickler run: | - tox --override server.use_pickler=false -e test + python3 -m pip install tox~=4.11 tox-docker~=4.1 + tox --override -e test - name: Run tests with pickler run: | tox --override server.use_pickler=true -e test diff --git a/tests/__init__.py b/tests/__init__.py index f52b147f..8acc22aa 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -10,10 +10,10 @@ def db(): # for the connection both work for GitHub Actions and concourse db = gp.database( params={ - "host": environ["PGHOST"], - "dbname": environ["TESTDB"], - "user": environ["PGUSER"], - "password": environ["PGPASSWORD"], + "host": environ.get("PGHOST", "localhost"), + "dbname": environ.get("TESTDB", environ.get("USER")), + "user": environ.get("PGUSER", environ.get("USER")), + "password": environ.get("PGPASSWORD"), } ) db._execute( @@ -37,7 +37,7 @@ def db(): @pytest.fixture() def con(): host = "localhost" - dbname = environ["TESTDB"] + dbname = environ.get("TESTDB", environ.get("USER")) con = f"postgresql://{host}/{dbname}" yield con diff --git a/tox.ini b/tox.ini index 65448a74..0d8019fd 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,6 @@ envlist = test version = postgres12 user = {env:USER} testdb = {env:USER} -password = password host = localhost port = 65432 python_version = python39 @@ -20,21 +19,31 @@ use_pickler = true dockerfile = server/{[server]version}-{[server]python_version}.Dockerfile environment = POSTGRES_USER={[server]user} - POSTGRES_PASSWORD={[server]password} + POSTGRES_HOST_AUTH_METHOD=trust POSTGRES_DB={[server]testdb} ports = {[server]port}:5432/tcp healthcheck_cmd = - PGPASSWORD=$POSTGRES_PASSWORD psql \ - --user=$POSTGRES_USER --dbname=$POSTGRES_DB \ - --host=localhost --single-transaction \ + psql --single-transaction \ + --user=$POSTGRES_USER --dbname=$POSTGRES_DB --host=localhost \ --command="SELECT version();" healthcheck_timeout = 1 healthcheck_retries = 30 healthcheck_interval = 1 healthcheck_start_period = 1 -[testenv:remote] +[testenv:test-remote] +docker = server +deps = -r requirements-dev.txt +commands = + pytest --exitfirst --override-ini=server_use_pickler={[server]use_pickler} {posargs} +setenv = + PGHOST={[server]host} + PGPORT={[server]port} + PGUSER={[server]user} + TESTDB={[server]testdb} + +[testenv:test] docker = server deps = -r requirements-dev.txt commands = From a56034c95050fff6b072703a8c15dc17a727f90a Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 04:31:11 -0500 Subject: [PATCH 29/47] Move healthcheck to dockerfile --- .github/workflows/test.yml | 4 ++-- server/initdb.sh | 2 +- server/postgres12-python311.Dockerfile | 7 +++++++ server/postgres12-python39.Dockerfile | 7 +++++++ tests/conftest.py | 2 +- tox.ini | 27 +++++++------------------- 6 files changed, 25 insertions(+), 24 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 29d29298..67274c57 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: - name: Run tests without pickler run: | python3 -m pip install tox~=4.11 tox-docker~=4.1 - tox --override -e test + tox --override=docker:server.dockerfile=server/${{ matrix.server-version }}-${{ matrix.server-python-version }}.Dockerfile -e test-remote -- --override-ini=server_use_pickler=false - name: Run tests with pickler run: | - tox --override server.use_pickler=true -e test + tox --override=docker:server.dockerfile=server/${{ matrix.server-version }}-${{ matrix.server-python-version }}.Dockerfile -e test-remote -- --override-ini=server_use_pickler=true diff --git a/server/initdb.sh b/server/initdb.sh index 1d4275c7..f02bd7ba 100644 --- a/server/initdb.sh +++ b/server/initdb.sh @@ -6,4 +6,4 @@ set -o nounset -o xtrace -o errexit -o pipefail echo "logging_collector = ON" echo "log_statement = 'all'" echo "log_destination = 'csvlog'" -} >>"$PGDATA"/postgresql.conf \ No newline at end of file +} >>"$PGDATA"/postgresql.conf diff --git a/server/postgres12-python311.Dockerfile b/server/postgres12-python311.Dockerfile index baab0b21..58962428 100644 --- a/server/postgres12-python311.Dockerfile +++ b/server/postgres12-python311.Dockerfile @@ -2,3 +2,10 @@ FROM postgres:12-bookworm COPY build.sh initdb.sh /tmp/ RUN bash /tmp/build.sh + +HEALTHCHECK --interval=1s --timeout=1s --start-period=1s --retries=30 CMD psql \ + --single-transaction \ + --user=$POSTGRES_USER \ + --dbname=$POSTGRES_DB \ + --host=localhost \ + --command="SELECT version();" diff --git a/server/postgres12-python39.Dockerfile b/server/postgres12-python39.Dockerfile index 642128f3..39de51f7 100644 --- a/server/postgres12-python39.Dockerfile +++ b/server/postgres12-python39.Dockerfile @@ -2,3 +2,10 @@ FROM postgres:12-bullseye COPY build.sh initdb.sh /tmp/ RUN bash /tmp/build.sh + +HEALTHCHECK --interval=1s --timeout=1s --start-period=1s --retries=30 CMD psql \ + --single-transaction \ + --user=$POSTGRES_USER \ + --dbname=$POSTGRES_DB \ + --host=localhost \ + --command="SELECT version();" diff --git a/tests/conftest.py b/tests/conftest.py index 62c92346..67ac45cb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,7 @@ def pytest_addoption(parser: pytest.Parser): parser.addini( "server_use_pickler", type="bool", - default=False, + default=True, help="Use pickler to deserialize UDFs on server.", ) diff --git a/tox.ini b/tox.ini index 0d8019fd..3e65baa0 100644 --- a/tox.ini +++ b/tox.ini @@ -7,36 +7,25 @@ envlist = test [server] -version = postgres12 user = {env:USER} testdb = {env:USER} host = localhost port = 65432 -python_version = python39 -use_pickler = true [docker:server] -dockerfile = server/{[server]version}-{[server]python_version}.Dockerfile +dockerfile = server/postgres12-python39.Dockerfile environment = POSTGRES_USER={[server]user} POSTGRES_HOST_AUTH_METHOD=trust POSTGRES_DB={[server]testdb} ports = {[server]port}:5432/tcp -healthcheck_cmd = - psql --single-transaction \ - --user=$POSTGRES_USER --dbname=$POSTGRES_DB --host=localhost \ - --command="SELECT version();" -healthcheck_timeout = 1 -healthcheck_retries = 30 -healthcheck_interval = 1 -healthcheck_start_period = 1 [testenv:test-remote] docker = server deps = -r requirements-dev.txt commands = - pytest --exitfirst --override-ini=server_use_pickler={[server]use_pickler} {posargs} + pytest --exitfirst {posargs} setenv = PGHOST={[server]host} PGPORT={[server]port} @@ -44,16 +33,14 @@ setenv = TESTDB={[server]testdb} [testenv:test] -docker = server deps = -r requirements-dev.txt commands = - pytest --exitfirst --override-ini=server_use_pickler={[server]use_pickler} {posargs} + pytest --exitfirst {posargs} setenv = - PGHOST={[server]host} - PGPORT={[server]port} - PGPASSWORD={[server]password} - PGUSER={[server]user} - TESTDB={[server]testdb} + PGHOST={env:PGHOST:localhost} + PGPORT={env:PGPORT:5432} + PGUSER={env:PGUSER:{env:USER}} + TESTDB={env:TESTDB:{env:USER}} [testenv:lint] deps = {[testenv:test]deps} From 3e744d00e48700daaa9a8ff030e4404f6d15bebc Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 04:45:45 -0500 Subject: [PATCH 30/47] Create Python venv --- server/initdb.sh | 3 +++ tox.ini | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/server/initdb.sh b/server/initdb.sh index f02bd7ba..0400aeff 100644 --- a/server/initdb.sh +++ b/server/initdb.sh @@ -7,3 +7,6 @@ set -o nounset -o xtrace -o errexit -o pipefail echo "log_statement = 'all'" echo "log_destination = 'csvlog'" } >>"$PGDATA"/postgresql.conf + +python3 -m venv "$HOME"/venv +source "$HOME"/venv/bin/activate diff --git a/tox.ini b/tox.ini index 3e65baa0..ce0318f0 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,10 @@ # in multiple virtualenvs. This configuration file will run the # test suite on all supported python versions. To use it, "pip install tox" # and then run "tox" from this directory. +# +# NOTE: +# - tox will only override or substitute sections registered as envs, while +# - Any section can be used as a source of substitution. [tox] envlist = test @@ -63,7 +67,6 @@ skip_install = true commands = pydocstyle {posargs:greenplumpython} -# Not handled by Tox [pytest] addopts = --doctest-modules doctest_optionflags = NORMALIZE_WHITESPACE From b1117a564083143c52d3479b4e9f229506c88988 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 05:02:39 -0500 Subject: [PATCH 31/47] Install python3-venv --- server/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/build.sh b/server/build.sh index e66d4da7..573d968f 100644 --- a/server/build.sh +++ b/server/build.sh @@ -8,7 +8,8 @@ apt-get update apt-get install --no-install-recommends -y \ postgresql-plpython3-"$PG_MAJOR_VERSION" \ postgresql-"$PG_MAJOR_VERSION"-pgvector \ - python3-pip + python3-pip \ + python3-venv apt-get autoclean POSTGRES_USER_SITE=$(su --login postgres --session-command "python3 -m site --user-site") From 5b19116ade62e6beaa19150c115c3154c3a41b9c Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 05:19:01 -0500 Subject: [PATCH 32/47] Ignore test cases for pickler --- .github/workflows/test.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 67274c57..23bf2b17 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,9 +33,18 @@ jobs: python-version: ${{ matrix.python-version }} cache: pip - name: Run tests without pickler - run: | - python3 -m pip install tox~=4.11 tox-docker~=4.1 - tox --override=docker:server.dockerfile=server/${{ matrix.server-version }}-${{ matrix.server-python-version }}.Dockerfile -e test-remote -- --override-ini=server_use_pickler=false + run: >- + python3 -m pip install tox~=4.11 tox-docker~=4.1 && + tox + --override=docker:server.dockerfile=server/${{ matrix.server-version }}-${{ matrix.server-python-version }}.Dockerfile + -e test-remote + -- + --override-ini=server_use_pickler=false + --ignore=tests/test_use_pickler.py - name: Run tests with pickler - run: | - tox --override=docker:server.dockerfile=server/${{ matrix.server-version }}-${{ matrix.server-python-version }}.Dockerfile -e test-remote -- --override-ini=server_use_pickler=true + run: >- + tox + --override=docker:server.dockerfile=server/${{ matrix.server-version }}-${{ matrix.server-python-version }}.Dockerfile + -e test-remote + -- + --override-ini=server_use_pickler=true From 51214ce4fadf9b656b8963a5e56bcdd37de9e886 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 05:25:06 -0500 Subject: [PATCH 33/47] Fix missing newline --- .github/workflows/test.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 23bf2b17..2d10ab38 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,18 +33,18 @@ jobs: python-version: ${{ matrix.python-version }} cache: pip - name: Run tests without pickler - run: >- - python3 -m pip install tox~=4.11 tox-docker~=4.1 && - tox - --override=docker:server.dockerfile=server/${{ matrix.server-version }}-${{ matrix.server-python-version }}.Dockerfile - -e test-remote - -- - --override-ini=server_use_pickler=false - --ignore=tests/test_use_pickler.py + run: | + python3 -m pip install tox~=4.11 tox-docker~=4.1 && \ + tox \ + --override=docker:server.dockerfile=server/${{ matrix.server-version }}-${{ matrix.server-python-version }}.Dockerfile \ + -e test-remote \ + -- \ + --override-ini=server_use_pickler=false \ + --ignore=tests/test_use_pickler.py \ - name: Run tests with pickler - run: >- - tox - --override=docker:server.dockerfile=server/${{ matrix.server-version }}-${{ matrix.server-python-version }}.Dockerfile - -e test-remote - -- - --override-ini=server_use_pickler=true + run: | + tox \ + --override=docker:server.dockerfile=server/${{ matrix.server-version }}-${{ matrix.server-python-version }}.Dockerfile \ + -e test-remote \ + -- \ + --override-ini=server_use_pickler=true \ From 2d8fefbb79fcf4837aa20cde77ce085ec3b638b2 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 21:06:11 -0500 Subject: [PATCH 34/47] Install dill when using pickler --- greenplumpython/dataframe.py | 14 +++++++------- tests/__init__.py | 34 ++++++++++++++++++++++++++-------- tests/test_binaryExpr.py | 2 +- tests/test_column.py | 2 +- tests/test_dataframe.py | 14 +++++++------- tests/test_use_pickler.py | 18 +++++++++++++++++- 6 files changed, 59 insertions(+), 25 deletions(-) diff --git a/greenplumpython/dataframe.py b/greenplumpython/dataframe.py index 461be747..bad6757c 100644 --- a/greenplumpython/dataframe.py +++ b/greenplumpython/dataframe.py @@ -798,9 +798,9 @@ def refresh(self) -> "DataFrame": .. highlight:: python .. code-block:: Python - >>> cursor.execute("DROP TABLE IF EXISTS const_table;") + >>> cursor.execute("DROP TABLE IF EXISTS t_refresh;") >>> nums = db.create_dataframe(rows=[(i,) for i in range(5)], column_names=["num"]) - >>> df = nums.save_as("const_table", column_names=["num"], temp=False).order_by("num")[:] + >>> df = nums.save_as("t_refresh", column_names=["num"], temp=False).order_by("num")[:] >>> df ----- num @@ -812,7 +812,7 @@ def refresh(self) -> "DataFrame": 4 ----- (5 rows) - >>> cursor.execute("INSERT INTO const_table(num) VALUES (5);") + >>> cursor.execute("INSERT INTO t_refresh(num) VALUES (5);") >>> df ----- num @@ -836,7 +836,7 @@ def refresh(self) -> "DataFrame": 5 ----- (6 rows) - >>> cursor.execute("DROP TABLE const_table;") + >>> cursor.execute("DROP TABLE t_refresh;") Note: `cursor` is a predefined `Psycopg Cursor `_ @@ -913,7 +913,7 @@ def save_as( .. code-block:: Python >>> nums = db.create_dataframe(rows=[(i,) for i in range(5)], column_names=["num"]) - >>> df = nums.save_as("const_table", column_names=["num"], temp=True) + >>> df = nums.save_as("t_saved", column_names=["num"], temp=True) >>> df.order_by("num")[:] ----- num @@ -925,8 +925,8 @@ def save_as( 4 ----- (5 rows) - >>> const_table = db.create_dataframe(table_name="const_table") - >>> const_table.order_by("num")[:] + >>> t_saved = db.create_dataframe(table_name="t_saved") + >>> t_saved.order_by("num")[:] ----- num ----- diff --git a/tests/__init__.py b/tests/__init__.py index 8acc22aa..f8cbce7f 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -4,9 +4,30 @@ import greenplumpython as gp - -@pytest.fixture() -def db(): +# NOTE: This UDF must **not** depend on picklers, such as dill. +@gp.create_function +def pip_install(requirements: str) -> str: + import subprocess as sp + import sys + + assert sys.executable, "Python executable is required to install packages." + cmd = [ + sys.executable, + "-m", + "pip", + "install", + "--requirement", + "/dev/stdin", + ] + try: + output = sp.check_output(cmd, text=True, stderr=sp.STDOUT, input=requirements) + return output + except sp.CalledProcessError as e: + raise Exception(e.stdout) + + +@pytest.fixture(scope="session") +def db(server_use_pickler: bool): # for the connection both work for GitHub Actions and concourse db = gp.database( params={ @@ -20,16 +41,13 @@ def db(): """ CREATE EXTENSION IF NOT EXISTS plpython3u; CREATE EXTENSION IF NOT EXISTS vector; - """, - has_results=False, - ) - db._execute( - """ DROP SCHEMA IF EXISTS test CASCADE; CREATE SCHEMA test; """, has_results=False, ) + if server_use_pickler: + print(db.apply(lambda: pip_install("dill==0.3.6"))) yield db db.close() diff --git a/tests/test_binaryExpr.py b/tests/test_binaryExpr.py index 58c9955c..0e3acaa0 100644 --- a/tests/test_binaryExpr.py +++ b/tests/test_binaryExpr.py @@ -49,7 +49,7 @@ def test_expr_bin_equal_2expr(db: gp.Database): def test_expr_bin_equal_bool(db: gp.Database): rows = [(True,), (False,), (False,), (True,)] t = db.create_dataframe(rows=rows, column_names=["id"]).save_as( - "temp1", temp=True, column_names=["id"] + "temp6", temp=True, column_names=["id"] ) b5: Callable[[gp.DataFrame], gp.Expr] = lambda t: t["id"] == True assert len(list(t[b5])) == 2 diff --git a/tests/test_column.py b/tests/test_column.py index d0639eb0..a355d0b9 100644 --- a/tests/test_column.py +++ b/tests/test_column.py @@ -10,7 +10,7 @@ def dataframe(db: gp.Database): rows = [(1,), (2,), (3,)] t = db.create_dataframe(rows=rows, column_names=["id"]) - t = t.save_as("const_dataframe", temp=True, column_names=["id"]) + t = t.save_as(temp=True, column_names=["id"]) return t diff --git a/tests/test_dataframe.py b/tests/test_dataframe.py index 9a82bc08..c7d8e8e3 100644 --- a/tests/test_dataframe.py +++ b/tests/test_dataframe.py @@ -16,7 +16,7 @@ def t(db: gp.Database): def test_const_dataframe_columns(db: gp.Database): columns = {"a": [1, 2, 3], "b": [1, 2, 3]} t = db.create_dataframe(columns=columns) - t = t.save_as("const_dataframe", column_names=["a", "b"], temp=True) + t = t.save_as(column_names=["a", "b"], temp=True) assert sorted([tuple(row.values()) for row in t]) == [(1, 1), (2, 2), (3, 3)] assert list(next(iter(t)).keys()) == ["a", "b"] @@ -25,7 +25,7 @@ def test_const_dataframe_columns(db: gp.Database): def test_const_dataframe_rows(db: gp.Database): rows = [(1,), (2,), (3,)] t = db.create_dataframe(rows=rows, column_names=["id"]) - t = t.save_as("const_dataframe", column_names=["id"], temp=True) + t = t.save_as(column_names=["id"], temp=True) assert sorted([tuple(row.values()) for row in t]) == sorted(rows) assert len(next(iter(t)).keys()) == 1 @@ -291,10 +291,10 @@ def test_iter_break(db: gp.Database): def test_table_refresh_add_rows(db: gp.Database): nums = db.create_dataframe(rows=[(i,) for i in range(10)], column_names=["num"]) - t = nums.save_as("const_table", column_names=["num"], temp=True) + t = nums.save_as(column_names=["num"], temp=True) assert len(list(t)) == 10 - db._execute("INSERT INTO const_table(num) VALUES (10);", has_results=False) + db._execute(f"INSERT INTO {t._qualified_table_name}(num) VALUES (10);", has_results=False) assert len(list(t)) == 10 t.refresh() @@ -304,12 +304,12 @@ def test_table_refresh_add_rows(db: gp.Database): def test_table_refresh_add_columns(db: gp.Database): # Initial DataFrame nums = db.create_dataframe(rows=[(i,) for i in range(10)], column_names=["num"]) - t = nums.save_as("const_table", column_names=["num"], temp=True) + t = nums.save_as(column_names=["num"], temp=True) assert list(next(iter(t)).keys()) == ["num"] assert sorted(row["num"] for row in t) == sorted(list(range(10))) # Add a new column - db._execute("ALTER TABLE const_table ADD num_copy int;", has_results=False) + db._execute(f"ALTER TABLE {t._qualified_table_name} ADD num_copy int;", has_results=False) assert len(next(iter(t)).keys()) == 1 for row in next(iter(t)).keys(): assert row == "num" @@ -320,7 +320,7 @@ def test_table_refresh_add_columns(db: gp.Database): assert row["num_copy"] is None # Update column - db._execute("UPDATE const_table SET num_copy=num;", has_results=False) + db._execute(f"UPDATE {t._qualified_table_name} SET num_copy=num;", has_results=False) for row in t: assert row["num_copy"] is None # Refresh DataFrame contents diff --git a/tests/test_use_pickler.py b/tests/test_use_pickler.py index 84af0e90..dd4a9a80 100644 --- a/tests/test_use_pickler.py +++ b/tests/test_use_pickler.py @@ -2,6 +2,22 @@ import greenplumpython as gp +from tests import db -def test_option(server_use_pickler: bool): + +def test_pickler_option(server_use_pickler: bool): assert server_use_pickler == True + +from dataclasses import dataclass + +def test_pickler_outside_class(db: gp.Database): + @dataclass + class Int: + val: int + + @gp.create_function + def add_one(i: int) -> Int: + return Int(i + 1) + + for row in db.apply(lambda: add_one(1), expand=True): + assert row["val"] == 2 From 06e876c7a27874bef755888cf9628709a8e3f333 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 21:08:38 -0500 Subject: [PATCH 35/47] Fix lint error --- tests/__init__.py | 1 + tests/test_use_pickler.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index f8cbce7f..fb496f01 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -4,6 +4,7 @@ import greenplumpython as gp + # NOTE: This UDF must **not** depend on picklers, such as dill. @gp.create_function def pip_install(requirements: str) -> str: diff --git a/tests/test_use_pickler.py b/tests/test_use_pickler.py index dd4a9a80..06297804 100644 --- a/tests/test_use_pickler.py +++ b/tests/test_use_pickler.py @@ -1,15 +1,16 @@ import pytest import greenplumpython as gp - from tests import db def test_pickler_option(server_use_pickler: bool): assert server_use_pickler == True + from dataclasses import dataclass + def test_pickler_outside_class(db: gp.Database): @dataclass class Int: @@ -18,6 +19,6 @@ class Int: @gp.create_function def add_one(i: int) -> Int: return Int(i + 1) - + for row in db.apply(lambda: add_one(1), expand=True): assert row["val"] == 2 From 2939c7b4c9d13138e45b27adca38b720d0be1de0 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 21:41:29 -0500 Subject: [PATCH 36/47] Make dill test optional --- .github/workflows/test.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2d10ab38..6459ce44 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ on: jobs: build: - name: python${{ matrix.python-version }}; ${{ matrix.server-version }}-${{ matrix.server-python-version }} + name: python${{ matrix.python-version }}; ${{ matrix.server-version }}-python3${{ matrix.server-python3-version }} runs-on: ubuntu-latest strategy: fail-fast: true @@ -20,10 +20,10 @@ jobs: include: - server-base-image: "postgres:12-bullseye" server-version: postgres12 - server-python-version: python39 + server-python3-version: 9 - server-base-image: "postgres:12-bookworm" server-version: postgres12 - server-python-version: python311 + server-python3-version: 11 steps: - uses: actions/checkout@v2 @@ -36,15 +36,16 @@ jobs: run: | python3 -m pip install tox~=4.11 tox-docker~=4.1 && \ tox \ - --override=docker:server.dockerfile=server/${{ matrix.server-version }}-${{ matrix.server-python-version }}.Dockerfile \ + --override=docker:server.dockerfile=server/${{ matrix.server-version }}-python3${{ matrix.server-python3-version }}.Dockerfile \ -e test-remote \ -- \ --override-ini=server_use_pickler=false \ --ignore=tests/test_use_pickler.py \ - name: Run tests with pickler + if: matrix.python-version == 3.${{ matrix.server-python3-version }} run: | tox \ - --override=docker:server.dockerfile=server/${{ matrix.server-version }}-${{ matrix.server-python-version }}.Dockerfile \ + --override=docker:server.dockerfile=server/${{ matrix.server-version }}-python3${{ matrix.server-python3-version }}.Dockerfile \ -e test-remote \ -- \ --override-ini=server_use_pickler=true \ From 7f8a0abf0cf7b78b36a7b12bcea241b51d68d296 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 21:50:57 -0500 Subject: [PATCH 37/47] Try to fix if in jobs --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6459ce44..69d97e41 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: --override-ini=server_use_pickler=false \ --ignore=tests/test_use_pickler.py \ - name: Run tests with pickler - if: matrix.python-version == 3.${{ matrix.server-python3-version }} + if: matrix.python-version == '3.${{ matrix.server-python3-version }}' run: | tox \ --override=docker:server.dockerfile=server/${{ matrix.server-version }}-python3${{ matrix.server-python3-version }}.Dockerfile \ From 6ed522cc669511bacf6fbc63a03568711cc89c42 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 21:53:40 -0500 Subject: [PATCH 38/47] Test if --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 69d97e41..5abc19b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: --override-ini=server_use_pickler=false \ --ignore=tests/test_use_pickler.py \ - name: Run tests with pickler - if: matrix.python-version == '3.${{ matrix.server-python3-version }}' + if: 1 == 2 run: | tox \ --override=docker:server.dockerfile=server/${{ matrix.server-version }}-python3${{ matrix.server-python3-version }}.Dockerfile \ From e992dc31f0b409b0045bbdd0074c2ea17288a605 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 21:55:50 -0500 Subject: [PATCH 39/47] Retry if --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5abc19b4..69d97e41 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: --override-ini=server_use_pickler=false \ --ignore=tests/test_use_pickler.py \ - name: Run tests with pickler - if: 1 == 2 + if: matrix.python-version == '3.${{ matrix.server-python3-version }}' run: | tox \ --override=docker:server.dockerfile=server/${{ matrix.server-version }}-python3${{ matrix.server-python3-version }}.Dockerfile \ From 893995d085fd8f6fa8bab30157ad731b1c430ff6 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 22:05:45 -0500 Subject: [PATCH 40/47] Try fix if again --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 69d97e41..e8cd21b8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: --override-ini=server_use_pickler=false \ --ignore=tests/test_use_pickler.py \ - name: Run tests with pickler - if: matrix.python-version == '3.${{ matrix.server-python3-version }}' + if: ${{ matrix.python-version == '3.${{ matrix.server-python3-version }}' }} run: | tox \ --override=docker:server.dockerfile=server/${{ matrix.server-version }}-python3${{ matrix.server-python3-version }}.Dockerfile \ From 98566f5b83e6ed1e9fb9e1f90b19ecd0b94ed8a3 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 22:09:45 -0500 Subject: [PATCH 41/47] Use format() in if --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e8cd21b8..0731e587 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: --override-ini=server_use_pickler=false \ --ignore=tests/test_use_pickler.py \ - name: Run tests with pickler - if: ${{ matrix.python-version == '3.${{ matrix.server-python3-version }}' }} + if: ${{ matrix.python-version == format("3.{}", matrix.server-python3-version) }} run: | tox \ --override=docker:server.dockerfile=server/${{ matrix.server-version }}-python3${{ matrix.server-python3-version }}.Dockerfile \ From 58a0cfe2e4eb1e42320bd56e91d288b25d46d784 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 22:16:06 -0500 Subject: [PATCH 42/47] Use single quote --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0731e587..4cc2f8ac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,10 +20,10 @@ jobs: include: - server-base-image: "postgres:12-bullseye" server-version: postgres12 - server-python3-version: 9 + server-python3-version: "9" - server-base-image: "postgres:12-bookworm" server-version: postgres12 - server-python3-version: 11 + server-python3-version: "11" steps: - uses: actions/checkout@v2 @@ -41,8 +41,8 @@ jobs: -- \ --override-ini=server_use_pickler=false \ --ignore=tests/test_use_pickler.py \ - - name: Run tests with pickler - if: ${{ matrix.python-version == format("3.{}", matrix.server-python3-version) }} + - name: Run tests with pickler if python versions match + if: ${{ matrix.python-version == format('3.{}', matrix.server-python3-version) }} run: | tox \ --override=docker:server.dockerfile=server/${{ matrix.server-version }}-python3${{ matrix.server-python3-version }}.Dockerfile \ From 1daa435e430151964fc0f9c5b9e56c0d55421f71 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 22:18:41 -0500 Subject: [PATCH 43/47] FIx format string --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4cc2f8ac..52fb82a9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: --override-ini=server_use_pickler=false \ --ignore=tests/test_use_pickler.py \ - name: Run tests with pickler if python versions match - if: ${{ matrix.python-version == format('3.{}', matrix.server-python3-version) }} + if: ${{ matrix.python-version == format('3.{0}', matrix.server-python3-version) }} run: | tox \ --override=docker:server.dockerfile=server/${{ matrix.server-version }}-python3${{ matrix.server-python3-version }}.Dockerfile \ From ecfde8abf1cf93bc2b4d2f6cbd571b686f70be52 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 22:23:13 -0500 Subject: [PATCH 44/47] Clean github actions config --- .github/workflows/test.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 52fb82a9..c1e90ed5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,20 +10,18 @@ on: jobs: build: - name: python${{ matrix.python-version }}; ${{ matrix.server-version }}-python3${{ matrix.server-python3-version }} + name: python${{ matrix.python-version }}; ${{ server }} runs-on: ubuntu-latest strategy: fail-fast: true matrix: python-version: ["3.9", "3.11"] - server-base-image: ["postgres:12-bullseye", "postgres:12-bookworm"] + server: ["postgres12-python39", "postgres12-python311"] include: - server-base-image: "postgres:12-bullseye" - server-version: postgres12 - server-python3-version: "9" + server-python-version: "3.9" - server-base-image: "postgres:12-bookworm" - server-version: postgres12 - server-python3-version: "11" + server-python-version: "3.11" steps: - uses: actions/checkout@v2 @@ -36,16 +34,16 @@ jobs: run: | python3 -m pip install tox~=4.11 tox-docker~=4.1 && \ tox \ - --override=docker:server.dockerfile=server/${{ matrix.server-version }}-python3${{ matrix.server-python3-version }}.Dockerfile \ + --override=docker:server.dockerfile=server/${{ server }}.Dockerfile \ -e test-remote \ -- \ --override-ini=server_use_pickler=false \ --ignore=tests/test_use_pickler.py \ - name: Run tests with pickler if python versions match - if: ${{ matrix.python-version == format('3.{0}', matrix.server-python3-version) }} + if: ${{ matrix.python-version == matrix.server-python-version }} run: | tox \ - --override=docker:server.dockerfile=server/${{ matrix.server-version }}-python3${{ matrix.server-python3-version }}.Dockerfile \ + --override=docker:server.dockerfile=server/${{ server }}.Dockerfile \ -e test-remote \ -- \ --override-ini=server_use_pickler=true \ From c0b021e7ecbc52bc22f33d79248b1ff406031e50 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 22:26:15 -0500 Subject: [PATCH 45/47] Fix error in variables --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c1e90ed5..f68f9f2e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ on: jobs: build: - name: python${{ matrix.python-version }}; ${{ server }} + name: python${{ matrix.python-version }}; ${{ matrix.server }} runs-on: ubuntu-latest strategy: fail-fast: true @@ -34,7 +34,7 @@ jobs: run: | python3 -m pip install tox~=4.11 tox-docker~=4.1 && \ tox \ - --override=docker:server.dockerfile=server/${{ server }}.Dockerfile \ + --override=docker:server.dockerfile=server/${{ matrix.server }}.Dockerfile \ -e test-remote \ -- \ --override-ini=server_use_pickler=false \ @@ -43,7 +43,7 @@ jobs: if: ${{ matrix.python-version == matrix.server-python-version }} run: | tox \ - --override=docker:server.dockerfile=server/${{ server }}.Dockerfile \ + --override=docker:server.dockerfile=server/${{ matrix.server }}.Dockerfile \ -e test-remote \ -- \ --override-ini=server_use_pickler=true \ From 32e71725e192080e9c97c01182de42e6f1350f27 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 22:30:25 -0500 Subject: [PATCH 46/47] Fix wrong server config --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f68f9f2e..46e9e38f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,9 +18,9 @@ jobs: python-version: ["3.9", "3.11"] server: ["postgres12-python39", "postgres12-python311"] include: - - server-base-image: "postgres:12-bullseye" + - server: "postgres12-python39" server-python-version: "3.9" - - server-base-image: "postgres:12-bookworm" + - server: "postgres12-python311" server-python-version: "3.11" steps: From 226e623f633ef3ba69dc36e9c88c32c9547b6049 Mon Sep 17 00:00:00 2001 From: Xuebin Su Date: Sun, 12 Nov 2023 22:43:17 -0500 Subject: [PATCH 47/47] Add doc --- .github/workflows/test.yml | 4 ++-- README.dev.md | 8 +++++++- tox.ini | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 46e9e38f..41bca741 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: python3 -m pip install tox~=4.11 tox-docker~=4.1 && \ tox \ --override=docker:server.dockerfile=server/${{ matrix.server }}.Dockerfile \ - -e test-remote \ + -e test-container \ -- \ --override-ini=server_use_pickler=false \ --ignore=tests/test_use_pickler.py \ @@ -44,6 +44,6 @@ jobs: run: | tox \ --override=docker:server.dockerfile=server/${{ matrix.server }}.Dockerfile \ - -e test-remote \ + -e test-container \ -- \ --override-ini=server_use_pickler=true \ diff --git a/README.dev.md b/README.dev.md index f5828b03..2278c706 100644 --- a/README.dev.md +++ b/README.dev.md @@ -36,12 +36,18 @@ The tests will create connection to the Greenplum cluster. So the `PGPORT` needs export PGPORT=6000 ``` -Test with the default python version: +Test with the default python version and a local database server: ``` tox -e test ``` +To run tests against a database server in container: +``` +python3 -m pip install tox-docker +tox -e test-container +``` + Run a specified test case(s): ``` diff --git a/tox.ini b/tox.ini index ce0318f0..711f73c9 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ environment = ports = {[server]port}:5432/tcp -[testenv:test-remote] +[testenv:test-container] docker = server deps = -r requirements-dev.txt commands =