Merge branch 'release/1.9.1'

.github/workflows/pythonpackage.yml

@@ -0,0 +1,55 @@
+name: Test PyGeoprocessing
+
+on: [push, pull_request]
+
+jobs:
+  Test:
+    runs-on: ${{ matrix.os }}
+    env:
+        PACKAGES: "shapely numpy scipy cython rtree!=0.9.1 pytest flake8 gdal<3"
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        python-version: [3.6, 3.7]
+        os: [ubuntu-16.04, windows-latest, macos-latest]
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: setup-conda
+      uses: s-weigand/[email protected]
+      with:
+        update-conda: false
+        python-version: ${{ matrix.python-version }}
+        conda-channels: defaults
+    - name: Install dependencies
+      shell: bash
+      run: conda upgrade -y pip setuptools
+
+    - name: Install PyGeoprocessing (Windows)
+      if: matrix.os == 'windows-latest'
+      env:
+          PIP_EXTRA_INDEX_URL: "http://pypi.naturalcapitalproject.org/simple/"
+          PIP_TRUSTED_HOST: "pypi.naturalcapitalproject.org"
+          PIP_PREFER_BINARY: 1
+      shell: bash
+      # Replace numpy and scipy with PyPI versions to circumvent import issue.
+      # https://stackoverflow.com/a/37110747/299084
+      run: |
+          $CONDA/python -m pip install $PACKAGES
+          $CONDA/python setup.py install
+
+    - name: Install PyGeoprocessing (Linux, Mac)
+      if: matrix.os != 'windows-latest'
+      run: |
+          conda install $PACKAGES
+          python setup.py install
+
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: pytest

.travis.yml

@@ -0,0 +1,12 @@
+language: python
+
+os: linux
+
+services:
+  - docker
+
+install:
+  - docker pull natcap/pygeoprocessing-test:0.0.1
+
+script:
+  - docker run -it --rm -v `pwd`:/usr/local/pygeoprocessing natcap/pygeoprocessing-test:0.0.1

HISTORY.rst

@@ -1,6 +1,38 @@
 Release History
 ===============
 
+1.9.1 (2019-12-19)
+------------------
+* Fixed a compilation issue on Mac OS X Catalina related to the compilation
+  of a template in the file iteration component of the out-of-core percentile
+  function.
+* Resolved a compilation issue on Mac OS X (Mavericks and later) where
+  pygeoprocessing would not compile unless some additional compiler and linker
+  flags were provided.  These are now accounted for in the package's compilation
+  steps in ``setup.py``.
+* ``pygeoprocessing.symbolic.evaluate_raster_calculator_expression``
+  no longer depends on ``sympy`` for its expression evaluation.
+
+1.9.0 (2019-10-22)
+------------------
+* Fixed a memory error issue that could occur on multiple flow direction flow
+  accumulation calculations.
+* Added PEP518-compatible build dependencies to ``pyproject.toml``, which has
+  been added to source distributions of pygeoprocessing.
+* Added an out-of-core high performance raster percentile function at
+  pygeoprocessing.raster_band_percentile.
+* Added a ``pygeoprocessing.symbolic.evaluate_raster_calculator_expression``
+  function that can parse a string expression and a map of symbols to
+  calculate a raster calculation operation.
+* Modified ``pygeoprocessing.routing.fill_pits`` to create a single band
+  raster of the input DEM raster/path band rather than a copy of the input
+  DEM raster irrespective of the band number.
+* Fixed an issue that would cause the bounding box generated by
+  ```transform_bounding_box`` to flip the min and max coordinates in some
+  spatial contexts.
+* Defaulting compression algorithm back to LZW as ZSTD is not a widely
+  implemented compression standard for geotiffs.
+
 1.8.0 (2019-08-12)
 ------------------
 * Added a ``'numpy_type'`` field to the result of ``get_raster_info`` that
@@ -14,6 +46,12 @@ Release History
   ``raster_driver_creation_tuple``. This allows the caller to create any type
   of ``GDAL`` writable driver along with the option list associated with that
   driver.
+* Added a ``'numpy_type'`` field to the result of ``get_raster_info`` that
+  contains the equivalent numpy datatype of the GDAL type in the raster. This
+  includes functionality differentate between the unsigned and signed
+  ``gdal.GDT_Byte`` vs. ``numpy.int8`` and ``numpy.uint8``.
+* Changed default compression routine for GeoTIFFs to ZSTD (thanks Facebook
+  https://facebook.github.io/zstd/).
 * Added a ``'file_list'`` key to the dictionary returned by
   ``get_raster_info`` and ``get_vector_info`` that contains a list of all the
   files associated with that GIS object. The first parameter of these lists
@@ -66,9 +104,6 @@ Release History
   or string ids. In all cases changing ``layer_index`` to ``layer_id`` in
   the functions: ``get_vector_info``, ``reproject_vector``, ``warp_raster``,
   ``rasterize``, ``calculate_disjoint_polygon_set``, and ``mask_raster``.
-* Added a ``gtiff_creation_options`` parameter to ``reclassify_raster`` to be
-  consistent with the rest of the raster creation functions in
-  PyGeoprocessing.
 
 1.6.1 (2019-02-13)
 ------------------

MANIFEST.in

@@ -1,6 +1,7 @@
 include HISTORY.rst
 include LICENSE.txt
 include README.rst
+include pyproject.toml
 global-include *.pyx
 global-include *.pxd
 global-include *.h

pygeoprocessing-docker/Dockerfile

@@ -0,0 +1,38 @@
+FROM conda/miniconda3
+
+# this builds natcap/pygeoprocessing-test, should be run in the same directory
+# as the pygeoprocessing repository root like so:
+# docker run -it --rm -v `cwd`:/usr/local/pygeoprocessing natcap/pygeoprocessing-test:[TAG]
+# the [TAG] is defined in the .travis.yml file
+
+USER root
+RUN apt-get update \
+&& apt-get install -y \
+    build-essential \
+    git \
+    libspatialindex-c4v5 \
+    mercurial \
+&& rm -rf /var/lib/apt/lists/*
+
+SHELL ["/bin/bash", "-c"]
+RUN conda create -y --name py37 python=3.7 && conda clean -a -y
+RUN conda run -v -n py37 conda install -c conda-forge gdal=2.4.1
+RUN conda run -v -n py37 pip install --no-cache-dir \
+    cython \
+    ecoshard==0.3.1 \
+    pytest \
+    pytest-cov \
+    mock \
+    numpy \
+    retrying \
+    rtree \
+    scipy \
+    setuptools-scm \
+    shapely \
+    sympy \
+    taskgraph && conda clean -a -y
+
+RUN conda init bash && echo "source activate py37" > ~/.bashrc
+WORKDIR /usr/local/pygeoprocessing
+RUN echo "python setup.py install && pytest -xv tests" > /usr/local/run_pgp_tests.sh && chmod a+x /usr/local/run_pgp_tests.sh
+ENTRYPOINT /bin/bash -xci /usr/local/run_pgp_tests.sh

pygeoprocessing-docker/VERSION

@@ -0,0 +1 @@
+0.0.1

pyproject.toml

@@ -0,0 +1,8 @@
+[build-system]
+# these are the minimum packages needed in order to execute the pygeoprocessing build.
+# Setuptools, wheel are from pep508.
+# NOTE: GDAL is *not* require here because the compiled cython module will
+# dynamically import GDAL via python's import system.  This behavior means
+# that we can provide a much easier build experience so long as GDAL is
+# available at runtime.
+requires = ['setuptools', 'wheel', 'setuptools_scm', 'numpy', 'cython']

requirements.txt

@@ -3,7 +3,7 @@
 # This file records the packages and requirements needed in order for
 # pygeoprocessing to work as expected.
 
-GDAL>=2.0,<3.0
+GDAL>=2.2.0
 numpy>=1.10.1
 scipy>=0.14.1,!=0.19.1
 Shapely>=1.6.4

setup.py

@@ -1,4 +1,6 @@
 """setup.py module for PyGeoprocessing."""
+import platform
+
 import numpy
 from setuptools.extension import Extension
 from setuptools import setup
@@ -12,6 +14,14 @@
     requirements='\n'.join(['    ' + r for r in _REQUIREMENTS]))
 LONG_DESCRIPTION += '\n' + open('HISTORY.rst').read() + '\n'
 
+# Since OSX Mavericks, the stdlib has been renamed.  So if we're on OSX, we
+# need to be sure to define which standard c++ library to use.  I don't have
+# access to a pre-Mavericks mac, so hopefully this won't break on someone's
+# older system.  Tested and it works on Mac OSX Catalina.
+compiler_and_linker_args = []
+if platform.system() == 'Darwin':
+    compiler_and_linker_args = ['-stdlib=libc++']
+
 setup(
     name='pygeoprocessing',
     description="PyGeoprocessing: Geoprocessing routines for GIS",
@@ -57,6 +67,8 @@
             include_dirs=[
                 numpy.get_include(),
                 'src/pygeoprocessing/routing'],
+            extra_compile_args=compiler_and_linker_args,
+            extra_link_args=compiler_and_linker_args,
             language="c++",
         ),
         Extension(
@@ -65,14 +77,18 @@
             include_dirs=[
                 numpy.get_include(),
                 'src/pygeoprocessing/routing'],
+            extra_compile_args=compiler_and_linker_args,
+            extra_link_args=compiler_and_linker_args,
             language="c++",
         ),
         Extension(
-             "pygeoprocessing.geoprocessing_core",
-             sources=[
-                 'src/pygeoprocessing/geoprocessing_core.pyx'],
-             include_dirs=[numpy.get_include()],
-             language="c++"
+            "pygeoprocessing.geoprocessing_core",
+            sources=[
+                'src/pygeoprocessing/geoprocessing_core.pyx'],
+            include_dirs=[numpy.get_include()],
+            extra_compile_args=compiler_and_linker_args,
+            extra_link_args=compiler_and_linker_args,
+            language="c++"
         ),
     ]
 )