Merge pull request #473 from asfadmin/test

Breaking Change: Optionalize Sklearn Package

.github/workflows/run-pytest.yml

@@ -11,8 +11,8 @@ jobs:
       - name: Install dependancies
         run: |
           python3 -m pip install --upgrade pip
-          python3 -m pip install -r requirements.txt
+          python3 -m pip install .[extras,test]
 
       - name: Run the Test Suite
         run: |
-          pytest -n auto .
+          python3 -m pytest -n auto .

CHANGELOG.md

@@ -8,6 +8,13 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ------
 
+## [2.0.0](https://github.com/asfadmin/Discovery-WKTUtils/compare/v1.1.6...v2.0.0)
+
+### Changed
+- Makes `sklearn` and `requests` optional dependency (required for `RepairWKT.py` functionality). Installable via `python3 -m pip install WKTUtils[extras]`
+
+------
+
 ## [1.1.6](https://github.com/asfadmin/Discovery-WKTUtils/compare/v1.1.5...v1.1.6)
 
 ### Fixed

WKTUtils/Input.py

@@ -1,5 +1,4 @@
 import dateparser
-import re
 from geomet import wkt, InvalidGeoJSONException
 
 

WKTUtils/RepairWKT.py

@@ -1,27 +1,35 @@
+import re
 from .asf_env import get_config
 from .Input import parse_wkt_util
-
 import logging
-import requests
-import shapely.wkt
-import shapely.ops
-import re
 import json
 import pyproj
 import geopandas
 import numpy as np
 import pyproj
 import geopandas
-from geomet import wkt, InvalidGeoJSONException
-from sklearn.neighbors import NearestNeighbors
+from geomet import wkt as geomet_wkt, InvalidGeoJSONException
 from shapely.geometry import Polygon, LineString, Point
+import shapely.wkt
+import shapely.ops
+
+try:
+    import requests
+    from sklearn.neighbors import NearestNeighbors
+except ImportError:
+    requests = None
+    NearestNeighbors = None
+
 
 # Accepts a single wkt string, or a tuple with crs projection
 # (Or a list of the above, mixed)
 # Accepted: wkt_str, (wkt_str,), (wkt_str, crs), [wkt_str, (wkt_str,crs)]
 
 class simplifyWKT():
     def __init__(self, wkt_obj, default_crs="EPSG:4326"):
+        if requests is None or NearestNeighbors is None:
+               raise ImportError(f'Could not find required packages for class "simplifyWKT". Install additional dependencies via pip.\nEx: `python3 -m pip install WKTUtils[extras]`')
+
         self.shapes = []
         self.errors = None
         self.repairs = []
@@ -81,7 +89,7 @@ def __init__(self, wkt_obj, default_crs="EPSG:4326"):
             # (Needed until https://github.com/geomet/geomet/issues/58 is fixed)
             wkt_str = wkt_str.upper()
             wkt_str = wkt_str.replace(" Z ", " ").replace(" M ", " ").replace(" ZM ", " ")
-            wkt_json = wkt.loads(wkt_str)
+            wkt_json = geomet_wkt.loads(wkt_str)
         except AttributeError as e:
             self.errors = { 'errors': [{'type': 'ATTRIBUTE', 'report': 'Could not parse WKT: {0}.'.format(str(e))}] }
             return
@@ -314,36 +322,36 @@ def __convexHullShape(self, wkt_obj):
         for item in wkt_obj:
             # If a json / geojson:
             if isinstance(item, type({})):
-                wkt_shapely.append(shapely.wkt.loads(wkt.dumps(item)))
+                wkt_shapely.append(shapely.wkt.loads(geomet_wkt.dumps(item)))
             # Else you got it from shapely:
             elif getattr(item, "geom_type", None) != None:
                 converted_from_shapely = True
                 wkt_shapely.append(item)
         # Check for simple case:
         if len(wkt_shapely) == 1 and wkt_shapely[0].geom_type.upper() in ["POINT","MULTIPOINT","LINESTRING","POLYGON"]:
             hulled_shape = wkt_shapely[0].convex_hull
-            return hulled_shape if converted_from_shapely else wkt.loads(shapely.wkt.dumps(hulled_shape))
+            return hulled_shape if converted_from_shapely else geomet_wkt.loads(shapely.wkt.dumps(hulled_shape))
         # Have to merge the coords:
-        wkt_json = [wkt.loads(shapely.wkt.dumps(shape)) for shape in wkt_shapely]
+        wkt_json = [geomet_wkt.loads(shapely.wkt.dumps(shape)) for shape in wkt_shapely]
         all_coords = self.__getAllCoords(wkt_json)
         if len(all_coords) == 0:
             return None
 
         # Convex_hull and add the new shape:
         MultiPoint = {'type': 'MultiPoint', 'coordinates': all_coords }
         # Quicky convert to shapely obj for the convex hull:
-        shape = shapely.wkt.loads(wkt.dumps(MultiPoint)).convex_hull
+        shape = shapely.wkt.loads(geomet_wkt.dumps(MultiPoint)).convex_hull
         # If they passed in a shapely object, return one. Else return a geojson
         if converted_from_shapely:
             return shape
         else:
             # else convert back to geojson:
-            return wkt.loads(shapely.wkt.dumps(shape))
+            return geomet_wkt.loads(shapely.wkt.dumps(shape))
 
     def __jsonToShapely(self, list_of_shapes):
         shapely_shapes = []
         for shape in list_of_shapes:
-            shape = shapely.wkt.loads(wkt.dumps(shape))
+            shape = shapely.wkt.loads(geomet_wkt.dumps(shape))
             shapely_shapes.append(shape)
         return shapely_shapes
 
@@ -377,13 +385,13 @@ def __mergeShapelyList(self, shapely_list):
 
     # Takes either a list, or single geomet/shapely wkt, and returns a unique list of points:
     def __getAllCoords(self, wkt_obj):
-        if not isinstance(wkt, type([])):
+        if not isinstance(wkt_obj, type([])):
             wkt_obj = [wkt_obj]
 
         for i, single_shape in enumerate(wkt_obj):
             # If the shape is from shapely:
             if getattr(single_shape, "geom_type", None) != None:
-                wkt_obj[i] = wkt.loads(shapely.wkt.dumps(single_shape))
+                wkt_obj[i] = geomet_wkt.loads(shapely.wkt.dumps(single_shape))
 
         match_coords = r'(\[\s*' +self.regex_digit+ r'\s*,\s*' +self.regex_digit+ r'\s*\])'
         coords = re.findall(match_coords, str(wkt_obj))
@@ -419,7 +427,7 @@ def getJsonWKT(str_type, coords):
                 new_shape = shapely.wkt.dumps(LineString( coords ))
             elif str_type.upper() == "POINT":
                 new_shape = shapely.wkt.dumps(Point( coords[0] ))
-            return wkt.loads(new_shape)
+            return geomet_wkt.loads(new_shape)
 
         def getClampedCoords(wkt_json):
             # num_coords out of lat +/- 90
@@ -506,14 +514,14 @@ def distance(p1, p2):
             org_num_points = len(getCoords(wkt_json))
             current_num_points = org_num_points
             closest_distance = getClosestPointDist(wkt_json)
-            wkt_shapely = shapely.wkt.loads(wkt.dumps(wkt_json))
+            wkt_shapely = shapely.wkt.loads(geomet_wkt.dumps(wkt_json))
             while (current_num_points > 300 or closest_distance < 0.004) and attempts < 10:
                 # Set the tolerance/closest_distance for the next loop around:
                 logging.debug('The shape\'s length is {0}, simplifying further with tolerance {1}'.format(current_num_points, tolerance ))
                 attempts += 1
                 wkt_shapely = wkt_shapely.simplify(tolerance, preserve_topology=True)
                 tolerance *= 5
-                wkt_json = wkt.loads(shapely.wkt.dumps(wkt_shapely))
+                wkt_json = geomet_wkt.loads(shapely.wkt.dumps(wkt_shapely))
                 current_num_points = len(getCoords(wkt_json))
                 closest_distance = getClosestPointDist(wkt_json)
             # If it couldn't simplify enough:
@@ -527,23 +535,23 @@ def distance(p1, p2):
                     'report': 'Simplified shape from {0} points to {1} points, after {2} iterations.'.format(org_num_points, current_num_points, attempts)
                 })
                 logging.debug(self.repairs[-1])
-            return wkt.loads(shapely.wkt.dumps(wkt_shapely))
+            return geomet_wkt.loads(shapely.wkt.dumps(wkt_shapely))
 
         ## REPAIR MERGED WKT START:
         # Quick sanity check. No clue if it's actually possible to hit this:
         if single_wkt.geom_type.upper() not in ["POLYGON", "LINESTRING", "POINT"]:
             self.errors = {'errors': [{'type': 'VALUE', 'report': 'Could not simplify WKT down to single shape.'}] }
             return
         # You can't edit coords in shapely. You have to create a new shape w/ the new coords:
-        wkt_json = wkt.loads(shapely.wkt.dumps(single_wkt))
+        wkt_json = geomet_wkt.loads(shapely.wkt.dumps(single_wkt))
         wkt_json = getClampedCoords(wkt_json)
         wkt_json = simplifyPoints(wkt_json)
         # Clamp, simplify, *then* wrap, to help simplify be more accuate w/ points outside of poles
         if self.errors != None:
             return
         wrapped_coords = getWrappedCoords(wkt_json)
-        wkt_wrapped = wkt.dumps(wrapped_coords)
-        wkt_unwrapped = wkt.dumps(getUnwrappedCoords(wrapped_coords))
+        wkt_wrapped = geomet_wkt.dumps(wrapped_coords)
+        wkt_unwrapped = geomet_wkt.dumps(getUnwrappedCoords(wrapped_coords))
         return wkt_unwrapped, wkt_wrapped
 
     def __runWKTsAgainstCMR(self):
@@ -556,10 +564,10 @@ def CMRSendRequest(cmr_coords):
             return r.status_code, r.text
 
         # NOTE: Only polygons get sent here. Linestring and point wkt's have already returned.
-        wkt_obj_wrapped = wkt.loads(self.wkt_wrapped)
-        wkt_obj_unwrapped = wkt.loads(self.wkt_unwrapped)
+        wkt_obj_wrapped = geomet_wkt.loads(self.wkt_wrapped)
+        wkt_obj_unwrapped = geomet_wkt.loads(self.wkt_unwrapped)
 
-        cmr_coords = parse_wkt_util(wkt.dumps(wkt_obj_wrapped)).split(':')[1].split(',')
+        cmr_coords = parse_wkt_util(geomet_wkt.dumps(wkt_obj_wrapped)).split(':')[1].split(',')
         status_code, text = CMRSendRequest(cmr_coords)
         if status_code != 200:
             if 'Points must be provided in counter-clockwise order.' in text:
@@ -592,8 +600,8 @@ def CMRSendRequest(cmr_coords):
                 self.errors = { 'errors': [{'type': 'UNKNOWN', 'report': 'Unknown CMR error: {0}'.format(text)}]}
                 return
 
-        self.wkt_wrapped = wkt.dumps(wkt_obj_wrapped)
-        self.wkt_unwrapped = wkt.dumps(wkt_obj_unwrapped)
+        self.wkt_wrapped = geomet_wkt.dumps(wkt_obj_wrapped)
+        self.wkt_unwrapped = geomet_wkt.dumps(wkt_obj_unwrapped)
 
 def repairWKT(wkt_str, default_crs="EPSG:4326"):
     return simplifyWKT(wkt_str, default_crs=default_crs).get_simplified_json()

build/mergehook-testsuite.yml

@@ -7,11 +7,13 @@ env:
 phases:
   install:
     runtime-versions:
-      python: 3.7
+      python: 3.8
     commands:
-      - python3.7 -m venv ${VENV}
+      - python3.8 -m venv ${VENV}
       - . ./${VENV}/bin/activate
-      - python3.7 -m pip install -r requirements.txt --upgrade
+      - python3.8 -m pip install --upgrade pip
+      - python3.8 -m pip install .[extras,test]
+      # - python3.7 -m pip install -r requirements.txt --upgrade
   build:
     commands:
       # TEST: Run the test suite...

build/upload-pypi.yml

@@ -7,15 +7,15 @@ env:
 phases:
   install:
     runtime-versions:
-      python: 3.7
+      python: 3.8
     commands:
-      - python3.7 -m venv ${VENV}
+      - python3.8 -m venv ${VENV}
       - . ./${VENV}/bin/activate
-      - python3.7 -m pip install -r requirements.txt --upgrade
+      - python3.8 -m pip install -r requirements.txt --upgrade
   build:
     commands:
       # Setup the wheel to upload:
-      - python3.7 setup.py bdist_wheel
+      - python3.8 setup.py bdist_wheel
       # Run a quick check on it for errors:
       - twine check dist/*
       # Set creds for the account:

requirements.txt

@@ -52,7 +52,6 @@ scikit-learn==0.24.2
 scipy==1.7.1
 Shapely==1.7.1
 six==1.16.0
-sklearn==0.0
 threadpoolctl==2.2.0
 toml==0.10.2
 tzlocal==3.0

setup.py

@@ -1,9 +1,19 @@
 import setuptools
 
-
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
+extra_requirements = [
+    'requests',
+    'scikit-learn', 
+]
+
+test_requirements = [
+    "pytest==6.2.5",
+    "pytest-automation==1.1.2",
+    'pytest-xdist==2.4.0',
+]
+
 setuptools.setup(
     name="WKTUtils",
     # version= Moved to pyproject.toml,
@@ -13,26 +23,23 @@
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/asfadmin/Discovery-WKTUtils.git",
+    extras_require={"extras": extra_requirements, "test": test_requirements},
     packages=setuptools.find_packages(),
     # package_data= {'WKTUtils': ['VERSION']},
     classifiers=[
         "Programming Language :: Python :: 3",
         "License :: OSI Approved :: BSD License",
         "Operating System :: OS Independent",
     ],
-    python_requires='>=3.6',
+    python_requires='>=3.8',
     install_requires=[
         'dateparser',
         'defusedxml',
-        'Fiona',
         'geomet',
         'geopandas',
         'kml2geojson',
         'pyshp',
         'PyYAML',
-        'regex',
-        'requests',
-        'Shapely',
-        'sklearn',
-    ]
+        'Shapely'
+    ],
 )