Merge pull request #25 from openaddresses/coordinate-order

CSV Coordinate Order

openaddr/conform.py

@@ -51,25 +51,6 @@ def gdal_error_handler(err_class, err_num, err_msg):
 
 UNZIPPED_DIRNAME = 'unzipped'
 
-geometry_types = {
-    ogr.wkbPoint: 'Point',
-    ogr.wkbPoint25D: 'Point 2.5D',
-    ogr.wkbLineString: 'LineString',
-    ogr.wkbLineString25D: 'LineString 2.5D',
-    ogr.wkbLinearRing: 'LinearRing',
-    ogr.wkbPolygon: 'Polygon',
-    ogr.wkbPolygon25D: 'Polygon 2.5D',
-    ogr.wkbMultiPoint: 'MultiPoint',
-    ogr.wkbMultiPoint25D: 'MultiPoint 2.5D',
-    ogr.wkbMultiLineString: 'MultiLineString',
-    ogr.wkbMultiLineString25D: 'MultiLineString 2.5D',
-    ogr.wkbMultiPolygon: 'MultiPolygon',
-    ogr.wkbMultiPolygon25D: 'MultiPolygon 2.5D',
-    ogr.wkbGeometryCollection: 'GeometryCollection',
-    ogr.wkbGeometryCollection25D: 'GeometryCollection 2.5D',
-    ogr.wkbUnknown: 'Unknown'
-    }
-
 # extracts:
 # - '123' from '123 Main St'
 # - '123 1/2' from '123 1/2 Main St'
@@ -475,6 +456,7 @@ def ogr_source_to_csv(source_config, source_path, dest_path):
     inSpatialRef = in_layer.GetSpatialRef()
     srs = source_config.data_source["conform"].get("srs", None)
 
+    # Skip Transformation is the EPSG code is superfluous
     if srs is not None:
         # OGR may have a projection, but use the explicit SRS instead
         if srs.startswith(u"EPSG:"):
@@ -510,6 +492,7 @@ def ogr_source_to_csv(source_config, source_path, dest_path):
     # Set up a transformation from the source SRS to EPSG:4326
     outSpatialRef = osr.SpatialReference()
     outSpatialRef.ImportFromEPSG(4326)
+
     if int(osgeo.__version__[0]) >= 3:
         # GDAL 3 changes axis order: https://github.com/OSGeo/gdal/issues/1546
         outSpatialRef.SetAxisMappingStrategy(osgeo.osr.OAMS_TRADITIONAL_GIS_ORDER)
@@ -616,9 +599,13 @@ def csv_source_to_csv(source_config, source_path, dest_path):
 
         else:
             # CSV sources: replace the source's lat/lon columns with OA:GEOM
-            old_latlon = [source_config.data_source["conform"]["lat"], source_config.data_source["conform"]["lon"]]
-            old_latlon.extend([s.upper() for s in old_latlon])
-            out_fieldnames = [fn for fn in reader.fieldnames if fn not in old_latlon]
+            old_ll = [
+                source_config.data_source["conform"]["lon"],
+                source_config.data_source["conform"]["lat"]
+            ]
+
+            old_ll.extend([s.upper() for s in old_ll])
+            out_fieldnames = [fn for fn in reader.fieldnames if fn not in old_ll]
             out_fieldnames.append(GEOM_FIELDNAME)
 
         # Write the extracted CSV file
@@ -683,11 +670,14 @@ def _transform_to_4326(srs):
 
         in_spatial_ref = osr.SpatialReference()
         in_spatial_ref.ImportFromEPSG(epsg_id)
+
         out_spatial_ref = osr.SpatialReference()
         out_spatial_ref.ImportFromEPSG(4326)
 
-        # GDAL 3 changes axis order: https://github.com/OSGeo/gdal/issues/1546
-        out_spatial_ref.SetAxisMappingStrategy(osgeo.osr.OAMS_TRADITIONAL_GIS_ORDER)
+        if int(osgeo.__version__[0]) >= 3:
+            # GDAL 3 changes axis order: https://github.com/OSGeo/gdal/issues/1546
+            in_spatial_ref.SetAxisMappingStrategy(osgeo.osr.OAMS_TRADITIONAL_GIS_ORDER)
+            out_spatial_ref.SetAxisMappingStrategy(osgeo.osr.OAMS_TRADITIONAL_GIS_ORDER)
 
         _transform_cache[srs] = osr.CoordinateTransformation(in_spatial_ref, out_spatial_ref)
     return _transform_cache[srs]
@@ -755,13 +745,12 @@ def row_extract_and_reproject(source_config, source_row):
 
 
     # Reproject the coordinates if necessary
-    if "srs" in data_source["conform"]:
+    if "srs" in data_source["conform"] and data_source["conform"]["srs"] != "EPSG:4326":
         try:
             srs = data_source["conform"]["srs"]
-            point = ogr.CreateGeometryFromWkt(source_geom)
-            point.Transform(_transform_to_4326(srs))
-
-            source_geom = point.ExportToWkt()
+            geom = ogr.CreateGeometryFromWkt(source_geom)
+            geom.Transform(_transform_to_4326(srs))
+            source_geom = geom.ExportToWkt()
         except (TypeError, ValueError) as e:
             if not (source_x == "" or source_y == ""):
                 _L.debug("Could not reproject %s %s in SRS %s", source_x, source_y, srs)

openaddr/tests/__init__.py

@@ -777,9 +777,9 @@ def test_single_jp_fukushima2(self):
         self.assertEqual(rows[1]['STREET'], u'田沢字躑躅ケ森')
         self.assertEqual(rows[2]['NUMBER'], u'22-9')
         self.assertEqual(rows[2]['STREET'], u'小田字正夫田')
-        self.assertEqual(rows[0]['GEOM'], 'POINT (37.706391 140.480007)')
-        self.assertEqual(rows[1]['GEOM'], 'POINT (37.707664 140.486267)')
-        self.assertEqual(rows[2]['GEOM'], 'POINT (37.710239 140.41875)')
+        self.assertEqual(rows[0]['GEOM'], 'POINT (140.480007 37.706391)')
+        self.assertEqual(rows[1]['GEOM'], 'POINT (140.486267 37.707664)')
+        self.assertEqual(rows[2]['GEOM'], 'POINT (140.41875 37.710239)')
 
     def test_single_utah(self):
         ''' Test complete process_one.process on data that uses file selection with mixed case (issue #104)

openaddr/tests/conform.py

@@ -1818,16 +1818,22 @@ def test_lake_man_split2(self):
             rows = list(csv.DictReader(fp))
             self.assertEqual(rows[0]['NUMBER'], '1')
             self.assertEqual(rows[0]['STREET'], 'Spectrum Pointe Dr #320')
+            self.assertEqual(rows[0]['GEOM'], 'POINT (-122.25925 37.802613)')
             self.assertEqual(rows[1]['NUMBER'], '')
             self.assertEqual(rows[1]['STREET'], '')
+            self.assertEqual(rows[1]['GEOM'], 'POINT (-122.256718 37.802528)')
             self.assertEqual(rows[2]['NUMBER'], '300')
             self.assertEqual(rows[2]['STREET'], 'E Chapman Ave')
+            self.assertEqual(rows[2]['GEOM'], 'POINT (-122.257941 37.802969)')
             self.assertEqual(rows[3]['NUMBER'], '1')
             self.assertEqual(rows[3]['STREET'], 'Spectrum Pointe Dr #320')
+            self.assertEqual(rows[3]['GEOM'], 'POINT (-122.258971 37.800748)')
             self.assertEqual(rows[4]['NUMBER'], '1')
             self.assertEqual(rows[4]['STREET'], 'Spectrum Pointe Dr #320')
+            self.assertEqual(rows[4]['GEOM'], 'POINT (-122.256954 37.800714)')
             self.assertEqual(rows[5]['NUMBER'], '1')
             self.assertEqual(rows[5]['STREET'], 'Spectrum Pointe Dr #320')
+            self.assertEqual(rows[5]['GEOM'], 'POINT (-122.25764 37.804359)')
 
     def test_nara_jp(self):
         "Test case from jp-nara.json"
@@ -1847,6 +1853,8 @@ def test_lake_man_3740(self):
         with open(dest_path) as fp:
             rows = list(csv.DictReader(fp))
             x,y= wkt_pt(rows[0]['GEOM'])
+
+            # POINT (-122.2592495 37.8026123)
             self.assertAlmostEqual(-122.2592495, x, places=4)
             self.assertAlmostEqual(37.8026123, y, places=4)
 

openaddr/tests/conforms/jp-nara.json

@@ -15,7 +15,8 @@
             },
             "conform": {
                 "format": "csv",
-                "lon": "\u7def\u5ea6",
+                "lon": "\u7d4c\u5ea6",
+                "lat": "\u7def\u5ea6",
                 "srs": "EPSG:4612",
                 "number": "auto_number",
                 "number": {
@@ -27,7 +28,6 @@
                     "separator": "-"
                 },
                 "street": "\u5927\u5b57\u30fb\u753a\u4e01\u76ee\u540d",
-                "lat": "\u7d4c\u5ea6",
                 "encoding": "SHIFT_JISX0213"
             },
             "license": "http://nlftp.mlit.go.jp/ksj/other/yakkan.html"

openaddr/tests/sources/jp-fukushima2.json

@@ -19,6 +19,7 @@
             "attribution": "Japanese Ministry of Land, Infrastructure and Transport",
             "conform": {
                 "format": "csv",
+                "lon": "経度",
                 "lat": "緯度",
                 "srs": "EPSG:4612",
                 "number": {
@@ -30,7 +31,6 @@
                     "separator": "-"
                 },
                 "street": "大字・町丁目名",
-                "lon": "経度",
                 "encoding": "SHIFT_JISX0213"
             },
             "license": {

setup.py

@@ -33,7 +33,7 @@
     },
     test_suite = 'openaddr.tests',
     install_requires = [
-        'boto == 2.49.0', 'dateutils == 0.6.6', 'ijson == 2.4',
+        'dateutils == 0.6.6', 'ijson == 2.4',
 
         'simplejson == 3.17.2',
 
@@ -44,27 +44,17 @@
         'uritemplate == 3.0.0',
 
         # http://docs.python-requests.org/en/master/
-        'requests == 2.22.0',
+        'requests == 2.27.1',
 
         # https://github.com/patrys/httmock
         'httmock == 1.3.0',
 
-        # https://boto3.readthedocs.org
-        'boto3 == 1.11.5',
-
         # https://github.com/openaddresses/pyesridump
         'esridump == 1.10.1',
 
         # Used in openaddr.parcels
         'Shapely == 1.7.1',
 
-        # Used in dotmaps preview to support S3-backed SQLite mbtiles
-        # https://rogerbinns.github.io/apsw/
-        'apsw == 3.9.2.post1',
-
-        # http://pythonhosted.org/itsdangerous/
-        'itsdangerous == 1.1.0',
-
         # https://github.com/tilezen/mapbox-vector-tile
         'mapbox-vector-tile==1.2.0',
         'future==0.16.0',