Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

added geocoding function #50

Merged
merged 46 commits into from
Nov 20, 2023
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
dc07743
added geocoding function
mtravis Oct 16, 2023
8094953
add more tests
felix-schott Oct 18, 2023
cbc7bf0
Merge pull request #52 from felix-schott/36_add-cli-tests
cholmes Oct 18, 2023
cd80183
added osmnx to requirements
mtravis Oct 19, 2023
e03dc60
made changes to downloads.py
mtravis Oct 20, 2023
e26df79
Some performance quick wins for the geopandas implementation
theroggy Nov 3, 2023
d08dc00
Update process.py
theroggy Nov 3, 2023
ce0bc83
Updated geocoded function and added test
mtravis Nov 4, 2023
9d488c6
Updated settings source
mtravis Nov 4, 2023
290558a
import geocode function to tests
mtravis Nov 4, 2023
9ffae1e
Fixed geocode test
mtravis Nov 4, 2023
d10fb90
added location arg to cli.py
mtravis Nov 5, 2023
02203c4
fixed import settings
mtravis Nov 6, 2023
04bd224
added exception for when location geojson isn't true geojson
mtravis Nov 6, 2023
f1703b2
Merge pull request #53 from theroggy/Some-performance-quick-wins-for-…
cholmes Nov 6, 2023
d2955aa
Update download_buildings.py
mtravis Nov 6, 2023
b5c97d2
Fixed wkt to geojson for geocode
mtravis Nov 6, 2023
f723da6
Update download_buildings.py
mtravis Nov 6, 2023
3e5a22c
added geocoding function
mtravis Oct 16, 2023
cd001aa
added osmnx to requirements
mtravis Oct 19, 2023
38a8a56
made changes to downloads.py
mtravis Oct 20, 2023
4781358
Updated geocoded function and added test
mtravis Nov 4, 2023
ebb811d
Updated settings source
mtravis Nov 4, 2023
4ffb84d
import geocode function to tests
mtravis Nov 4, 2023
db97396
Fixed geocode test
mtravis Nov 4, 2023
0811250
added location arg to cli.py
mtravis Nov 5, 2023
d94a6ec
fixed import settings
mtravis Nov 6, 2023
46025ea
added exception for when location geojson isn't true geojson
mtravis Nov 6, 2023
40b7030
Update download_buildings.py
mtravis Nov 6, 2023
751ad60
Fixed wkt to geojson for geocode
mtravis Nov 6, 2023
68d984b
Update download_buildings.py
mtravis Nov 6, 2023
db12b37
added geocoding function
mtravis Oct 16, 2023
50d8837
made changes to downloads.py
mtravis Oct 20, 2023
99b03f9
Updated geocoded function and added test
mtravis Nov 4, 2023
acb6e7c
Updated settings source
mtravis Nov 4, 2023
f05cb25
added location arg to cli.py
mtravis Nov 5, 2023
88bb7ba
fixed import settings
mtravis Nov 6, 2023
581fb43
added exception for when location geojson isn't true geojson
mtravis Nov 6, 2023
e44a5bf
Update download_buildings.py
mtravis Nov 6, 2023
64229ba
Update download_buildings.py
mtravis Nov 6, 2023
7dc815c
add cli test for --location, fix some minor things, change dst from p…
felix-schott Nov 6, 2023
5e7338c
merging
felix-schott Nov 6, 2023
f3c7174
remove cache dir
felix-schott Nov 6, 2023
fd87791
remove option where unused, tidy up a bit
felix-schott Nov 6, 2023
0daad7c
changed country_iso in cli geocode tests
mtravis Nov 7, 2023
e545565
update readme, simplify geojson_to_quadkey, change geocode() to retur…
felix-schott Nov 16, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions open_buildings/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from open_buildings.download_buildings import download as download_buildings
from open_buildings.overture.add_columns import process_parquet_files
from open_buildings.overture.partition import process_db
from open_buildings.settings import Source
from settings import Source
from datetime import datetime, timedelta
from tabulate import tabulate
import boto3 # Required for S3 operations
Expand Down Expand Up @@ -42,7 +42,8 @@ def handle_comma_separated(ctx, param, value):
@click.option('-s', '--silent', is_flag=True, default=False, help='Suppress all print outputs.')
@click.option('--overwrite', default=False, is_flag=True, help='Overwrite the destination file if it already exists.')
@click.option('--verbose', default=False, is_flag=True, help='Print detailed logs with timestamps.')
def get_buildings(geojson_input, dst, source, country_iso, silent, overwrite, verbose):
@click.option('--geocode', default=False, is_flag=True, help='Use city or region name')
def get_buildings(geojson_input, geocode, dst, source, country_iso, silent, overwrite, verbose):
"""Tool to extract buildings in common geospatial formats from large archives of GeoParquet data online. GeoJSON
input can be provided as a file or piped in from stdin. If no GeoJSON input is provided, the tool will read from stdin.

Expand Down Expand Up @@ -74,7 +75,7 @@ def get_buildings(geojson_input, dst, source, country_iso, silent, overwrite, ve
else:
mtravis marked this conversation as resolved.
Show resolved Hide resolved
geojson_data = json.load(click.get_text_stream('stdin'))
mtravis marked this conversation as resolved.
Show resolved Hide resolved

download_buildings(geojson_data, source=source, generate_sql=False, dst=dst, silent=silent, overwrite=overwrite, verbose=verbose, country_iso=country_iso)
download_buildings(geojson_data, generate_sql=False, dst=dst, silent=silent, overwrite=overwrite, verbose=verbose, country_iso=country_iso)

@google.command('benchmark')
@click.argument('input_path', type=click.Path(exists=True))
Expand Down Expand Up @@ -162,6 +163,12 @@ def benchmark(
@click.option(
'--overwrite', is_flag=True, help="Whether to overwrite any existing output files."
)
@click.option(
'--geocode',
type=str,
is_flag=True,
help="Use city or region name instead of geojson"
)
@click.option(
'--process',
type=click.Choice(['duckdb', 'pandas', 'ogr']),
Expand Down
69 changes: 39 additions & 30 deletions open_buildings/download_buildings.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import json
import click
from math import tan, cos, log, pi
from shapely.geometry import shape
from shapely.geometry import shape, box
from typing import Dict, Any, Union
import mercantile
import duckdb
Expand All @@ -15,7 +15,7 @@
import subprocess
from shapely import wkb
import shutil

import osmnx
from open_buildings.settings import Source, Format, settings

def geojson_to_quadkey(data: dict) -> str:
Expand Down Expand Up @@ -43,6 +43,11 @@ def geojson_to_wkt(data: dict) -> str:
geometry = shape(data['geometry'])
return geometry.wkt

def geocode_to_wkt(data: str):
location = osmnx.geocode_to_gdf(data)
wkt = box(*location.total_bounds)
return wkt

def quadkey_to_geojson(quadkey: str) -> dict:
# Convert the quadkey to tile coordinates
tile = mercantile.quadkey_to_tile(quadkey)
Expand Down Expand Up @@ -81,49 +86,53 @@ def quadkey(geojson_input):
geojson_data = json.load(click.get_text_stream('stdin'))

result = geojson_to_quadkey(geojson_data)
print()
click.echo(result)

@cli.command()
@click.argument('geojson_input', type=click.File('r'), required=False)
def WKT(geojson_input):
"""Convert GeoJSON to Well Known Text."""
@click.option('--geocode', type=str, is_flag=True, help='geocode using a city or region')
def WKT(geojson_input, geocode_input):
mtravis marked this conversation as resolved.
Show resolved Hide resolved
"""Convert GeoJSON to Well Known Text."""
if geojson_input:
geojson_data = json.load(geojson_input)
else:
geojson_data = json.load(click.get_text_stream('stdin'))

result = geojson_to_wkt(geojson_data)
geojson_data = json.load(click.get_text_stream('stdin'))
if geocode_input:
result = geocode_to_wkt(geocode_input)
else:
result = geojson_to_wkt(geojson_data)
click.echo(result)


@click.command()
@click.argument('geojson_input', type=click.File('r'), required=False)
mtravis marked this conversation as resolved.
Show resolved Hide resolved
@click.option('--only-quadkey', is_flag=True, help='Include only the quadkey in the WHERE clause.')
@click.option('--local', is_flag=True, help='Use local path for parquet files instead of the S3 URL.')
def sql(geojson_input, only_quadkey, local):
"""Generate an SQL query based on the input GeoJSON."""
# def sql(geojson_input, only_quadkey, local):
# """Generate an SQL query based on the input GeoJSON."""

# Read the GeoJSON
if geojson_input:
geojson_data = json.load(geojson_input)
else:
geojson_data = json.load(click.get_text_stream('stdin'))

quadkey = geojson_to_quadkey(geojson_data)
wkt = geojson_to_wkt(geojson_data)

# Adjust the path in read_parquet based on the --local flag
path = '*.parquet' if local else 's3://us-west-2.opendata.source.coop/cholmes/overture/geoparquet-country-quad-2/*.parquet'
base_sql = f"select * from read_parquet('{path}')"
# # Read the GeoJSON
# if geojson_input:
# geojson_data = json.load(geojson_input)
# else:
# geojson_data = json.load(click.get_text_stream('stdin'))

# quadkey = geojson_to_quadkey(geojson_data)
# wkt = geojson_to_wkt(geojson_data)

# # Adjust the path in read_parquet based on the --local flag
# path = '*.parquet' if local else 's3://us-west-2.opendata.source.coop/cholmes/overture/geoparquet-country-quad-2/*.parquet'
# base_sql = f"select * from read_parquet('{path}')"

# Construct the WHERE clause based on the options
where_clause = f"WHERE quadkey LIKE '{quadkey}%'"
if not only_quadkey:
where_clause += f" AND\nST_Within(ST_GeomFromWKB(geometry), ST_GeomFromText('{wkt}'))"
# # Construct the WHERE clause based on the options
# where_clause = f"WHERE quadkey LIKE '{quadkey}%'"
# if not only_quadkey:
# where_clause += f" AND\nST_Within(ST_GeomFromWKB(geometry), ST_GeomFromText('{wkt}'))"

sql_query = f"{base_sql},\n{where_clause}"
full_sql_query = f"COPY ('{sql_query}' TO 'buildings.fgb' WITH (FORMAT GDAL, DRIVER 'FlatGeobuf')"
click.echo(full_sql_query)
# sql_query = f"{base_sql},\n{where_clause}"
# full_sql_query = f"COPY ('{sql_query}' TO 'buildings.fgb' WITH (FORMAT GDAL, DRIVER 'FlatGeobuf')"
# click.echo(full_sql_query)

@cli.command()
@click.argument('quadkey_input', type=str)
Expand All @@ -132,9 +141,9 @@ def quad2json(quadkey_input):
result = quadkey_to_geojson(quadkey_input)
click.echo(json.dumps(result, indent=2))


def download(
geojson_data: Dict[str, Any],
geojson_data: Dict[str, Any],
geocode: Optional[str] = None,
mtravis marked this conversation as resolved.
Show resolved Hide resolved
dst: Union[Path, str] = "buildings.json",
source: Union[Source, str] = Source.OVERTURE,
format: Optional[Union[Format, str]] = None,
Expand Down
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ click
duckdb
pandas
geopandas
osmnx
shapely
openlocationcode
tabulate
Expand Down
Loading