diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml new file mode 100644 index 00000000..f9148f0a --- /dev/null +++ b/.github/workflows/build_and_deploy.yml @@ -0,0 +1,197 @@ +name: Build and Deploy to ECR and ECS +# We want this to build our docker container, push to ECR ("build" job) +# and then perform a rolling update to ECS with the new image ( "deploy" job) + +# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#on +on: + push: + branches: + - master + - python3.10-upgrade + +concurrency: + group: testing_environment + cancel-in-progress: false + + +jobs: + + build: + name: Build and Push to ECR + if: "!contains(github.event.head_commit.message, 'skip ci')" + #needs: pre-build + runs-on: ais-runner + #runs-on: ubuntu-latest + + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + # This is a github function? Ref doc: https://github.com/actions/checkout#checkout-a-different-branch + - name: Checkout commit + uses: actions/checkout@v4 + + - name: Get github commit sha ID + run: echo "GITHUB_SHA_SHORT=$(echo $GITHUB_SHA | cut -c 1-6)" >> $GITHUB_ENV + + # https://github.com/marketplace/actions/microsoft-teams-notification + - name: Notify build start + uses: jdcargile/ms-teams-notification@v1.4 + with: + GITHUB-TOKEN: ${{ github.token }} + ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI }} + notification-summary: Building and testing of new AIS docker image started for commit id ${{ env.GITHUB_SHA_SHORT }} + notification-color: 17a2b8 + timezone: America/New_York + + + # https://github.com/aws-actions/amazon-ecr-login + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Identify production cluster, either blue or green + id: prod-cluster-color + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + PROD_ENDPOINT: ${{ secrets.PROD_ENDPOINT }} + run: | + # Note: a simple dig doesn't work from in office. + # Run the command manually first so we're sure it works, otherwise the var assignment hides errors. + aws route53 list-resource-record-sets --hosted-zone-id ${{ secrets.PHILACITY_ZONE_ID }} --query "ResourceRecordSets[?Name == '${{ secrets.PROD_ENDPOINT }}.']" | grep -o "blue\|green" + echo "PROD_COLOR=$(aws route53 list-resource-record-sets --hosted-zone-id ${{ secrets.PHILACITY_ZONE_ID }} --query "ResourceRecordSets[?Name == '${{ secrets.PROD_ENDPOINT }}.']" | grep -o "blue\|green")" >> $GITHUB_ENV + + - name: Set engine hostname based on prod color + env: + PROD_ENDPOINT: ${{ secrets.PROD_ENDPOINT }} + run: | + if [[ "$PROD_COLOR" -eq "blue" ]]; then + echo "ENGINE_HOST=${{ secrets.BLUE_ENGINE_CNAME }}" >> $GITHUB_ENV + elif [[ "$PROD_COLOR" -eq "green" ]]; then + echo "ENGINE_HOST=${{ secrets.GREEN_ENGINE_CNAME }}" >> $GITHUB_ENV + fi + + - name: git fetch and pull failsafe + working-directory: /home/ubuntu/ais + run: git fetch && git pull + + - name: Build the Docker image using docker-compose + # Run directly in our ais folder, necessary to get some secrets in the container + working-directory: /home/ubuntu/ais + env: + ENGINE_DB_HOST: ${{ env.ENGINE_HOST }} + ENGINE_DB_PASS: ${{ secrets.ENGINE_DB_PASS }} + run: | + docker-compose -f build-test-compose.yml build --no-cache + + - name: Start the Docker image using docker-compose + # Run directly in our ais folder, necessary to get some secrets in the container + working-directory: /home/ubuntu/ais + env: + ENGINE_DB_HOST: ${{ env.ENGINE_HOST }} + ENGINE_DB_PASS: ${{ secrets.ENGINE_DB_PASS }} + run: docker-compose -f build-test-compose.yml up -d + + - name: Run API pytests to ensure image build is good + env: + ENGINE_DB_HOST: ${{ env.ENGINE_HOST }} + ENGINE_DB_PASS: ${{ secrets.ENGINE_DB_PASS }} + run: | + docker exec ais bash -c 'cd /ais && pytest /ais/ais/tests/api/ -vvv -ra --showlocals --tb=native' + + - name: Confirm nginx configuration is good + run: docker exec ais bash -c 'nginx -t' + + - name: Simple curl query check + run: curl http://localhost:8080/search/1234%20Market%20Street + + # https://github.com/aws-actions/amazon-ecr-login + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Docker Push to ECR + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + ECR_REPOSITORY_URL: ${{ secrets.ECR_REPOSITORY_url }} + run: | + docker tag ais:latest $ECR_REPOSITORY_URL:latest + docker push $ECR_REPOSITORY_URL:latest + + # https://github.com/marketplace/actions/microsoft-teams-notification + - name: Notify build status + if: always() + uses: jdcargile/ms-teams-notification@v1.4 + with: + GITHUB-TOKEN: ${{ github.token }} + ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI }} + notification-summary: Build and push to ECR; ${{ job.status }} for commit ID ${{ env.GITHUB_SHA_SHORT }}! + notification-color: ${{ job.status == 'success' && '28a745' || 'dc3545' }} + timezone: America/New_York + + deploy: + + name: Deploy to prod ECS cluster + # needs prior job of 'build' to not fail. + needs: build + runs-on: ubuntu-latest + + + steps: + + # https://github.com/aws-actions/amazon-ecr-login + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + # Set $PROD_COLOR env var through the complicated method github actions requires + # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#setting-an-environment-variable + - name: Identify production cluster, either blue or green + id: prod-cluster-color + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + PROD_ENDPOINT: ${{ secrets.PROD_ENDPOINT }} + run: | + # Note: a simple dig doesn't work from in office. + # Run the command manually first so we're sure it works, otherwise the var assignment hides errors. + aws route53 list-resource-record-sets --hosted-zone-id ${{ secrets.PHILACITY_ZONE_ID }} --query "ResourceRecordSets[?Name == '${{ secrets.PROD_ENDPOINT }}.']" | grep -o "blue\|green" + echo "PROD_COLOR=$(aws route53 list-resource-record-sets --hosted-zone-id ${{ secrets.PHILACITY_ZONE_ID }} --query "ResourceRecordSets[?Name == '${{ secrets.PROD_ENDPOINT }}.']" | grep -o "blue\|green")" >> $GITHUB_ENV + + - name: Force deploy to ECS cluster + run: | + echo "Deploying to $PROD_COLOR" + aws ecs update-service --cluster ais-$PROD_COLOR-cluster \ + --service ais-$PROD_COLOR-api-service --force-new-deployment --region us-east-1 + aws ecs wait services-stable --cluster ais-$PROD_COLOR-cluster \ + --service ais-$PROD_COLOR-api-service --region us-east-1 + - name: Confirm LB target group health + run: | + blue_tg_arn=$(aws elbv2 describe-target-groups | grep "blue-tg" | grep TargetGroupArn| cut -d"\"" -f4) + green_tg_arn=$(aws elbv2 describe-target-groups | grep "green-tg" | grep TargetGroupArn| cut -d"\"" -f4) + if [[ "$PROD_COLOR" -eq "blue" ]]; then + echo "blue" + aws elbv2 describe-target-health --target-group-arn $blue_tg_arn | grep "\"healthy\"" + echo $? + elif [[ "$PROD_COLOR" -eq "green" ]]; then + echo "green" + aws elbv2 describe-target-health --target-group-arn $green_tg_arn | grep "\"healthy\"" + echo $? + fi + + # https://github.com/marketplace/actions/microsoft-teams-notification + - name: Notify build status + if: always() + uses: jdcargile/ms-teams-notification@v1.4 + with: + GITHUB-TOKEN: ${{ github.token }} + ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI }} + notification-summary: Deployed to ${{ env.PROD_COLOR }} ECS cluster. + notification-color: ${{ job.status == 'success' && '28a745' || 'dc3545' }} + timezone: America/New_York diff --git a/.github/workflows/pull_request_test.yml b/.github/workflows/pull_request_test.yml new file mode 100644 index 00000000..a2a67c09 --- /dev/null +++ b/.github/workflows/pull_request_test.yml @@ -0,0 +1,109 @@ +name: Build and Test Docker Image for PRs + +# Trigger workflow on changes to these paths in stated branch +# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#on +on: + pull_request: + branches: + - master + - python3.10-upgrade + #types: [opened, edited, repoened, review_requested] + +concurrency: + group: testing_environment + cancel-in-progress: false + + +jobs: + + build: + name: Build and Test docker container for PRs + if: "!contains(github.event.head_commit.message, 'skip ci')" + runs-on: ais-runner + #runs-on: ubuntu-latest + # https://github.community/t/sharing-a-variable-between-jobs/16967/13 + # save address for use in other jobs. + + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + # This is a github function? Ref doc: https://github.com/actions/checkout#checkout-a-different-branch + - name: Checkout PR branch + uses: actions/checkout@v4 + + # https://github.com/aws-actions/amazon-ecr-login + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Print current working directory + run: pwd + + # Set $PROD_COLOR env var through the complicated method github actions requires + # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#setting-an-environment-variable + - name: Identify production cluster, either blue or green + id: prod-cluster-color + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + PROD_ENDPOINT: ${{ secrets.PROD_ENDPOINT }} + run: | + # Note: a simple dig doesn't work from in office. + # Run the command manually first so we're sure it works, otherwise the var assignment hides errors. + #echo "PROD_COLOR=$(dig ${{ secrets.PROD_ENDPOINT }} +short | grep -o "blue\|green")" >> $GITHUB_ENV + #dig ${{ secrets.PROD_ENDPOINT }} +short | grep -o "blue\|green" + aws route53 list-resource-record-sets --hosted-zone-id ${{ secrets.PHILACITY_ZONE_ID }} --query "ResourceRecordSets[?Name == '${{ secrets.PROD_ENDPOINT }}.']" | grep -o "blue\|green" + echo "PROD_COLOR=$(aws route53 list-resource-record-sets --hosted-zone-id ${{ secrets.PHILACITY_ZONE_ID }} --query "ResourceRecordSets[?Name == '${{ secrets.PROD_ENDPOINT }}.']" | grep -o "blue\|green")" >> $GITHUB_ENV + + - name: Set engine hostname to the production database for testing against. + env: + PROD_ENDPOINT: ${{ secrets.PROD_ENDPOINT }} + run: | + if [[ "$PROD_COLOR" -eq "blue" ]]; then + echo "ENGINE_HOST=${{ secrets.BLUE_ENGINE_CNAME }}" >> $GITHUB_ENV + elif [[ "$PROD_COLOR" -eq "green" ]]; then + echo "ENGINE_HOST=${{ secrets.GREEN_ENGINE_CNAME }}" >> $GITHUB_ENV + fi + + - name: Build the Docker image using docker-compose + # Run directly in our ais folder, necessary to get some secrets in the container + working-directory: /home/ubuntu/ais + env: + ENGINE_DB_HOST: ${{ env.ENGINE_HOST }} + ENGINE_DB_PASS: ${{ secrets.ENGINE_DB_PASS }} + run: COMPOSE_DOCKER_CLI_BUILD=1 docker-compose -f build-test-compose.yml build --no-cache + + - name: Start the Docker image using docker-compose + # Run directly in our ais folder, necessary to get some secrets in the container + working-directory: /home/ubuntu/ais + env: + ENGINE_DB_HOST: ${{ env.ENGINE_HOST }} + ENGINE_DB_PASS: ${{ secrets.ENGINE_DB_PASS }} + run: docker-compose -f build-test-compose.yml up -d + + - name: Run API pytests to ensure image build is good + env: + ENGINE_DB_HOST: ${{ env.ENGINE_HOST }} + ENGINE_DB_PASS: ${{ secrets.ENGINE_DB_PASS }} + run: | + docker exec ais bash -c 'cd /ais && pytest /ais/ais/tests/api/ -vvv -ra --showlocals --tb=native' + + - name: Confirm nginx configuration is good + run: docker exec ais bash -c 'nginx -t' + + - name: Simple curl query check + run: curl http://localhost:8080/search/1234%20Market%20Street + + + # https://github.com/marketplace/actions/microsoft-teams-notification + - name: Notify job progress + if: always() + uses: jdcargile/ms-teams-notification@v1.4 + with: + GITHUB-TOKEN: ${{ github.token }} + ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI }} + notification-summary: Build status; ${{ job.status }} for branch ${{ env.GITHUB_REF }} + notification-color: ${{ job.status == 'success' && '28a745' || 'dc3545' }} + timezone: America/New_York diff --git a/.gitignore b/.gitignore index cbb8f8fd..c0eba2cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,27 @@ +# ignore migrations folder +migrations/ + +# Intermediate files using for swapping DNS records +route53-prod-change.json +route53-stage-change.json + +# ignore this config file +ais-config.sh + +# Git keys +passyunk-private.key +passyunk-public.key +*.key + +log/ + +# docker-compose build secrets +config-secrets.sh +config-secrets.py +route53-change.json +*.csv + + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -56,6 +80,10 @@ local_settings.py # Flask instance folder instance/* +# AIS supplemental bash config +ais-config.sh +ais/engine/bin/ais-config.sh + # Sphinx documentation docs/_build/ @@ -67,6 +95,7 @@ target/ # pyenv .python-version +venv # dotenv .env @@ -89,3 +118,8 @@ instance/config.py ais/engine/log/ tmp/ ais/api/profiling/ + +.vscode/ + +# migrations +/migrations/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..d0009af2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +FROM python:3.10.8-slim-bullseye +MAINTAINER CityGeo + +# note, have these declared in your .env file and then use docker-compose to build +# only docker-compose uses .env files +ENV ENGINE_DB_HOST=$ENGINE_DB_HOST +ENV ENGINE_DB_PASS=$ENGINE_DB_PASS +ENV GREEN_ENGINE_CNAME=$GREEN_ENGINE_CNAME +ENV BLUE_ENGINE_CNAME=$BLUE_ENGINE_CNAME + +RUN apt-get update -y && \ + apt-get upgrade -y && \ + apt-get install nginx gcc git build-essential vim dnsutils -y && \ + apt-get clean -y && \ + apt-get autoremove -y + +# Automated key for accessing private git repo +RUN mkdir /root/.ssh && chmod 600 /root/.ssh +# Add github to the list of known hosts so our SSH pip installs work later +RUN ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts +COPY ssh-config /root/.ssh/config +COPY passyunk-private.key /root/.ssh/passyunk-private.key +RUN chmod 600 /root/.ssh/config; chmod 600 /root/.ssh/passyunk-private.key + +# Make the AIS cloned into the root, /ais +# Note: Install python reqs at the system level, no need for venv in a docker container +# also caused some issues for me. +RUN mkdir -p /ais +RUN git clone https://github.com/CityOfPhiladelphia/ais --branch python3.10-upgrade /ais +RUN pip install --upgrade pip && \ + pip install -r /ais/requirements.txt + +# Copy our secrets into the flask speciic secret path +COPY ./instance/config.py /ais/instance/config.py + +# Actually install our AIS package +RUN cd /ais && pip3 install . +RUN mkdir -p /ais/instance + +COPY docker-build-files/50x.html /var/www/html/50x.html +COPY docker-build-files/nginx.conf /etc/nginx/nginx.conf +COPY docker-build-files/entrypoint.sh /entrypoint.sh + +RUN chmod +x /entrypoint.sh +ENTRYPOINT /entrypoint.sh $ENGINE_DB_HOST $ENGINE_DB_PASS diff --git a/README.md b/README.md index 6989c72d..b835646f 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,13 @@ AIS provides a unified view of City data for an address. - integration environment for address-centric data - API +## Production Processes +This AIS repository is used in 2 distinct ways. +1. Built into a docker container and pushed to AWS ECR where it will run in ECS. This is done using the 'build-test-compose.yml' docker-compose file, and should be programmatically done in the Github Actions file [build_and_deploy.yml](.github/workflows/build_and_deploy.yml) +If the action fails, you can troubleshoot manually by simply following the steps laid out in build_and_deploy.yml on our production AIS build server. + +2. Installed and run directly on our production AIS build server, where we run various build engine scripts to create database tables from various sources. These tables are then pushed up to our AWS RDS PostgreSQL instances for use by our ECS cluster. + ## Development To develop locally: @@ -31,3 +38,55 @@ To develop locally: 6. Create an empty file at `/ais/instance/config.py`. To run engine scripts, you'll need to add dictionary to this called `DATABASE` mapping database names to connection strings. (TODO: commit sample instance config) 7. Rename `.env.sample` in the root directory to `.env` and add real environment settings. (TODO: commit `.env.sample`) 8. `honcho start`. This will start start serving over port 5000. Note that this is blocked on CityNet, so you'll have to be on a public network to access `http://0.0.0.0:5000`. + +## Docker Container Dev + +For building the docker container, you'll need some environment/build arg variables first. Copy the example .env file used with docker-compose and populate it: + +1. `cp env.example .env && chmod +x .env` +2. populate it, set the $ENGINE_DB_HOST var to your database CNAME or IP. Note that in our build deploy process at citygeo this is done automatically and is not needed. + +Check to make sure docker-compose is populating your args: + +1. docker-compose -f build-test-compose.yml config + +Note that you may need to set ENGINE_DB_HOST to a direct IP instead of a CNAME to get it working in-office. +Now run the 'pull-private-passyunkdata.sh' script to download CSVs needed in the DockerFile. + +2. `chmod +x pull-private-passyunkdata.sh; ./pull-private-passyunkdata.sh` + +Then build and start the container. + +3. Via docker-compose: `docker-compose -f build-test-compose.yml up --build -d` + 1. Directly: +``` +docker build -t ais:latest . +docker run -itd --name ais -p 8080:8080 -e ENGINE_DB_HOST=$ENGINE_DB_HOST -e ENGINE_DB_PASS= $ENGINE_DB_PASS ais:latest` +``` + +If the container could successfully contact the DB then it should stay up and running. You may now run tests to confirm functionality. + +4. `docker exec ais bash -c 'cd /ais && . ./env/bin/activate && pytest /ais/ais/api/tests/'` + +## Testing +The API and the Engine can be tested separately using pytest after sourcing the virtual environment `venv`. +**Important Note** If you want to run pytests against a locally running database, you can either set your ENGINE_DB_HOST and ENGINE_DB_PASS parameters to the local instance and password, OR you can export DEV_TEST='true' to have this automatically use the local creds, as specified in your .env file. + +```bash +export DEV_TEST='true' +pytest $WORKING_DIRECTORY/ais/tests/engine -vvv -ra --showlocals --tb=native --disable-warnings --skip=$skip_engine_tests + +pytest $WORKING_DIRECTORY/ais/tests/api -vvv -ra --showlocals --tb=native --disable-warnings --skip=$skip_api_tests +``` + +To make direct queries using AIS, you can run the following on the dev box: + +``` +source ~/.env/bin/activate +export DEV_TEST='true' +gunicorn application --bind 0.0.0.0:8080 --workers 4 --worker-class gevent --access-logfile '-' --log-level 'notice' +curl localhost:8080/search/1234%20Market%20Street | jq . + +``` + +For reasons currently unknown, the `tests/api/test_views.py` cannot be tested on their own -- almost all the tests will fail with a 404 Response Error -- so all `api` tests must be run simultaneously. diff --git a/ais/__init__.py b/ais/__init__.py index 9ca53323..f1c35492 100644 --- a/ais/__init__.py +++ b/ais/__init__.py @@ -1,26 +1,73 @@ -from flask import Flask -from flask_cachecontrol import FlaskCacheControl +import os +from flask import Flask, g from flask_cors import CORS from flask_sqlalchemy import SQLAlchemy -from flask_script import Manager -from flask_migrate import Migrate, MigrateCommand -# from flasgger import Swagger, MK_SANITIZER +from flask_migrate import Migrate +from dotenv import load_dotenv +import dns.resolver +pardir = os.path.abspath('..') + +# load from .env file +try: + print('Loading the .env file, will use database info if found there...') + load_dotenv() +except Exception: + load_dotenv(pardir + '/ais/.env') # Create app instance -app = Flask(__name__, instance_relative_config=True) +app = Flask('__name__') +# load non-sensitive configurations from config.py +app.config.from_object('config') +# load sensitive configurations from instance/config.py +# Note these will both be imported in app.config so don't have any conflicting values +# in either that will overwrite the other. +# reference "Instance Folders": https://flask.palletsprojects.com/en/2.3.x/config/ -# Allow cross-origin requests -CORS(app) +# First import in the sensitive secrets from the "instance" folder +# Config path will be here if run in our built Docker image +if os.path.isfile('/ais/instance/config.py'): + print(f'Loading /ais/instance/config.py as our secrets instance config..') + app.config.from_pyfile('/ais/instance/config.py') +# Otherwise, default to the current working directory which should work when ais is installed as a package locally. +else: + print(f'Loading {os.getcwd() + "/config.py"} as our secrets instance config..') + app.config.from_pyfile('instance/config.py') -# Allow caching of responses -FlaskCacheControl(app) +# Then import non-sensitive things from regular config.py. +# config path will be here if run in our built Docker image +if os.path.isfile('/ais/config.py'): + print(f'Loading /ais/config.py as our Flask config..') + app.config.from_pyfile('/ais/config.py') +# Otherwise, default to the current working directory which should work when ais is installed as a package locally. +else: + print(f'Loading {os.getcwd() + "/config.py"} as our Flask config..') + app.config.from_pyfile(os.getcwd() + '/config.py') + -# Load default config -app.config.from_object('config') +# Assert we were passed an ENGINE_DB_HOST +assert os.environ['ENGINE_DB_HOST'], 'Please set ENGINE_DB_HOST in an environment variable!' +assert os.environ['ENGINE_DB_PASS'], 'Please set ENGINE_DB_PASS in an environment variable!' +db_host = os.environ['ENGINE_DB_HOST'] +db_pass = os.environ['ENGINE_DB_PASS'] + +print(f'DB host passed: {db_host}') + +# Debug print if we got our creds as env variables (necessary for how we run it in docker/ECS) +# format is: "postgresql://postgres:postgres@localhost/DBNAME" +app.config['SQLALCHEMY_DATABASE_URI'] = f'postgresql://ais_engine:{db_pass}@{db_host}/ais_engine' +# Init database extension +app_db = SQLAlchemy(app) -# Patch config with instance values -app.config.from_pyfile('config.py') +# Close database sessions in case our app is killed. +@app.teardown_appcontext +def teardown_app_context(exception=None): + db = getattr(g, '_database', None) + if db is not None: + db.session.remove() + +# Allow cross-origin requests +CORS(app) # Add profiler to app, if configured if app.config.get('PROFILE', False): @@ -31,19 +78,5 @@ from raven.contrib.flask import Sentry sentry = Sentry(app, dsn=app.config['SENTRY_DSN']) -# Init database extension -app_db = SQLAlchemy(app) - -# Init manager and register commands -manager = Manager(app) -manager.add_command('db', MigrateCommand) - -# Import engine manager here to avoid circular imports -from ais.engine.manage import manager as engine_manager -manager.add_command('engine', engine_manager) - # Init migration extension migrate = Migrate(app, app_db) - -# # Swaggerify App -# Swagger(app, sanitizer=MK_SANITIZER) diff --git a/ais/api/serializers.py b/ais/api/serializers.py index 3cc83a0c..c5558850 100644 --- a/ais/api/serializers.py +++ b/ais/api/serializers.py @@ -1,5 +1,6 @@ import json -from collections import OrderedDict, Iterable +from collections import OrderedDict +from collections.abc import Iterable from geoalchemy2.shape import to_shape from ais import app, util #, app_db as db from ais.models import Address, ENGINE_SRID diff --git a/ais/api/views.py b/ais/api/views.py index c8b865c9..9b55e368 100644 --- a/ais/api/views.py +++ b/ais/api/views.py @@ -7,7 +7,7 @@ from collections import OrderedDict from itertools import chain from flask import Response, request, redirect, url_for -from flask_cachecontrol import cache_for +from flask_cachecontrol import cache_for, ResponseIsSuccessfulOrRedirect # from flasgger.utils import swag_from from geoalchemy2.shape import to_shape from geoalchemy2.functions import ST_Transform @@ -132,7 +132,7 @@ def unmatched_response(**kwargs): @app.route('/unknown/') -@cache_for(hours=1) +@cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) def unknown_cascade_view(**kwargs): query = kwargs.get('query') @@ -164,12 +164,10 @@ def unknown_cascade_view(**kwargs): error = json_error(404, 'Could not find any addresses matching query.', {'query': query, 'normalized': normalized_address, 'search_type': search_type}) return json_response(response=error, status=404) - # return unmatched_response(query=query, parsed=parsed, normalized_address=normalized_address, - # search_type=search_type, address=address) # CASCADE TO STREET SEGMENT cascadedseg = StreetSegment.query \ - .filter_by_seg_id(seg_id) #if seg_id else None + .filter_by_seg_id(seg_id) cascadedseg = cascadedseg.first() @@ -177,17 +175,11 @@ def unknown_cascade_view(**kwargs): error = json_error(404, 'Could not find any addresses matching query.', {'query': query, 'normalized': normalized_address, 'search_type': search_type}) return json_response(response=error, status=404) - # return unmatched_response(query=query, parsed=parsed, normalized_address=normalized_address, - # search_type=search_type, address=address) # Get address side of street centerline segment seg_side = "R" if cascadedseg.right_from % 2 == address.address_low % 2 and cascadedseg.right_to != 0 else "L" # Check if address low num is within centerline seg full address range with parity: from_num, to_num = (cascadedseg.right_from, cascadedseg.right_to) if seg_side == "R" else (cascadedseg.left_from, cascadedseg.left_to) - # if not from_num <= address.address_low <= to_num: - # error = json_error(404, 'Address number is out of range.', - # {'query': query, 'normalized': normalized_address, 'search_type': search_type}) - # return json_response(response=error, status=404) # Get geom from true_range view item with same seg_id true_range_stmt = ''' @@ -205,8 +197,6 @@ def unknown_cascade_view(**kwargs): side_delta = true_range_result[1] - true_range_result[0] cascade_geocode_type = 'true_range' else: - # side_delta = cascadedseg.right_to - cascadedseg.right_from if seg_side == "R" \ - # else cascadedseg.left_to - cascadedseg.left_from side_delta = to_num - from_num cascade_geocode_type = 'full_range' if side_delta == 0: @@ -239,16 +229,13 @@ def unknown_cascade_view(**kwargs): addresses = (address,) paginator = Paginator(addresses) - #addresses_count = paginator.collection_size # Validate the pagination page_num, error = validate_page_param(request, paginator) if error: - # return json_response(response=error, status=error['status']) return json_response(response=error, status=404) srid = request.args.get('srid') if 'srid' in request.args else config['DEFAULT_API_SRID'] - # crs = {'type': 'name', 'properties': {'name': 'EPSG:{}'.format(srid)}} crs = {'type': 'link', 'properties': {'type': 'proj4', 'href': 'http://spatialreference.org/ref/epsg/{}/proj4/'.format(srid)}} # Render the response @@ -266,14 +253,12 @@ def unknown_cascade_view(**kwargs): sa_data=sa_data ) result = serializer.serialize_many(addresses_page) - # result = serializer.serialize_many(addresses_page) if addresses_count > 1 else serializer.serialize(next(addresses_page)) return json_response(response=result, status=200) @app.route('/addresses/') -@cache_for(hours=1) -# @swag_from('docs/addresses.yml') +@cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) def addresses(query): """ Looks up information about the address given in the query. Response is an @@ -319,7 +304,6 @@ def addresses(query): try: parsed = PassyunkParser(MAX_RANGE=int(max_range)).parse(query) - # parsed = PassyunkParser().parse(query) search_type = parsed['type'] normalized_address = parsed['components']['output_address'] except: @@ -341,18 +325,8 @@ def addresses(query): addr_num = str(low_num) + '-' + str(high_num) if high_num else low_num base_address_no_num_suffix = '{} {}'.format(addr_num, street_full) search_type = parsed['type'] - # loose_filters = OrderedDict([ - # # ('seg_id',int(parsed['components']['cl_seg_id'])), - # ('street_name',parsed['components']['street']['name']), - # ('address_low',low_num if low_num is not None else full_num), - # ('address_low_suffix',parsed['components']['address']['addr_suffix']), - # ('address_low_frac',parsed['components']['address']['fractional']), - # ('street_predir',parsed['components']['street']['predir']), - # ('street_postdir',parsed['components']['street']['postdir']), - # ('street_suffix',parsed['components']['street']['suffix']), - # ]) + loose_filters = OrderedDict([ - # ('seg_id',int(parsed['components']['cl_seg_id'])), ('seg_id',seg_id), ('address_low',low_num if low_num is not None else full_num), ('address_low_suffix',parsed['components']['address']['addr_suffix']), @@ -556,7 +530,7 @@ def process_query(addresses, match_type): @app.route('/block/') -@cache_for(hours=1) +@cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) # @swag_from('docs/block.yml') def block(query): """ @@ -646,7 +620,7 @@ def block(query): @app.route('/owner/') -@cache_for(hours=1) +@cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) # @swag_from('docs/owner.yml') def owner(query): query = query.strip('/') @@ -700,7 +674,7 @@ def owner(query): @app.route('/account/') -@cache_for(hours=1) +@cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) # @swag_from('docs/account.yml') def account(query): """ @@ -763,7 +737,7 @@ def account(query): @app.route('/pwd_parcel/') -@cache_for(hours=1) +@cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) # @swag_from('docs/pwd_parcel.yml') def pwd_parcel(query): """ @@ -833,7 +807,7 @@ def pwd_parcel(query): @app.route('/dor_parcel/') -@cache_for(hours=1) +@cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) # @swag_from('docs/mapreg.yml') def dor_parcel(query): """ @@ -891,7 +865,7 @@ def dor_parcel(query): @app.route('/intersection/') -@cache_for(hours=1) +@cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) # @swag_from('docs/intersection.yml') def intersection(query): ''' @@ -1010,7 +984,7 @@ def intersection(query): @app.route('/reverse_geocode/') -@cache_for(hours=1) +@cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) # @swag_from('docs/reverse_geocode.yml') def reverse_geocode(query): @@ -1135,7 +1109,7 @@ def reverse_geocode(query): @app.route('/service_areas/') -@cache_for(hours=1) +@cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) # @swag_from('docs/service_areas.yml') def service_areas(query): @@ -1208,7 +1182,7 @@ def service_areas(query): #@app.route('/street/') -#@cache_for(hours=1) +#@cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) def street(query): error = json_error(404, 'Could not find any addresses matching query.', {'query': query}) @@ -1222,7 +1196,7 @@ def street(query): # return response @app.route('/search/') -@cache_for(hours=1) +@cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) # @swag_from('docs/search.yml') def search(query): """ diff --git a/ais/commands.py b/ais/commands.py new file mode 100644 index 00000000..34302a0b --- /dev/null +++ b/ais/commands.py @@ -0,0 +1,13 @@ +import click +from ais.engine.commands import engine + + +@click.group() +def cli(): + pass + +# for whatever reaosn, setting the console_scripts in setup.py to point +# at this script sets __name__ to this. +if __name__ == 'ais.commands': + cli.add_command(engine) + cli() diff --git a/ais/engine/bin/ais-utils.sh b/ais/engine/bin/ais-utils.sh new file mode 100644 index 00000000..3ad1c371 --- /dev/null +++ b/ais/engine/bin/ais-utils.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# v2 functions for running with jenkins and +# the modern ECS/Fargate cluster implementation + +send_teams() { + TEXT=$(echo $1) + # env var from config-secrets.sh + WEBHOOK_URL=$TEAMS_WEBHOOK_URL + MESSAGE=$( echo ${TEXT} | sed 's/"/\"/g' | sed "s/'/\'/g" ) + JSON="{\"text\": \"
${MESSAGE}<\/pre>\" }"
+    curl -H "Content-Type: application/json" -d "${JSON}" "${WEBHOOK_URL}" -s > /dev/null
+}
+
+
+get_prod_env() {
+    # dig against AWS resolvers doesn't work in-office
+    #prod_lb_cname=$(dig ais-prod.citygeo.phila.city +short | grep -o "blue\|green")
+
+    # determine our prod env by checking what cnames are in our prod records in our hosted zone
+    # and then grepping for either blue or green.
+    # The hosted-zone-id is for our citygeo.phila.city hosted zone in AWS.
+    prod_lb_color=$(aws route53 list-resource-record-sets \
+                    --hosted-zone-id $PHILACITY_ZONE_ID \
+                    --query "ResourceRecordSets[?Name == '$PROD_ENDPOINT.']" | grep -o "blue\|green"
+    )
+
+    # Return either blue or green string
+    echo $prod_lb_color
+}
+
diff --git a/ais/engine/bin/build_and_deploy.sh b/ais/engine/bin/build_and_deploy.sh
new file mode 100755
index 00000000..1a5839f2
--- /dev/null
+++ b/ais/engine/bin/build_and_deploy.sh
@@ -0,0 +1,699 @@
+#!/usr/bin/env bash
+# V2 worked on by Roland
+
+# exit when any command fails
+set -e
+# Debug bash output (prints every command run)
+#set -x
+
+# Accept tests to skip
+while [[ $# -gt 0 ]] && [[ "$1" == "--"* ]] ;
+do
+    opt="$1";
+    shift;              #expose next argument
+    case "$opt" in
+        "--" ) break 2;;
+        "--skip-api-tests" )
+           skip_api_tests=="$1"; shift;;
+        "--skip-api-tests="* )
+           skip_api_tests=="${opt#*=}";;
+        "--skip-engine-tests" )
+           skip_engine_tests="$1"; shift;;
+        "--skip-engine-tests="* )
+           skip_engine_tests="${opt#*=}";;
+        *) echo >&2 "Invalid option: $@"; exit 1;;
+   esac
+done
+
+
+# Enable ERR trap inheritance
+set -o errtrace
+
+# Cleanup command that will run if something in the script fails.
+function cleanup {
+    echo "Error! Exited prematurely at $BASH_COMMAND!!"
+    echo "running reenable_taskin_alarm.."
+    reenable_taskin_alarm
+}
+trap cleanup ERR
+
+
+WORKING_DIRECTORY=/home/ubuntu/ais
+LOG_DIRECTORY=$WORKING_DIRECTORY/ais/engine/log
+cd $WORKING_DIRECTORY
+echo "Working directory is $WORKING_DIRECTORY"
+
+git_commit=$(git rev-parse HEAD)
+git_branch=$(git rev-parse --abbrev-ref HEAD)
+echo "Current git commit id is: $git_commit, branch: $git_branch"
+
+# Has our send_teams and get_prod_env functions
+source $WORKING_DIRECTORY/ais/engine/bin/ais-utils.sh
+source $WORKING_DIRECTORY/ais/engine/bin/ais-config.sh
+
+# dump location used in mutltiple functions, so export it.
+export DB_DUMP_FILE_LOC=$WORKING_DIRECTORY/ais/engine/backup/ais_engine.dump
+# Remove it to start fresh and save disk space
+rm -f $DB_DUMP_FILE_LOC
+
+# NOTE: postgres connection information is also stored in ~/.pgpass!!!
+# postgres commands should use passwords in there depending on the hostname.
+
+datestamp=$(date +%Y-%m-%d)
+start_dt=$(date +%Y%m%d%T)
+echo "Started: "$start_dt
+
+
+use_exit_status() {
+    # Use exit status to send correct success/failure message to Teams and console
+    # $1 int - exit status, typically of last process via $?
+    # $2 text - failure message to echo and send to Teams
+    # $3 text - success message to echo and send to Teams
+
+    if [ $1 -ne 0 ]
+    then
+        echo "$2"
+        send_teams "$2"
+        exit 1;
+    else
+        echo "$3"
+        send_teams "$3"
+    fi
+}
+
+
+check_for_prior_runs() {
+    kill_subfunction() {
+      # Find the pid of the process, if it's still running, and then kill it.
+      echo -e "\nChecking for still running '$1' processes..."
+      pids=( $(ps ax | grep "bash $1" | grep -v grep | awk '{ print $1 }') )
+      # Check the number of processes found.
+      # If it's greater than 2, it means multiple instances are running.
+      if [ ${#pids[@]} -gt 2 ]; then
+        echo "Multiple instances of '$1' are running."
+        # Kill all instances except for the current one.
+        for pid in "${pids[@]}"; do
+          if [ "$pid" != "$BASHPID" ]; then
+            echo "Killing process with PID: $pid"
+            kill $pid
+          fi
+        done
+      else
+        echo "'$1' is not currently running multiple instances."
+      fi
+    }
+    # Check for build_and_deploy.sh processes
+    kill_subfunction "build_and_deploy.sh"
+    # Also check for build_engine.sh commands that could potentially be running
+    # without an invoking build_and_deploy.sh also running. Stranger things have happened.
+    kill_subfunction "build_engine.sh"
+}
+
+
+activate_venv_source_libaries() {
+    if [ ! -d $WORKING_DIRECTORY/venv ]; then
+        echo -e "\nActivating/creating venv.."
+        python3.10 -m venv $WORKING_DIRECTORY/venv 
+        source $WORKING_DIRECTORY/venv/bin/activate
+        # Add the ais folder with our __init__.py so we can import it as a python module
+        export PYTHONPATH="${PYTHONPATH}:$WORKING_DIRECTORY/ais"
+        pip install wheel
+        # Add github to the list of known hosts so our SSH pip installs work later
+        ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
+        pip install -r $WORKING_DIRECTORY/requirements.txt
+    else
+        echo "Activating virtual environment"
+        source $WORKING_DIRECTORY/venv/bin/activate
+        # Add the ais folder with our __init__.py so we can import it as a python module
+        export PYTHONPATH="${PYTHONPATH}:$WORKING_DIRECTORY/ais"
+    fi
+}
+
+
+# not always a given
+ensure_passyunk_updated() {
+    echo -e "\nUpdating passyunk_automation specifically.."
+    file $WORKING_DIRECTORY/ssh-config
+    file $WORKING_DIRECTORY/passyunk-private.key
+    cp $WORKING_DIRECTORY/ssh-config ~/.ssh/config 
+    cp $WORKING_DIRECTORY/passyunk-private.key ~/.ssh/passyunk-private.key
+    pip install git+ssh://git@private-git/CityOfPhiladelphia/passyunk_automation.git
+    pip install git+https://github.com/CityOfPhiladelphia/passyunk
+}
+
+
+git_pull_ais_repo() {
+    cd $WORKING_DIRECTORY
+    git fetch
+    git pull
+    cd -
+}
+
+
+# CREATE ENGINE LOG FILES
+setup_log_files() {
+    mkdir -p $LOG_DIRECTORY
+    error_file="build_errors_"
+    out_file="build_log_"
+    error_file_loc=$LOG_DIRECTORY/$error_file$datestamp.txt
+    out_file_loc=$LOG_DIRECTORY/$out_file$datestamp.txt
+    warmup_lb_error_file_loc=$LOG_DIRECTORY/warmup_lb_error-$datestamp.txt
+    touch $error_file_loc
+    touch $out_file_loc
+    touch $warmup_lb_error_file_loc
+}
+
+
+# Check for and load credentials
+# This wil be needed for the engine build to pull in data.
+# config-secrets.sh contains AWS 
+check_load_creds() {
+    echo -e "\nLoading credentials and passwords into the environment"
+    python3 $WORKING_DIRECTORY/write-secrets-to-env.py
+    file $WORKING_DIRECTORY/config.py
+    file $WORKING_DIRECTORY/instance/config.py
+    file $WORKING_DIRECTORY/.env
+    source $WORKING_DIRECTORY/.env
+}
+
+
+# Make sure our creds for AWS are correct and account names match our profile names.
+check_aws_creds() {
+    file ~/.aws/credentials
+    # Default will be for our citygeo account
+    aws sts get-caller-identity --profile default | grep '880708401960'
+    aws sts get-caller-identity --profile mulesoft | grep '975050025792'
+}
+
+# Get AWS production environment
+identify_prod() {
+    echo -e "\nFinding the production environment via CNAME"
+    # export to environment var so it can be accessed by sub-python scripts run in this script.
+    export prod_color=$(get_prod_env || {
+      echo "Could not find the production environment" ;
+      exit 1 ;
+    })
+    echo "Production environment is: $prod_color"
+    if [[ "$prod_color" == "blue" ]]; then
+        export staging_color="green" 
+    else
+        export staging_color="blue"
+    fi
+
+    # dynamically retrieve ARNs and DNS information because resources could be dynamically re-made by terraform.
+    export staging_tg_arn=$(aws elbv2 describe-target-groups | grep "${staging_color}-tg" | grep TargetGroupArn| cut -d"\"" -f4)
+    export prod_tg_arn=$(aws elbv2 describe-target-groups | grep "${prod_color}-tg" | grep TargetGroupArn| cut -d"\"" -f4)
+
+    export prod_lb_uri=$(aws elbv2 describe-load-balancers --names ais-${prod_color}-api-alb --query "LoadBalancers[*].DNSName" --output text)
+    export staging_lb_uri=$(aws elbv2 describe-load-balancers --names ais-${staging_color}-api-alb --query "LoadBalancers[*].DNSName" --output text)
+
+    export prod_db_uri=$(aws rds describe-db-instances --db-instance-identifier ais-engine-${prod_color} --query "DBInstances[*].Endpoint.Address" --output text)
+    export staging_db_uri=$(aws rds describe-db-instances --db-instance-identifier ais-engine-${staging_color} --query "DBInstances[*].Endpoint.Address" --output text)
+}
+
+
+# Clean up old docker images/containers so we don't run out of storage.
+cleanup_docker() {
+    # TEMP stop docker to conserve memory
+    # don't fail on this, so pipe to true
+    echo "Attempting to stop docker containers if they exist..."
+    docker stop ais 2>/dev/null || true
+    docker rm ais 2>/dev/null || true
+
+    # Cleanup any other containers that may or may not exist.
+    yes | docker system prune
+    yes | docker image prune
+}
+
+
+# RUN BUILD ENGINE
+# Note: you need to have your ais/instance/config.py populated
+# with database connection info for this to work!
+# See check_load_creds function.
+build_engine() {
+    echo -e "\nStarting new engine build"
+    send_teams "Starting new engine build."
+    bash $WORKING_DIRECTORY/ais/engine/bin/build_engine.sh > >(tee -a $out_file_loc) 2> >(tee -a $error_file_loc >&2)
+    send_teams "Engine build has completed."
+    end_dt=$(date +%Y%m%d%T)
+    echo "Time Summary: "
+    echo "Started: "$start_dt
+    echo "Finished: "$end_dt
+}
+
+
+engine_tests() {
+    echo -e "\nRunning engine tests, comparing local build tables against what is currently in prod RDS ($prod_db_uri).."
+    # Set these so it'll use the prod RDS instance
+
+    # Compare prod against local
+    export ENGINE_TO_TEST='localhost'
+    export ENGINE_TO_COMPARE=$prod_db_uri
+
+    # Unused by the engine pytests, but required by ais/__init__.py to start up ais at all.
+    export ENGINE_DB_HOST=$ENGINE_TO_TEST
+
+    export ENGINE_DB_PASS=$RDS_ENGINE_DB_PASS
+    cd $WORKING_DIRECTORY
+    pytest $WORKING_DIRECTORY/ais/tests/engine -vvv -ra --showlocals --tb=native --disable-warnings --skip=$skip_engine_tests
+    use_exit_status $? "Engine tests failed" "Engine tests passed"
+    # unset
+}
+
+
+api_tests() {
+    echo -e "\nRunning api_tests..."
+    cd $WORKING_DIRECTORY
+    # Set these so it'll use our local build for API tests
+    export ENGINE_DB_HOST='localhost'
+    export ENGINE_DB_PASS=$LOCAL_ENGINE_DB_PASS
+    pytest $WORKING_DIRECTORY/ais/tests/api -vvv -ra --showlocals --tb=native --disable-warnings --skip=$skip_api_tests 
+    use_exit_status $? "API tests failed" "API tests passed"
+}
+
+
+# Make a copy (Dump) the newly built local engine db
+dump_local_db() {
+    echo -e "\nDumping the newly built engine database to $DB_DUMP_FILE_LOC"
+    send_teams "\nDumping the newly built engine database to $DB_DUMP_FILE_LOC"
+    export PGPASSWORD=$LOCAL_ENGINE_DB_PASS
+    mkdir -p $WORKING_DIRECTORY/ais/engine/backup
+    pg_dump -Fcustom -Z0 --create --clean -U ais_engine -h localhost -n public ais_engine > $DB_DUMP_FILE_LOC
+    use_exit_status $? "DB dump failed" "DB dump succeeded"
+}
+
+
+restart_staging_db() {
+    echo -e "\nRestarting RDS instance: $staging_db_uri"
+    echo "********************************************************************************************************"
+    echo "Please make sure the RDS instance identifier names are set to 'ais-engine-green' and 'ais-engine-blue'!!"
+    echo "We reboot the RDS instance by those names with the AWS CLI commmand 'aws rds reboot-db-instance'."
+    echo "********************************************************************************************************"
+    if [[ "$prod_color" == "blue" ]]; then
+        local stage_instance_identifier="ais-engine-green"
+        aws rds reboot-db-instance --region "us-east-1" --db-instance-identifier "ais-engine-green" --no-cli-pager | grep "DBInstanceStatus"
+    else
+        local stage_instance_identifier="ais-engine-blue"
+        aws rds reboot-db-instance --region "us-east-1" --db-instance-identifier "ais-engine-blue" --no-cli-pager | grep "DBInstanceStatus"
+    fi
+
+    # Check to see if the instance is ready
+    local max_attempts=30
+    local attempt=1
+    while [ $attempt -le $max_attempts ]; do
+        instance_status=$(aws rds describe-db-instances --region "us-east-1" \
+          --db-instance-identifier "$stage_instance_identifier" \
+          --query "DBInstances[0].DBInstanceStatus" --output text --no-cli-pager)
+
+        if [ "$instance_status" = "available" ]; then
+            echo "RDS instance is ready!"
+            break
+        fi
+
+        echo "Waiting for RDS instance to be ready... (Attempt: $attempt/$max_attempts)"
+        sleep 10
+        ((attempt++))
+    done
+
+    if [ $attempt -gt $max_attempts ]; then
+        echo "RDS instance did not become ready within the expected time."
+        exit 1
+    fi
+}
+
+
+# Check to see if the RDS instance is in an "available" state.
+check_rds_instance() {
+    # Initial sleep of 6 minutes because we're seeing the instance be available
+    # and then suddenly in a modifying state, so parameter group modification can take longer than we expect.
+    echo 'Checking RDS instance status..'
+    sleep 300
+
+    if [[ "$prod_color" == "blue" ]]; then
+        local stage_instance_identifier="ais-engine-green"
+    else
+        local stage_instance_identifier="ais-engine-blue"
+    fi
+
+    local max_attempts=90
+    local attempt=1
+    while [ $attempt -le $max_attempts ]; do
+        instance_status=$(aws rds describe-db-instances --region "us-east-1" \
+          --db-instance-identifier "$stage_instance_identifier" \
+          --query "DBInstances[0].DBInstanceStatus" --output text --no-cli-pager)
+
+        if [ "$instance_status" = "available" ]; then
+            echo "RDS instance is ready!"
+            break
+        fi
+
+        echo "Waiting for RDS instance to be ready... (Attempt: $attempt/$max_attempts)"
+        sleep 10
+        ((attempt++))
+    done
+
+    if [ $attempt -gt $max_attempts ]; then
+        echo "RDS instance did not become ready within the expected time."
+        exit 1
+    fi
+}
+
+
+modify_stage_scaling_out() {
+    # either disable or enable
+    action=$1
+
+    # 1 disable staging taskout action that scales out containers
+    if [[ "$action" == 'disable' ]]; then
+        echo 'Disabling ECS tasks and scale out..'
+        # 1. disable staging taskout action that scales out containers
+        aws cloudwatch disable-alarm-actions --alarm-names ais-${staging_color}-api-taskout
+        # 2. Set desired tasks to 0 so they don't blow up the db with health checks while restoring.
+        aws ecs update-service --cluster ais-${staging_color}-cluster --service ais-${staging_color}-api-service --desired-count 0
+    elif [[ "$action" == 'enable' ]]; then
+        echo 'Reenabling ECS tasks and scale out..'
+        aws cloudwatch enable-alarm-actions --alarm-names ais-${staging_color}-api-taskout
+        # Must allow back at least 1 instance so our later checks on the target groups works.
+        aws ecs update-service --cluster ais-${staging_color}-cluster --service ais-${staging_color}-api-service --desired-count 2
+    fi
+}
+
+
+# Update (Restore) AWS RDS instance to staging database
+# Note: you can somewhat track restore progress by looking at the db size:
+#SELECT pg_size_pretty( pg_database_size('ais_engine') );
+restore_db_to_staging() {
+    echo -e "\nRunning restore_db_to_staging.."
+    echo "Restoring the engine DB to $staging_db_uri"
+    send_teams "Restoring the engine DB to $staging_db_uri"
+
+    export PGPASSWORD=$RDS_SUPER_ENGINE_DB_PASS
+    db_pretty_size=$(psql -U postgres -h $staging_db_uri -d ais_engine -AXqtc "SELECT pg_size_pretty( pg_database_size('ais_engine') );")
+    echo "Database size before restore: $db_pretty_size"
+
+
+    # Get production color
+    if [[ "$prod_color" == "blue" ]]; then
+        local stage_instance_identifier="ais-engine-green"
+    else
+        local stage_instance_identifier="ais-engine-blue"
+    fi
+
+    #######################
+    # First let's modify parameters of the DB so restores go a bit faster
+
+    # Commands to create custom restore parameter group
+    #aws rds create-db-parameter-group --db-parameter-group-name ais-restore-parameters --db-parameter-group-family postgres12 --description "Params to speed up restore, DO NOT USE FOR PROD TRAFFIC"
+
+    # Change RDS instance to use faster but less "safe" restore parameters
+    # Unfortunately RDS does not allow us to modify "full_page_writes" which would definitely speed up restoring.
+    # loosely based off https://www.databasesoup.com/2014/09/settings-for-fast-pgrestore.html
+    # and https://stackoverflow.com/a/75147585
+
+    # This command actually modifies the parameter group "ais-restore-parameters" each time. Just nice to have the changes it makes explicitly in code.
+    #aws rds modify-db-parameter-group \
+    #    --db-parameter-group-name ais-restore-parameters \
+    #    --parameters "ParameterName=max_wal_size,ParameterValue=5120,ApplyMethod='immediate'" \
+    #    --parameters "ParameterName=max_wal_senders,ParameterValue=0,ApplyMethod='immediate'" \
+    #    --parameters "ParameterName=wal_keep_segments,ParameterValue=0,ApplyMethod='immediate'" \
+    #    --parameters "ParameterName=autovacuum,ParameterValue=off,ApplyMethod='immediate'" \
+    #    --parameters "ParameterName=shared_buffers,ParameterValue='{DBInstanceClassMemory/65536}',ApplyMethod='pending-reboot'" \
+    #    --parameters "ParameterName=synchronous_commit,ParameterValue=off,ApplyMethod='immediate'" \
+    #    --no-cli-pager
+
+    # modify stage rds to use restore parameter group
+    #aws rds modify-db-instance --db-instance-identifier $stage_instance_identifier --db-parameter-group-name ais-restore-parameters --apply-immediately --no-cli-pager
+
+    # Wait for instance status to be "available" and not "modifying".
+    check_rds_instance $stage_instance_identifier
+
+    # Manually drop and recreate the schema, mostly because extension recreates aren't included in a pg_dump
+    # We need to make extensions first to get shape field functionality, otherwise our restore won't work.
+    export PGPASSWORD=$RDS_SUPER_ENGINE_DB_PASS
+    psql -U postgres -h $staging_db_uri -d ais_engine -c "DROP SCHEMA IF EXISTS public CASCADE;"
+    # Recreate as ais_engine otherwise things get angry
+    export PGPASSWORD=$RDS_ENGINE_DB_PASS
+    psql -U ais_engine -h $staging_db_uri -d ais_engine -c "CREATE SCHEMA public;"
+    psql -U ais_engine -h $staging_db_uri -d ais_engine -c "GRANT ALL ON SCHEMA public TO postgres;"
+    psql -U ais_engine -h $staging_db_uri -d ais_engine -c "GRANT ALL ON SCHEMA public TO public;"
+
+    # Extensions can only be re-installed as postgres superuser
+    export PGPASSWORD=$RDS_SUPER_ENGINE_DB_PASS
+    psql -U postgres -h $staging_db_uri -d ais_engine -c "CREATE EXTENSION postgis WITH SCHEMA public;"
+    psql -U postgres -h $staging_db_uri -d ais_engine -c "CREATE EXTENSION pg_trgm WITH SCHEMA public;"
+    psql -U postgres -h $staging_db_uri -d ais_engine -c "GRANT ALL ON TABLE public.spatial_ref_sys TO ais_engine;"
+
+    # Will have lots of errors about things not existing during DROP statements because of manual public schema drop & remake but will be okay.
+    export PGPASSWORD=$RDS_ENGINE_DB_PASS
+    echo "Beginning restore with file $DB_DUMP_FILE_LOC, full command is:"
+    echo "time pg_restore -v -j 6 -h $staging_db_uri -d ais_engine -U ais_engine -c $DB_DUMP_FILE_LOC || true"
+    # Store output so we can determine if errors are actually bad
+    restore_output=$(time pg_restore -v -j 6 -h $staging_db_uri -d ais_engine -U ais_engine -c $DB_DUMP_FILE_LOC || true)
+    #echo $restore_output | grep 'errors ignored on restore'
+    sleep 10
+
+    # Check size after restore
+    export PGPASSWORD=$RDS_SUPER_ENGINE_DB_PASS
+    db_pretty_size=$(psql -U postgres -h $staging_db_uri -d ais_engine -AXqtc "SELECT pg_size_pretty( pg_database_size('ais_engine') );")
+    echo "Database size after restore: $db_pretty_size"
+    send_teams "Database size after restore: $db_pretty_size"
+
+    # Assert the value is greater than 8000 MB
+    db_size=$(echo $db_pretty_size | awk '{print $1}')
+    # Checking if the numeric value is less than 8000
+    if [ "$db_size" -lt 8000 ]; then
+        echo "Database size after restore is less than 8 GB!!"
+        exit 1
+    fi
+
+    # After restore, switch back to default RDS parameter group
+    #aws rds modify-db-instance --db-instance-identifier $stage_instance_identifier --db-parameter-group-name default.postgres12 --apply-immediately --no-cli-pager
+    sleep 60
+
+    # Wait for instance status to be "available" and not "modifying" or "backing-up". Can be triggered by restores it seems.
+    check_rds_instance $stage_instance_identifier
+
+    sleep 60
+}
+
+
+engine_tests_for_restored_rds() {
+    echo -e "Running engine tests, testing $staging_db_uri and comparing to $prod_db_uri.."
+    # Set these so it'll use the prod RDS instance
+
+    # Compare stage against prod
+    export ENGINE_TO_TEST=$staging_db_uri
+    export ENGINE_TO_COMPARE=$prod_db_uri
+
+    # Unused by the engine pytests, but required by ais/__init__.py to start up ais at all.
+    export ENGINE_DB_HOST=$ENGINE_TO_TEST
+    export ENGINE_DB_PASS=$RDS_ENGINE_DB_PASS
+
+    cd $WORKING_DIRECTORY
+    pytest $WORKING_DIRECTORY/ais/tests/engine -vvv -ra --showlocals --tb=native --disable-warnings --skip=$skip_engine_tests
+    use_exit_status $? "Engine tests failed" "Engine tests passed"
+    # unset
+}
+
+
+docker_tests() {
+    echo -e "\nRunning docker_tests, which pulls the docker image from the latest in ECR and runs tests.."
+    # Set these so it'll use the staging RDS instance
+    export ENGINE_DB_HOST=$staging_db_uri
+    export ENGINE_DB_PASS=$RDS_ENGINE_DB_PASS
+    # Login to ECR so we can pull the image, will  use our AWS creds sourced from .env
+    aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 880708401960.dkr.ecr.us-east-1.amazonaws.com
+
+    # Pull our latest docker image
+    docker pull 880708401960.dkr.ecr.us-east-1.amazonaws.com/ais:latest
+    # Spin up docker container from latest AIS image from ECR and test against staging database
+    # Note: the compose uses the environment variables for the database and password that we exported earlier
+    docker-compose -f ecr-test-compose.yml up --build -d
+    # Run API tests
+    docker exec ais bash -c "pytest /ais/ais/tests/api/ -vvv -ra --showlocals --tb=native --disable-warnings --skip=$skip_api_tests"
+}
+
+
+scale_up_staging() {
+    echo -e "\nRunning scale_up_staging..."
+    prod_tasks=$(aws ecs describe-clusters --clusters ais-${prod_color}-cluster | grep runningTasksCount | tr -s ' ' | cut -d ' ' -f3 | cut -d ',' -f1)
+    echo -e "\nCurrent running tasks in prod: $prod_tasks"
+    if (( $prod_tasks > 2 ))
+    then
+        # Must temporarily disable the alarm otherwise we'll get scaled back in seconds. We'll re-enable five minutes 
+        # after switching prod and staging.
+        aws cloudwatch disable-alarm-actions --alarm-names ais-${staging_color}-api-taskin
+        # throw in a sleep, can take a bit for the disable action to take effect.
+        sleep 15
+        aws ecs update-service --cluster ais-${staging_color}-cluster --service ais-${staging_color}-api-service --desired-count ${prod_tasks}
+        # Wait for cluster to be stable, e.g. all the containers are spun up.
+        # For changing from 2 to 5 instances (+3) in my experience it tooks about 3 minutes for the service to become stable.
+        aws ecs wait services-stable --cluster ais-${staging_color}-cluster \
+        --service ais-${staging_color}-api-service --region us-east-1 
+        # When we re-enable the alarm at the end of this entire script, the scalein alarm should
+        # start lowering the task count by 1 slowly based on the alarm 'cooldown' time.
+    else
+        echo "Staging has 2 running tasks, the minimum. Not running any scaling actions."
+    fi
+}
+
+
+# Once confirmed good, deploy latest AIS image from ECR to staging
+deploy_to_staging_ecs() {
+    echo -e "\nDeploying latest AIS image from ECR to $staging_color environment.."
+    # pipe to null because they're quite noisy
+    echo "running aws ecs update-service.."
+    aws ecs update-service --cluster ais-${staging_color}-cluster \
+    --service ais-${staging_color}-api-service --force-new-deployment --region us-east-1 \
+    1> /dev/null
+    echo "running aws ecs services-stable.."
+    aws ecs wait services-stable --cluster ais-${staging_color}-cluster \
+    --service ais-${staging_color}-api-service --region us-east-1 
+}
+
+
+# Check staging target group health
+check_target_health() {
+    echo -e "\nConfirming target group health.."
+    aws elbv2 describe-target-health --target-group-arn $staging_tg_arn | grep "\"healthy\"" 1> /dev/null
+}
+
+
+# Warm up load balancer against staging env?
+warmup_lb() {
+    echo -e "\nWarming up the load balancer for staging lb: $staging_color."
+    # Export creds again so this function can access them.
+    file $WORKING_DIRECTORY/.env
+    source $WORKING_DIRECTORY/.env
+    send_teams "Warming up the load balancer for staging lb: $staging_color."
+    python $WORKING_DIRECTORY/ais/engine/bin/warmup_lb.py --dbpass $LOCAL_ENGINE_DB_PASS --gatekeeper-key $GATEKEEPER_KEY
+    use_exit_status $? \
+        "AIS load balancer warmup failed.\nEngine build has been pushed but not deployed." \
+        "AIS load balancer warmup succeeded.\nEngine build has been pushed and deployed."
+}
+
+
+# Important step! Swaps the prod environments in Route 53!!
+swap_cnames() {
+    echo -e "\nSwapping prod/stage CNAMEs..."
+    # First let's swap the prod cname to our now ready-to-be-prod staging_lb_uri.
+    template='{
+  "Comment": "Modify %s ais record to %s",
+  "Changes": [{
+      "Action": "UPSERT",
+      "ResourceRecordSet": {
+        "Name": "%s.",
+        "Type": "CNAME",
+        "TTL": 5,
+        "ResourceRecords": [{ "Value": "%s" }]
+    }}]
+}'
+
+    # Swap the production DNS record to the ALB DNS we identified as staging
+    json_string=$(printf "$template" "prod" "$prod_color" "$PROD_ENDPOINT" "$staging_lb_uri")
+    echo "$json_string" > $WORKING_DIRECTORY/route53-temp-change.json
+    aws route53 change-resource-record-sets \
+        --hosted-zone-id $PHILACITY_ZONE_ID \
+        --profile default \
+        --change-batch file://$WORKING_DIRECTORY/route53-temp-change.json 1> /dev/null
+    json_string=$(printf "$template" "prod" "$prod_color" "$PROD_ENDPOINT" "$staging_lb_uri")
+    echo "$json_string" > $WORKING_DIRECTORY/route53-temp-change.json
+    aws route53 change-resource-record-sets \
+        --hosted-zone-id $MULESOFT_PHILACITY_ZONE_ID \
+        --profile mulesoft \
+        --change-batch file://$WORKING_DIRECTORY/route53-temp-change.json 1> /dev/null
+
+    # Swap the staging DNS record to the ALB DNS we identified as prod
+    json_string=$(printf "$template" "stage" "$staging_color" "$STAGE_ENDPOINT" "$prod_lb_uri")
+    echo "$json_string" > $WORKING_DIRECTORY/route53-temp-change.json
+    aws route53 change-resource-record-sets \
+        --hosted-zone-id $PHILACITY_ZONE_ID \
+        --profile default \
+        --change-batch file://$WORKING_DIRECTORY/route53-temp-change.json 1> /dev/null
+    json_string=$(printf "$template" "stage" "$staging_color" "$STAGE_ENDPOINT" "$prod_lb_uri")
+    echo "$json_string" > $WORKING_DIRECTORY/route53-temp-change.json
+    aws route53 change-resource-record-sets \
+        --hosted-zone-id $MULESOFT_PHILACITY_ZONE_ID \
+        --profile mulesoft \
+        --change-batch file://$WORKING_DIRECTORY/route53-temp-change.json 1> /dev/null
+    rm $WORKING_DIRECTORY/route53-temp-change.json
+
+    echo "Swapped prod cname to $staging_color successfully! Staging is now ${prod_color}."
+    send_teams "Swapped prod cname to $staging_color successfully! Staging is now ${prod_color}."
+}
+
+
+reenable_taskin_alarm() {
+    echo -e "\nSleeping for 5 minutes, then running scale-in alarm re-enable command..."
+    sleep 300
+    aws cloudwatch enable-alarm-actions --alarm-names ais-${staging_color}-api-taskin
+    echo "Alarm 'ais-${staging_color}-api-taskin' re-enabled."
+}
+
+
+# Runs various scripts that make necessary "report" tables off our built AIS tables that
+# aren't used by AIS.
+# These are used by other departments for various integrations, mainly related
+# to unique identifier for addresses. For example for DOR we call this PIN.
+make_reports_tables() {
+    echo -e "\nRunning engine make_reports.py script..."
+    #python $WORKING_DIRECTORY/ais/engine/bin/make_reports.py
+    send_teams "Making Reports..."
+    bash $WORKING_DIRECTORY/ais/engine/bin/make_reports.sh
+    send_teams "Reports have completed!"
+    
+}
+
+check_for_prior_runs
+
+activate_venv_source_libaries
+
+ensure_passyunk_updated
+
+setup_log_files
+
+check_load_creds
+
+git_pull_ais_repo
+
+check_aws_creds
+
+identify_prod
+
+cleanup_docker
+
+build_engine
+
+engine_tests
+
+api_tests
+
+dump_local_db
+
+modify_stage_scaling_out "disable"
+
+restart_staging_db
+
+restore_db_to_staging
+
+engine_tests_for_restored_rds
+
+modify_stage_scaling_out "enable"
+
+docker_tests
+
+scale_up_staging
+
+deploy_to_staging_ecs
+
+check_target_health
+
+warmup_lb
+
+swap_cnames
+
+reenable_taskin_alarm
+
+make_reports_tables
+
+echo "Finished successfully!"
diff --git a/ais/engine/bin/build_engine.sh b/ais/engine/bin/build_engine.sh
old mode 100644
new mode 100755
index 3d155297..21620c46
--- a/ais/engine/bin/build_engine.sh
+++ b/ais/engine/bin/build_engine.sh
@@ -1,160 +1,57 @@
 #!/usr/bin/env bash
 
-echo "Activating virtual environment"
-source ../../../env/bin/activate
-
-echo "Running the engine"
-
-echo "Loading Streets"
-ais engine run load_streets
-
-if [ $? -ne 0 ]
-then
-  echo "Loading table failed. Exiting."
-  exit 1;
-fi
-
-echo "Loading Street Aliases"
-ais engine run load_street_aliases
-
-if [ $? -ne 0 ]
-then
-  echo "Loading table failed. Exiting."
-  exit 1;
-fi
-
-echo "Making Intersections"
-ais engine run make_street_intersections
-
-if [ $? -ne 0 ]
-then
-  echo "Loading table failed. Exiting."
-  exit 1;
-fi
-
-echo "Loading OPA Properties"
-ais engine run load_opa_properties
-
-if [ $? -ne 0 ]
-then
-  echo "Loading table failed. Exiting."
-  exit 1;
-fi
-
-echo "Loading NG911 Address Points"
-ais engine run load_ng911_address_points
-
-if [ $? -ne 0 ]
-then
-  echo "Loading table failed. Exiting."
-  exit 1;
-fi
-
-echo "Loading DOR parcels"
-ais engine run load_dor_parcels
-
-if [ $? -ne 0 ]
-then
-  echo "Loading table failed. Exiting."
-  exit 1;
-fi
-
-echo "Loading DOR condos"
-ais engine run load_dor_condos
-
-if [ $? -ne 0 ]
-then
-  echo "Loading table failed. Exiting."
-  exit 1;
-fi
-
-echo "Loading PWD Parcels"
-ais engine run load_pwd_parcels
-
-
-if [ $? -ne 0 ]
-then
-  echo "Loading table failed. Exiting."
-  exit 1;
-fi
-
-echo "Loading Curbs"
-ais engine run load_curbs
-
-if [ $? -ne 0 ]
-then
-  echo "Loading table failed. Exiting."
-  exit 1;
-fi
-
-echo "Loading Addresses"
-ais engine run load_addresses
-
-if [ $? -ne 0 ]
-then
-  echo "Loading addresses failed. Exiting."
-  exit 1;
-fi
-
-#echo "Loading opa active accounts and matching pwd parcels for properties without pwd parcel match"
-#ais engine run get_pwd_matches_from_manual_opa_geocodes
-
-#if [ $? -ne 0 ]
-#then
-#  echo "Adding manual opa-pwd parcel matches failed. Exiting."
-#  exit 1;
-#fi
-
-echo "Geocoding Addresses"
-ais engine run geocode_addresses
-
-if [ $? -ne 0 ]
-then
-  echo "Geocoding addresses failed. Exiting."
-  exit 1;
-fi
-
-echo "Making Address Tags from Linked Addresses"
-ais engine run make_linked_tags
-
-if [ $? -ne 0 ]
-then
-  echo "Making address tags failed. Exiting."
-  exit 1;
-fi
-
-echo "Geocoding addresses from links"
-ais engine run geocode_addresses_from_links
-
-if [ $? -ne 0 ]
-then
-  echo "Geocoding addresses from links failed. Exiting."
-  exit 1;
-fi
-
-echo "Making Address Summary"
-ais engine run make_address_summary
-
-if [ $? -ne 0 ]
-then
-  echo "Making address summary failed. Exiting."
-  exit 1;
-fi
-
-echo "Loading Service Areas"
-ais engine run load_service_areas
-
-if [ $? -ne 0 ]
-then
-  echo "Loading service areas failed. Exiting."
-  exit 1;
-fi
-
-echo "Making Service Area Summary"
-ais engine run make_service_area_summary
-
-if [ $? -ne 0 ]
-then
-  echo "Making service area summary failed. Exiting."
-  exit 1;
-fi
+WORKING_DIRECTORY=/home/ubuntu/ais
+cd $WORKING_DIRECTORY
+
+export ORACLE_HOME=/usr/lib/oracle/18.5/client64
+export PATH=$PATH:$ORACLE_HOME/bin
+export LD_LIBRARY_PATH=$ORACLE_HOME/lib
+export PYTHONUNBUFFERED=TRUE
+echo "Setting DEV_TEST to true so we use the local database."
+export DEV_TEST="true"
+echo -e "\nActivating virtual environment"
+source $WORKING_DIRECTORY/venv/bin/activate
+# Add the ais folder with our __init__.py so we can import it as a python module
+export PYTHONPATH="${PYTHONPATH}:$WORKING_DIRECTORY/ais"
+
+source $WORKING_DIRECTORY/.env
+
+export ENGINE_DB_HOST="localhost"
+export ENGINE_DB_PASS=$LOCAL_ENGINE_DB_PASS
+
+echo "Running the engine build!"
+
+SCRIPTS=(
+  "load_streets" 
+  "load_street_aliases" 
+  "make_street_intersections"
+  "load_opa_properties"
+  "load_ng911_address_points"
+  "load_dor_parcels"
+  "load_dor_condos"
+  "load_pwd_parcels"
+  "load_curbs"
+  "load_addresses"
+  "geocode_addresses"
+  "make_linked_tags"
+  "geocode_addresses_from_links"
+  "make_address_summary"
+  "load_service_areas"
+  "make_service_area_summary"
+)
+
+run_script() {
+  echo ""
+  echo "********************************************************************************"
+  echo "Running script '$1'"
+  ais engine --script "$1"
+  if [[ $? -ne 0 ]]
+  then
+    echo "Loading table failed. Exiting."
+    exit 1;
+  fi
+}
+
+for script in "${SCRIPTS[@]}"; do
+  run_script $script
+done
diff --git a/ais/engine/bin/build_go.sh b/ais/engine/bin/build_go.sh
deleted file mode 100644
index f2911179..00000000
--- a/ais/engine/bin/build_go.sh
+++ /dev/null
@@ -1,168 +0,0 @@
-#!/usr/bin/env bash
-
-datestamp=$(date +%Y%m%d)
-start_dt=$(date +%Y%m%d%T)
-echo "Started: "$start_dt
-
-echo "Activating virtual environment"
-source ../../../env/bin/activate
-source ../../../bin/eb_env_utils.sh
-
-# GET LATEST CODE FROM GIT REPO
-git fetch origin && git pull
-cd ../../../env/src/passyunk
-git fetch origin && git pull
-cd -
-
-# CREATE ENGINE LOG FILES
-mkdir -p ../log
-error_file="build_errors_"
-out_file="build_log_"
-error_file_loc=../log/$error_file$datestamp.txt
-out_file_loc=../log/$out_file$datestamp.txt
-
-# RUN BUILD ENGINE
-echo "Building the engine."
-send_teams "Starting new engine build."
-bash build_engine.sh > >(tee -a $out_file_loc) 2> >(tee -a $error_file_loc >&2)
-send_teams "Engine build has completed."
-end_dt=$(date +%Y%m%d%T)
-echo "Time Summary: "
-echo "Started: "$start_dt
-echo "Finished: "$end_dt
-
-# Get AWS production environment
-echo "Finding the production environment"
-eb_prod_env=$(get_prod_env EB_PROD_ENV || {
-  echo "Could not find the production environment" ;
-  exit 1 ;
-})
-echo "Production environment is: "$eb_prod_env
-
-# Run tests
-echo "Running engine tests."
-send_teams "Running tests."
-error_file_loc=../log/pytest_engine_errors_$datestamp.txt
-out_file_loc=../log/pytest_engine_log_$datestamp.txt
-pytest ../tests/test_engine.py > >(tee -a $out_file_loc) 2> >(tee -a $error_file_loc >&2)
-if [ $? -ne 0 ]
-then
-  echo "Engine tests failed"
-  send_teams "Engine tests have failed."
-  exit 1;
-fi
-send_teams "Engine tests have passed."
-
-echo "Running API tests."
-error_file_loc=../log/pytest_api_errors_$datestamp.txt
-out_file_loc=../log/pytest_api_log_$datestamp.txt
-pytest ../../api/tests/  > >(tee -a $out_file_loc) 2> >(tee -a $error_file_loc >&2)
-if [ $? -ne 0 ]
-then
-  echo "API tests failed"
-  send_teams "API tests failed."
-  exit 1;
-fi
-send_teams "API tests passed."
-
-# Update (Restore) AWS RDS instance
-
-# Make a copy (Dump) the newly built local engine db
-echo "Copying the engine database."
-send_teams "Copying the engine database."
-mkdir -p ../backup
-db_dump_file_loc=../backup/ais_engine.dump
-pg_dump -Fc -U ais_engine -n public ais_engine > $db_dump_file_loc
-if [ $? -ne 0 ]
-then
-  echo "DB dump failed"
-  exit 1;
-fi
-
-# Get AWS staging environment
-echo "Finding the staging environment"
-eb_staging_env=$(get_staging_env EB_STAGING_ENV || {
-  echo "Could not find the staging environment" ;
-  exit 1 ;
-})
-echo "Staging environment is: "$eb_staging_env
-
-# Get dsn of staging RDS
-db_uri=$(get_db_uri $eb_staging_env || {
-  echo "Could not find the db uri" ;
-  exit 1 ;
-})  
-echo "Staging database uri: "$db_uri
-
-echo "Restoring the engine DB into the $eb_staging_env environment "
-send_teams "Restoring the engine DB into the "$eb_staging_env" environment."
-psql -U ais_engine -h $db_uri -d ais_engine -c "DROP SCHEMA IF EXISTS public CASCADE;"
-psql -U ais_engine -h $db_uri -d ais_engine -c "CREATE SCHEMA public;"
-psql -U ais_engine -h $db_uri -d ais_engine -c "GRANT ALL ON SCHEMA public TO postgres;"
-psql -U ais_engine -h $db_uri -d ais_engine -c "GRANT ALL ON SCHEMA public TO public;"
-psql -U ais_engine -h $db_uri -d ais_engine -c "CREATE EXTENSION postgis;"
-psql -U ais_engine -h $db_uri -d ais_engine -c "CREATE EXTENSION pg_trgm;"
-pg_restore -h $db_uri -d ais_engine -U ais_engine -c $db_dump_file_loc
-
-#if [ $? -ne 0 ]
-#then
-#  echo "DB restore failed"
-#  exit 1;
-#fi
-
-# Warm up load balancer
-echo "Warming up the load balancer."
-send_teams "Warming up the load balancer."
-python warmup_lb.py
-if [ $? -ne 0 ]
-then
-  echo "Warmup failed"
-  send_teams "AIS load balanacer warmup failed.\nEngine build has been pushed but not deployed."
-  exit 1;
-fi
-
-# Set staging environment to swap
-echo "Marking the $eb_staging_env environment as ready for deploy (swap)"
-send_teams "Marking the "$eb_staging_env" environment as ready for deploy (swap)."
-eb setenv -e $eb_staging_env SWAP=True --timeout 30
-
-# Deploy latest code and swap
-echo "Restarting the latest master branch build (requires travis CLI)"
-if ! hash travis ; then
-  echo "This step requires the Travis-CI CLI. To install and configure, see:
-  https://github.com/travis-ci/travis.rb#installation"
-  exit 1
-fi
-
-# Get last Travis build ID:
-LAST_BUILD=$(travis history --com --branch master --limit 1 | cut --fields=1 --delimiter=" ")
-
-# The build number has a number sign as the first character. We need to strip
-# it off.
-LAST_BUILD=${LAST_BUILD:1}
-send_teams "Restarting the latest master branch build."
-travis restart $LAST_BUILD --com
-# NOTE: Travis-CI will take over from here. Check in the .travis/deploy script
-# for further step.
-if [ $? -ne 0 ]
-then
-  echo "Travis build failed"
-  send_teams "Travis build failed.\nEngine build has been pushed but not deployed."
-  exit 1;
-fi
-send_teams "New AIS build has been deployed."
-
-# Reporting is called independently
-#echo "Making engine reports."
-#send_teams "Starting reporting."
-#error_file_loc=../log/reports_errors_$datestamp.txt
-#out_file_loc=../log/reports_log_$datestamp.txt
-#python make_reports.py  > >(tee -a $out_file_loc) 2> >(tee -a $error_file_loc >&2)
-#if [ $? -ne 0 ]
-#then
-#  echo "Reports failed"
-#  send_teams "Engine reports did not complete."
-#  exit 1;
-#fi
-#send_teams "Engine reports have completed."
-
diff --git a/ais/engine/bin/make_reports.py b/ais/engine/bin/make_reports.py
index af98647e..49dda80c 100644
--- a/ais/engine/bin/make_reports.py
+++ b/ais/engine/bin/make_reports.py
@@ -2,30 +2,68 @@
 from collections import OrderedDict
 from functools import partial
 import petl as etl
+import geopetl
 import cx_Oracle
 import psycopg2
-import geopetl
 from shapely.wkt import loads
 from shapely.ops import transform
 import pyproj
 from ais import app
 from ais.util import parse_url
 
+
+###################################
+# This script generates reports and writes them to tables in Databridge
+# address_summary, service_area_summary and true_range
+# They are for integrating standardized addresses with department records.
+# Department records meaning dor parcel_id, pwd parcel_id, OPA account number, eclipes location id
+# These departments will import these tables for their own usage.
+#
+# address_summary: contains standardized address components + primary keys and authoritative data
+# service_area_summary: for each address, has ids for each service area the point is located in
+# true_range: Interpolated address location along the street segment
+
+
+
 config = app.config
 read_db_string = config['DATABASES']['engine']
 write_db_string = config['DATABASES']['gis_ais']
+# write_db_string = config['DATABASES']['gis_ais_test']
 parsed_read_db_string = parse_url(read_db_string)
 parsed_write_db_string = parse_url(write_db_string)
-write_dsn = parsed_write_db_string['user'] + '/' + parsed_write_db_string['password'] + '@' + parsed_write_db_string[
-    'host']
+
 address_summary_write_table_name = 'ADDRESS_SUMMARY'
 service_area_summary_write_table_name = 'SERVICE_AREA_SUMMARY'
 dor_condo_error_table_name = 'DOR_CONDOMINIUM_ERROR'
 true_range_write_table_name = 'TRUE_RANGE'
 address_error_write_table_name = 'AIS_ADDRESS_ERROR'
 source_address_write_table_name = 'SOURCE_ADDRESS'
-read_conn = psycopg2.connect(
-    "dbname={db_name} user={user}".format(db_name=parsed_read_db_string['db_name'], user=parsed_read_db_string['user']))
+
+
+read_pass = parsed_read_db_string['password']
+read_user = parsed_read_db_string['user']
+read_host = parsed_read_db_string['host']
+read_db = parsed_read_db_string['db_name']
+read_dsn = f"dbname={read_db} host={read_host} user={read_user} password={read_pass}"
+read_conn = psycopg2.connect(read_dsn)
+
+def database_connect(dsn):
+    # Connect to database
+    db_connect = cx_Oracle.connect(dsn)
+    print('Connected to %s' % db_connect)
+    cursor = db_connect.cursor()
+    return cursor
+
+write_user = parsed_write_db_string['user']
+write_pw = parsed_write_db_string['password']
+write_host = parsed_write_db_string['host']
+write_dsn = f'{write_user}/{write_pw}@{write_host}'
+print('DEBUG: ' + write_dsn)
+oracle_cursor = database_connect(write_dsn)
+
+
+print(f'\nReading from local DB: {read_dsn}')
+print(f'Writing to: {write_dsn}\n'.replace(write_pw, 'CENSORED'))
 #########################################################################################################################
 ## UTILS
 #########################################################################################################################
@@ -68,46 +106,46 @@ def transform_coords(comps):
 
     return [x, y]
 
-
-mapping = OrderedDict([
-    ('id', 'id'),
-    ('address', 'address_low'),
-    ('address_suffix', 'address_low_suffix'),
-    ('address_fractional', 'address_low_frac'),
-    ('address_high', 'address_high'),
-    ('address_full', 'address_full'),
-    ('street_predir', 'street_predir'),
-    ('street_name', 'street_name'),
-    ('street_suffix', 'street_suffix'),
-    ('street_postdir', 'street_postdir'),
-    ('unit_type', 'unit_type'),
-    ('unit_num', 'unit_num'),
-    ('zip_code', 'zip_code'),
-    ('zip_4', 'zip_4'),
-    ('street_address', 'street_address'),
-    ('opa_account_num', 'opa_account_num'),
-    ('opa_owners', 'opa_owners'),
-    ('opa_address', 'opa_address'),
-    ('info_companies', 'info_companies'),
-    ('info_residents', 'info_residents'),
-    ('voters', 'voters'),
-    ('pwd_account_nums', 'pwd_account_nums'),
-    ('li_address_key', 'li_address_key'),
-    ('seg_id', 'seg_id'),
-    ('seg_side', 'seg_side'),
-    ('dor_parcel_id', 'dor_parcel_id'),
-    ('pwd_parcel_id', 'pwd_parcel_id'),
-    ('geocode_type', 'geocode_type'),
-    ('geocode_x', 'geocode_x'),
-    ('geocode_y', 'geocode_y'),
-    ('geocode_lat', 'geocode_lat'),
-    ('geocode_lon', 'geocode_lon'),
-    ('eclipse_location_id', 'eclipse_location_id'),
-    ('zoning_document_ids', 'zoning_document_ids'),
-    ('bin', 'bin'),
-    ('li_parcel_id', 'li_parcel_id'),
-    ('street_code', 'street_code')
-])
+address_summary_mapping = {
+    'id': 'id',
+    'address': 'address_low',
+    'address_suffix': 'address_low_suffix',
+    'address_fractional': 'address_low_frac',
+    'address_high': 'address_high',
+    'address_full': 'address_full',
+    'street_predir': 'street_predir',
+    'street_name': 'street_name',
+    'street_suffix': 'street_suffix',
+    'street_postdir': 'street_postdir',
+    'unit_type': 'unit_type',
+    'unit_num': 'unit_num',
+    'zip_code': 'zip_code',
+    'zip_4': 'zip_4',
+    'street_address': 'street_address',
+    'opa_account_num': 'opa_account_num',
+    'opa_owners': 'opa_owners',
+    'opa_address': 'opa_address',
+    'info_companies': 'info_companies',
+    'info_residents': 'info_residents',
+    'voters': 'voters',
+    'pwd_account_nums': 'pwd_account_nums',
+    'li_address_key': 'li_address_key',
+    'seg_id': 'seg_id',
+    'seg_side': 'seg_side',
+    'dor_parcel_id': 'dor_parcel_id',
+    'pwd_parcel_id': 'pwd_parcel_id',
+    'geocode_type': 'geocode_type',
+    'geocode_x': 'geocode_x',
+    'geocode_y': 'geocode_y',
+    'geocode_lat': 'geocode_lat',
+    'geocode_lon': 'geocode_lon',
+    'eclipse_location_id': 'eclipse_location_id',
+    'zoning_document_ids': 'zoning_document_ids',
+    'bin': 'bin',
+    'li_parcel_id': 'li_parcel_id',
+    'street_code': 'street_code',
+    'shape': 'shape'
+}
 
 
 def standardize_nulls(val):
@@ -132,40 +170,129 @@ def standardize_nulls(val):
 ##############
 # TRUE RANGE #
 ##############
-print("Writing true_range table...")
-etl.fromdb(read_conn, 'select * from true_range').tooraclesde(write_dsn, true_range_write_table_name)
+print(f"\nWriting {true_range_write_table_name} table...")
+#etl.fromdb(read_conn, 'select * from true_range').tooraclesde(write_dsn, true_range_write_table_name)
+rows = etl.fromdb(read_conn, 'select * from true_range')
+rows.tooraclesde(write_dsn, true_range_write_table_name)
+
+
 ########################
 # SERVICE AREA SUMMARY #
 ########################
-print("Writing service_area_summary table...")
-etl.fromdb(read_conn, 'select * from service_area_summary')\
-  .rename({'neighborhood_advisory_committee': 'neighborhood_advisory_committe'}, )\
-  .tooraclesde(write_dsn, service_area_summary_write_table_name)
+print(f"\nWriting {service_area_summary_write_table_name} table...")
+service_area_rows = etl.fromdb(read_conn, 'select * from service_area_summary')
+service_area_rows = etl.rename(service_area_rows, {'neighborhood_advisory_committee': 'neighborhood_advisory_committe'}, )
+service_area_rows.tooraclesde(write_dsn, service_area_summary_write_table_name)
+
+
 ########################
 # ADDRESS SUMMARY #
 ########################
-print("Creating transformed address_summary table...")
-# only export rows that have been geocoded:
-address_summary_out_table = etl.fromdb(read_conn, 'select * from address_summary') \
-    .addfield('address_full', (lambda a: make_address_full(
+print("\nCreating transformed ADDRESS_SUMMARY table...")
+# add address_full and transformed coords, as well as shape as WKT, and only export rows that have been geocoded:
+print('Grabbing fields from local database..')
+addr_summary_rows = etl.fromdb(read_conn, '''
+                select *, 
+                st_x(st_transform(st_setsrid(st_point(geocode_x, geocode_y), 2272), 4326)) as geocode_lon,
+                st_y(st_transform(st_setsrid(st_point(geocode_x, geocode_y), 2272), 4326)) as geocode_lat,
+                public.ST_AsText(st_point(geocode_x, geocode_y)) as shape
+                from address_summary;
+                ''')
+
+print('Synthesizing "ADDRESS_FULL" column..')
+addr_summary_rows = etl.addfield(addr_summary_rows, 'address_full', (lambda a: make_address_full(
     {'address_low': a['address_low'], 'address_low_suffix': a['address_low_suffix'],
-     'address_low_frac': a['address_low_frac'], 'address_high': a['address_high']}))) \
-    .addfield('temp_lonlat', (lambda a: transform_coords({'geocode_x': a['geocode_x'], 'geocode_y': a['geocode_y']}))) \
-    .addfield('geocode_lon', lambda a: a['temp_lonlat'][0]) \
-    .addfield('geocode_lat', lambda a: a['temp_lonlat'][1]) \
-    .cutout('temp_lonlat') \
-    .select(lambda s: s.geocode_x is not None) \
-    .fieldmap(mapping) 
-
-address_summary_out_table.todb(read_conn, "address_summary_transformed", create=True, sample=0)
-address_summary_out_table.tocsv("address_summary_transformed.csv", write_header=True)
+     'address_low_frac': a['address_low_frac'], 'address_high': a['address_high']})))
+    
+# Remove rows with null coordinates
+addr_summary_rows = etl.select(addr_summary_rows, lambda s: s.geocode_x is not None)
+
+# Rename field based on this dictionary
+# Note that its reversed to what you'd expect, values are the original field names
+# and the keys are what the fields are renamed to.
+addr_summary_rows = etl.fieldmap(addr_summary_rows, address_summary_mapping)
+
+# Cut out fields that aren't in our map to match it up with Oracle
+keep_fields = list(address_summary_mapping.keys())
+addr_summary_rows = etl.cut(addr_summary_rows, *keep_fields)
+
+temp_as_table_name = 'T_ADDRESS_SUMMARY'
+prod_as_table_name = 'ADDRESS_SUMMARY'
+
+try:
+    create_stmt = f'CREATE TABLE {temp_as_table_name} AS (SELECT * FROM {prod_as_table_name} WHERE 1=0)'
+    print(f'Creating Oracle table with statement: {create_stmt}')
+    oracle_cursor.execute(create_stmt)
+    oracle_cursor.execute('COMMIT')
+except Exception as e:
+    if 'ORA-00955' not in str(e):
+        raise e
+    else:
+        print(f'Table {temp_as_table_name} already exists.')
+
+# Assert our fields match between our devised petl object and the destination oracle table.
+field_stmt = "SELECT column_name FROM all_tab_cols WHERE table_name = 'T_ADDRESS_SUMMARY' AND owner = 'GIS_AIS'  AND column_name NOT LIKE 'SYS_%'"
+oracle_cursor.execute(field_stmt)
+oracle_fields = oracle_cursor.fetchall()
+oracle_fields = [x[0].lower() for x in oracle_fields]
+oracle_fields.remove('objectid')
+
+# Validate that we have the expected headers in our petl object
+addr_summary_rows.validate(header=tuple(oracle_fields))
+
+print('Writing to csv file..')
+addr_summary_rows.tocsv("address_summary_transformed.csv", write_header=True)
+
+print('Writing to temp table "T_ADDRESS_SUMMARY"..')
+addr_summary_rows.tooraclesde(dbo=write_dsn, table_name='T_ADDRESS_SUMMARY', srid=2272)
+
+grant_sql1 = "GRANT SELECT on {} to SDE".format(temp_as_table_name)
+grant_sql2 = "GRANT SELECT ON {} to GIS_SDE_VIEWER".format(temp_as_table_name)
+grant_sql3 = "GRANT SELECT ON {} to GIS_AIS_SOURCES".format(temp_as_table_name)
+
+
+# Swap prod/temp tables:
+# Oracle does not allow table modification within a transaction, so make individual transactions:
+
+# First make the temp table and setup permissions
+print('Renaming temp table to prod table to minimize downtime..')
+oracle_cursor.execute(grant_sql1)
+oracle_cursor.execute(grant_sql2)
+oracle_cursor.execute(grant_sql3)
+
+sql1 = 'ALTER TABLE {} RENAME TO {}_old'.format(prod_as_table_name, prod_as_table_name)
+sql2 = 'ALTER TABLE {} RENAME TO {}'.format(temp_as_table_name, prod_as_table_name)
+sql3 = 'DROP TABLE {}_old'.format(prod_as_table_name)
+
+
+try:
+    oracle_cursor.execute(sql1)
+except:
+    print("Could not rename {} table. Does it exist?".format(temp_as_table_name))
+    raise
+try:
+    oracle_cursor.execute(sql2)
+except:
+    print("Could not rename {} table. Does it exist?".format(prod_as_table_name))
+    rb_sql = 'ALTER TABLE {}_old RENAME TO {}'.format(prod_as_table_name, prod_as_table_name)
+    oracle_cursor.execute(rb_sql)
+    raise
+try:
+    oracle_cursor.execute(sql3)
+except:
+    print("Could not drop {}_old table. Do you have permission?".format(prod_as_table_name))
+    rb_sql1 = 'DROP TABLE {}'.format(temp_as_table_name)
+    oracle_cursor.execute(rb_sql1)
+    rb_sql2 = 'ALTER TABLE {}_old RENAME TO {}'.format(prod_as_table_name, prod_as_table_name)
+    oracle_cursor.execute(rb_sql2)
+    raise
+
 #########################
 # DOR CONDOMINIUM ERROR #
 #########################
-print("Writing dor_condominium_error table...")
-dor_condominium_error_table = etl.fromdb(read_conn, 'select * from dor_condominium_error') \
-    .rename({'parcel_id': 'mapref', 'unit_num': 'condounit',}) \
-    .tooraclesde(write_dsn, dor_condo_error_table_name)
+print(f"\nWriting to DOR_CONDOMINIUM_ERROR table...")
+dor_condominium_error_table = etl.fromdb(read_conn, 'select * from dor_condominium_error')
+dor_condominium_error_table = etl.rename(dor_condominium_error_table, {'parcel_id': 'mapref', 'unit_num': 'condounit',})
+dor_condominium_error_table.tooraclesde(write_dsn, dor_condo_error_table_name)
 
-print("Cleaning up...")
 read_conn.close()
diff --git a/ais/engine/bin/make_reports.sh b/ais/engine/bin/make_reports.sh
new file mode 100644
index 00000000..874be42c
--- /dev/null
+++ b/ais/engine/bin/make_reports.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -e
+
+WORKING_DIRECTORY=/home/ubuntu/ais
+cd $WORKING_DIRECTORY
+
+export ORACLE_HOME=/usr/lib/oracle/18.5/client64
+export PATH=$PATH:$ORACLE_HOME/bin
+export LD_LIBRARY_PATH=$ORACLE_HOME/lib
+export PYTHONUNBUFFERED=TRUE
+export ENGINE_DB_HOST='localhost'
+
+echo -e "\nActivating virtual environment"
+source $WORKING_DIRECTORY/venv/bin/activate
+source $WORKING_DIRECTORY/.env
+export ENGINE_DB_PASS=$LOCAL_ENGINE_DB_PASS
+
+# Add the ais folder with our __init__.py so we can import it as a python module
+export PYTHONPATH="${PYTHONPATH}:$WORKING_DIRECTORY/ais"
+
+echo "Starting NG911 address points report..."a
+#send_teams "Starting NG911 address points report."
+python $WORKING_DIRECTORY/ais/engine/bin/output_address_points_for_ng911.py
+
+echo "Running make_reports.py.."
+python $WORKING_DIRECTORY/ais/engine/bin/make_reports.py
+
+echo "Starting updating EPAM address points report."
+python $WORKING_DIRECTORY/ais/engine/bin/update_address_points.py
+
+echo "Engine reports have completed."
diff --git a/ais/engine/bin/output_address_points_for_ng911.py b/ais/engine/bin/output_address_points_for_ng911.py
index a303b9f7..003c41a7 100644
--- a/ais/engine/bin/output_address_points_for_ng911.py
+++ b/ais/engine/bin/output_address_points_for_ng911.py
@@ -5,6 +5,11 @@
 from ais import app
 from ais.util import parse_url
 
+############################
+# This script extracts NG911 geometry (proxy for the front door) for each address point and writes
+# it to the databridge-raw database, table name "ais.address_points_geocode_types_for_ng911"
+# Then, triggers a DAG that upserts records in the tripoli 911 database
+
 config = app.config
 target_table = 'ais.address_points_geocode_types_for_ng911'
 temp_csv = 'output_address_points_for_ng911.csv'
@@ -89,7 +94,7 @@
 
 # Trigger DAG to update NG911:
 workflow = 'etl_ng911_v0'
-print("Triggering downstream process...")
+print(f"Triggering downstream DAG {workflow}...")
 try:
     r = requests.post(
         airflow_trigger_creds.get('url').format(dag_name=workflow),
diff --git a/ais/engine/bin/output_spatial_tables.sh b/ais/engine/bin/output_spatial_tables.sh
index 31aba216..985f0a7b 100755
--- a/ais/engine/bin/output_spatial_tables.sh
+++ b/ais/engine/bin/output_spatial_tables.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -e
+set -e 
 
 postgis_dsn=$1
 oracle_dsn_gis_ais=$2
@@ -47,3 +47,5 @@ send_slack "Completed updating address summary in DataBridge."
 
 echo "Cleaning up."
 psql -U ais_engine -h localhost -d ais_engine -c "DROP TABLE address_summary_transformed;"
+
+
diff --git a/ais/engine/bin/reports_go.sh b/ais/engine/bin/reports_go.sh
index 3bd65fc9..a0327cec 100644
--- a/ais/engine/bin/reports_go.sh
+++ b/ais/engine/bin/reports_go.sh
@@ -1,37 +1,38 @@
 #!/usr/bin/env bash
 
-source config.sh
+source ais-config.sh
 
 datestamp=$(date +%Y%m%d)
 start_dt=$(date +%Y%m%d%T)
 echo "Started: "$start_dt
 
 echo "Activating virtual environment"
-source ../../../env/bin/activate
-source ../../../bin/eb_env_utils.sh
+WORKING_DIRECTORY=/home/ubuntu/ais
+echo "Working directory is $WORKING_DIRECTORY"
+cd $WORKING_DIRECTORY
+
+source $WORKING_DIRECTORY/venv/bin/activate
+source $WORKING_DIRECTORY/bin/eb_env_utils.sh
 
 echo "Starting reporting."
 send_teams "Starting reporting."
 
-echo "Starting NG911 address points report..."
+##################################################
+# TEMPORARY COMMENT OUT UNTIL WE'RE IN PRODUCTION 
+# -Roland 7/27/2023
+#echo "Starting NG911 address points report..."
 #send_teams "Starting NG911 address points report."
-python output_address_points_for_ng911.py
-if [ $? -ne 0 ]
-then
-  echo "Outputting address points for NG911 failed"
-  send_teams "Engine reports did not complete."
-  exit 1;
-fi
 
-python make_reports.py
-if [ $? -ne 0 ]
-then
-  echo "Reporting has failed"
-  send_teams "Engine reports did not complete."
-  exit 1;
-fi
+#python $WORKING_DIRECTORY/ais/engine/bin/output_address_points_for_ng911.py
+#if [ $? -ne 0 ]
+#then
+#  echo "Outputting address points for NG911 failed"
+#  send_teams "Engine reports did not complete."
+#  exit 1;
+#fi
+##################################################
 
-bash output_spatial_tables.sh $POSTGIS_CONN $ORACLE_CONN_GIS_AIS
+python $WORKING_DIRECTORY/ais/engine/bin/make_reports.py
 if [ $? -ne 0 ]
 then
   echo "Reporting has failed"
@@ -39,15 +40,26 @@ then
   exit 1;
 fi
 
-echo "Starting updating EPAM address points report."
-python update_address_points.py
-if [ $? -ne 0 ]
-then
-  echo "Reporting has failed"
-  send_teams "Engine reports did not complete."
-  exit 1;
-fi
+##################################################
+# TEMPORARY COMMENT OUT UNTIL WE'RE IN PRODUCTION 
+# bash output_spatial_tables.sh $POSTGIS_CONN $ORACLE_CONN_GIS_AIS
+# if [ $? -ne 0 ]
+# then
+#   echo "Reporting has failed"
+#   send_teams "Engine reports did not complete."
+#   exit 1;
+# fi
+
+# echo "Starting updating EPAM address points report."
+# python update_address_points.py
+# if [ $? -ne 0 ]
+# then
+#   echo "Reporting has failed"
+#   send_teams "Engine reports did not complete."
+#   exit 1;
+# fi
 
+echo "Engine reports have completed."
 send_teams "Engine reports have completed."
 
 end_dt=$(date +%Y%m%d%T)
diff --git a/ais/engine/bin/update_address_points.py b/ais/engine/bin/update_address_points.py
index 6f2d102c..98ffe15d 100644
--- a/ais/engine/bin/update_address_points.py
+++ b/ais/engine/bin/update_address_points.py
@@ -1,3 +1,4 @@
+
 import cx_Oracle
 import petl as etl
 import geopetl
diff --git a/ais/engine/bin/warmup_lb.py b/ais/engine/bin/warmup_lb.py
index 9e377baa..13eb561c 100644
--- a/ais/engine/bin/warmup_lb.py
+++ b/ais/engine/bin/warmup_lb.py
@@ -3,78 +3,117 @@
 import time
 from datetime import datetime
 import requests
+import sys, os
+import click
 
-base_path = 'http://api.phila.gov/ais_staging/v1/addresses/'
-gatekeeper_key = 'gatekeeperKey=4b1dba5f602359a4c6d5c3ed731bfb5b'
-warmup_address_table_name = 'address_summary'
-warmup_address_field = 'street_address'
-warmup_row_limit = 1000
-warmup_fraction_success = .9
-rate_limit = 5
-query_errors = {}
-datestring = datetime.today().strftime('%Y-%m-%d')
-error_file = '../log/warmup_lb_errors_{}.csv'.format(datestring)
-
-
-def RateLimited(maxPerSecond):
-    minInterval = 1.0 / float(maxPerSecond)
-
-    def decorate(func):
-        lastTimeCalled = [0.0]
-
-        def rateLimitedFunction(*args, **kargs):
-            elapsed = time.clock() - lastTimeCalled[0]
-            leftToWait = minInterval - elapsed
-            if leftToWait > 0:
-                time.sleep(leftToWait)
-            ret = func(*args, **kargs)
-            lastTimeCalled[0] = time.clock()
-
-            return ret
-        return rateLimitedFunction
-    return decorate
-
-
-
-from json.decoder import JSONDecodeError
-@RateLimited(rate_limit)
-def query_address(address):
-    try:
-        url = base_path + address + '?' + gatekeeper_key
-        # print(url)
-        r = requests.get(url)
-        return r.status_code
-    except requests.exceptions.HTTPError as e:
-        error = [e,'','']
-        query_errors[url] = error
-    except requests.exceptions.RequestException as e:
-        error = [e,'','']
-        query_errors[url] = error
-    except JSONDecodeError as e:
-        error = [e, r.raw.data, r.raw.read(100)]
-        query_errors[url] = error
-
-
-read_conn = psycopg2.connect("dbname=ais_engine user=ais_engine")
-address_count = etl.fromdb(read_conn, 'select count(*) as N from {}'.format(warmup_address_table_name))
-n = list(address_count.values('n'))[0]
-warmup_rows = etl.fromdb(read_conn, 'select {address_field} from {table} OFFSET floor(random()*{n}) limit {limit}'.format(address_field=warmup_address_field, table=warmup_address_table_name, n=n, limit=warmup_row_limit))
-# print(etl.look(warmup_rows))
-responses = warmup_rows.addfield('response_status', (lambda a: query_address(a['street_address']))).progress(100)
-# print(etl.look(responses))
-eval = responses.aggregate('response_status', len)
-print(etl.look(eval))
-f_200 = [(count/warmup_row_limit) for status, count in eval[1:] if status == 200][0]
-print(f_200)
-###########################
-# WRITE ERRORS OUT TO FILE #
-############################
-print("Writing errors to file...")
-error_table = []
-for url, error_vals in query_errors.items():
-    error_table.append([url, error_vals[0], error_vals[1]])
-etl.tocsv(error_table, error_file)
-exit(0) if f_200 > warmup_fraction_success else exit(1)
 
+@click.command()
+@click.option('--proxy', '-p', required=False,
+              help='proxy is only necessary when run from the office')
+@click.option('--dbpass', '-dp', required=True,
+              help='')
+@click.option('--gatekeeper-key', '-k', required=True,
+              help='')
+def main(proxy, dbpass, gatekeeper_key):
+    base_path = 'http://api.phila.gov/ais_staging/v1/search/'
 
+    warmup_address_table_name = 'address_summary'
+    warmup_address_field = 'street_address'
+    warmup_row_limit = 1000
+    warmup_fraction_success = .9
+    rate_limit = 5
+    query_errors = {}
 
+    datestamp = datetime.today().strftime('%Y-%m-%d')
+    log_directory= os.path.join(os.getcwd(), 'log/')
+    os.makedirs(log_directory, exist_ok=True)
+    warmup_lb_error_file = log_directory + f'/warmup_lb_error-{datestamp}.txt'
+
+    def RateLimited(maxPerSecond):
+        minInterval = 1.0 / float(maxPerSecond)
+
+        def decorate(func):
+            lastTimeCalled = [0.0]
+
+            def rateLimitedFunction(*args, **kargs):
+                elapsed = time.process_time() - lastTimeCalled[0]
+                leftToWait = minInterval - elapsed
+                if leftToWait > 0:
+                    time.sleep(leftToWait)
+                ret = func(*args, **kargs)
+                lastTimeCalled[0] = time.process_time()
+
+                return ret
+            return rateLimitedFunction
+        return decorate
+
+
+
+    from json.decoder import JSONDecodeError
+    @RateLimited(rate_limit)
+    def query_address(address):
+        try:
+            encoded_address = requests.utils.quote(address)
+            url = base_path + encoded_address + '?' + gatekeeper_key
+            #print(url)
+            if proxy:
+                proxies = { 'http': proxy,
+                        'htps': proxy }
+                r = requests.get(url, proxies=proxies, timeout=5)
+            else:
+                r = requests.get(url, timeout=5)
+            if r.status_code:
+                if int(r.status_code) != 200:
+                    print(f"Got a non-200 status code for {url}!: {r.status_code}")
+                return r.status_code
+            else:
+                return None
+        except requests.exceptions.HTTPError as e:
+            error = [e,'','']
+            query_errors[url] = error
+        except requests.exceptions.RequestException as e:
+            error = [e,'','']
+            query_errors[url] = error
+        except JSONDecodeError as e:
+            error = [e, r.raw.data, r.raw.read(100)]
+            query_errors[url] = error
+
+
+    read_conn = psycopg2.connect(f"dbname=ais_engine host=localhost user=ais_engine password={dbpass}")
+    address_count = etl.fromdb(read_conn, 'select count(*) as N from {}'.format(warmup_address_table_name))
+    n = list(address_count.values('n'))[0]
+    warmup_rows = etl.fromdb(read_conn, 'select {address_field} from {table} OFFSET floor(random()*{n}) limit {limit}'.format(address_field=warmup_address_field, table=warmup_address_table_name, n=n, limit=warmup_row_limit))
+    # print(etl.look(warmup_rows))
+    responses = warmup_rows.addfield('response_status', (lambda a: query_address(a['street_address']))).progress(100)
+    #print(etl.look(responses))
+    eval = responses.aggregate('response_status', len)
+    #print(etl.look(eval))
+
+    # count the amount of successful hits
+    f_200 = [(count/warmup_row_limit) for status, count in eval[1:] if (status == 200 and eval)]
+    
+    #print(f_200)
+    ###########################
+    # WRITE ERRORS OUT TO FILE #
+    ############################
+    print("Writing errors to file...")
+    error_table = []
+    for url, error_vals in query_errors.items():
+        error_table.append([url, error_vals[0], error_vals[1]])
+    etl.tocsv(error_table, warmup_lb_error_file)
+
+    if f_200:
+        # Compare the count against our limit of what we want to have succeeded
+        if f_200[0] > warmup_fraction_success:
+            exit(0)
+        else:
+            print('Too many failures encountered during warmup!')
+            exit(1)
+    else:
+        print('Unable to count successes for some reason?')
+        exit(1)
+
+
+
+if __name__ == '__main__':
+    main()
diff --git a/ais/engine/commands.py b/ais/engine/commands.py
new file mode 100644
index 00000000..d0f79d93
--- /dev/null
+++ b/ais/engine/commands.py
@@ -0,0 +1,17 @@
+import click
+import importlib
+
+@click.command()
+@click.option('-s', '--script', default=None)
+def engine(script):
+    if script:
+        print(f'Calling AIS engine script: {script}')
+        script = script.replace('.py','')
+        # dynamically pull in engine scripts as a module and call their main function
+        # The __init__.py in the ais/engine/scripts folder sets all the .py files
+        # in that directory to be importable.
+        mod = __import__("ais.engine.scripts.{}".format(script), fromlist=["main"])
+        mod.main()
+    if not script:
+        print('Please pass an arg to the --script flag.')
+
diff --git a/ais/engine/manage.py b/ais/engine/manage.py
index 8c6e975f..e889fa4f 100644
--- a/ais/engine/manage.py
+++ b/ais/engine/manage.py
@@ -25,7 +25,15 @@ def run(script):
         paths.append(path)
         
     for path in paths:
-        subprocess.call([sys.executable, path], env=os.environ.copy())
+        try:
+            result = subprocess.call([sys.executable, path], env=os.environ.copy())
+        except SystemExit as e:
+            result = e.code
+        sys.exit(result or 0)
+
+
+
+
 
 ## ACTIVATE BELOW WHEN running "ais db migrate"
 #Import database models with app context
diff --git a/ais/engine/scripts/__init__.py b/ais/engine/scripts/__init__.py
new file mode 100644
index 00000000..7ac79615
--- /dev/null
+++ b/ais/engine/scripts/__init__.py
@@ -0,0 +1,6 @@
+from os.path import dirname, basename, isfile, join
+import glob
+# Load everything in this directory as an importable sub module
+# so it can be called by ais/engine/commands.py
+modules = glob.glob(join(dirname(__file__), "*.py"))
+__all__ = [ basename(f)[:-3] for f in modules if isfile(f) and not f.endswith('__init__.py')]
diff --git a/ais/engine/scripts/extras/make_multiple_seg_matches.py b/ais/engine/scripts/extras/make_multiple_seg_matches.py
index 6792f910..e8f1a70c 100644
--- a/ais/engine/scripts/extras/make_multiple_seg_matches.py
+++ b/ais/engine/scripts/extras/make_multiple_seg_matches.py
@@ -92,7 +92,7 @@
 		geocode_row = geocode_rows[0]
 		geocode_shp = loads(geocode_row['geometry_wkt'])
 	except IndexError:
-		print('No parcel XY for {}'.format(multi_addr))
+		#print('No parcel XY for {}'.format(multi_addr))
 		continue
 
 	# Get seg IDs
@@ -140,4 +140,4 @@
 	print('Writing multiple seg lines...')
 	ais_db.bulk_insert(MULTI_SEG_TABLE, multi_seg_lines, geom_field='geometry', multi_geom=False)
 
-ais_db.close()
\ No newline at end of file
+ais_db.close()
diff --git a/ais/engine/scripts/geocode_addresses.py b/ais/engine/scripts/geocode_addresses.py
index 4a8ba0c5..a50d05ee 100644
--- a/ais/engine/scripts/geocode_addresses.py
+++ b/ais/engine/scripts/geocode_addresses.py
@@ -1,531 +1,455 @@
-import sys
-# import os
-# import csv
-# from math import sin, cos, atan2, radians, pi, degrees
 from datetime import datetime
 from shapely.wkt import loads, dumps
 from shapely.geometry import Point, LineString, MultiLineString
 import datum
 from ais import app, util
-from ais.models import Address
 # DEV
 import traceback
-# from pprint import pprint
-
-start = datetime.now()
-print('Starting...')
-
-'''
-SET UP
-'''
-
-config = app.config
-engine_srid = config['ENGINE_SRID']
-Parser = config['PARSER']
-parser = Parser()
-db = datum.connect(config['DATABASES']['engine'])
-engine_srid = config['ENGINE_SRID']
-
-# parcel_table = 'pwd_parcel'
-parcel_layers = config['BASE_DATA_SOURCES']['parcels']
-address_table = db['address']
-address_fields = [
-    'id',
-    'street_address',
-    'address_low',
-    'address_high',
-]
-seg_table = db['street_segment']
-seg_fields = [
-    'seg_id',
-    'left_from',
-    'left_to',
-    'right_from',
-    'right_to',
-]
-geocode_table = db['geocode']
-parcel_curb_table = db['parcel_curb']
-curb_table = db['curb']
-addr_street_table = db['address_street']
-addr_parcel_table = db['address_parcel']
-true_range_view = db['true_range']
-centerline_offset = config['GEOCODE']['centerline_offset']
-centerline_end_buffer = config['GEOCODE']['centerline_end_buffer']
-geocode_priority_map = config['ADDRESS_SUMMARY']['geocode_priority']
-WRITE_OUT = True
-
-# DEV - use this to work on only one street name at a time
-# TODO move this to config somewhere
-FILTER_STREET_NAME = ''
-WHERE_STREET_NAME = None
-WHERE_STREET_ADDRESS_IN = None
-WHERE_SEG_ID_IN = None
-if FILTER_STREET_NAME not in [None, '']:
-    WHERE_STREET_NAME = "street_name = '{}'".format(FILTER_STREET_NAME)
-    WHERE_STREET_ADDRESS_IN = "street_address in (select street_address from \
-		{} where {})".format(address_table.name, WHERE_STREET_NAME)
-    WHERE_SEG_ID_IN = "seg_id in (select seg_id from {} where {})" \
-        .format(seg_table.name, WHERE_STREET_NAME)
-
-if WRITE_OUT:
-    print('Dropping indexes...')
-    geocode_table.drop_index('street_address')
-
-    print('Deleting existing XYs...')
-    geocode_table.delete()
-
-    print('Deleting spatial address-parcels...')
-    spatial_stmt = '''
-		DELETE FROM address_parcel
-			WHERE match_type = 'spatial'
-	'''
-    db.execute(spatial_stmt)
-
-print('Reading streets from AIS...')
-seg_rows = seg_table.read(fields=seg_fields, geom_field='geom', \
-                          where=WHERE_STREET_NAME)
-seg_map = {}
-seg_geom_field = seg_table.geom_field
-for seg_row in seg_rows:
-    seg_id = seg_row['seg_id']
-    seg = {
-        'L': {
-            'low': seg_row['left_from'],
-            'high': seg_row['left_to'],
-        },
-        'R': {
-            'low': seg_row['right_from'],
-            'high': seg_row['right_to'],
-        },
-        'shape': loads(seg_row[seg_geom_field])
-    }
-    seg_map[seg_id] = seg
-
-print('Reading parcels...')
-parcel_xy_map = {}  # source name => parcel row id => centroid xy (Shapely)
-parcel_geom_map = {} # parcel row id => geom
-
-for parcel_layer_name, parcel_layer_def in parcel_layers.items():
-    # source_name = parcel_source['name']
-    source_table = parcel_layer_name + '_parcel'
-    print('  - {}'.format(parcel_layer_name))
-
-    # DEV
-    parcel_where = ''
-    if WHERE_STREET_NAME:
-        parcel_where = '{} and '.format(WHERE_STREET_NAME)
-    # if source_table == 'pwd_parcel':
-    parcel_stmt = '''
-        select
-            id,
-            ST_AsText(geom) as geom,
-            st_astext(st_centroid(geom)) as centroid
-        from {source_table}
-        where {where} st_intersects(st_centroid(geom), geom)
-        union
-        select
-            id,
-            ST_AsText(geom) as geom,
-            st_astext(st_pointonsurface(geom)) as centroid
-        from {source_table}
-        where {where} not st_intersects(st_centroid(geom), geom)
-    '''.format(where=parcel_where, source_table=source_table)
-    parcel_rows = db.execute(parcel_stmt)
-    parcel_layer_xy_map = {}
-    parcel_layer_geom_map = {}
-    for parcel_row in parcel_rows:
-        parcel_id = parcel_row['id']
-        xy = loads(parcel_row['centroid'])
-        poly = loads(parcel_row['geom'])
-        parcel_layer_xy_map[parcel_id] = xy
-        parcel_layer_geom_map[str(parcel_id)] = poly
-    parcel_xy_map[parcel_layer_name] = parcel_layer_xy_map
-    parcel_geom_map[parcel_layer_name] = parcel_layer_geom_map
-
-
-print('Reading true range...')
-true_range_rows = true_range_view.read(where=WHERE_SEG_ID_IN)
-for true_range_row in true_range_rows:
-    seg_id = true_range_row['seg_id']
-    seg_map[seg_id]['L']['true_low'] = true_range_row['true_left_from']
-    seg_map[seg_id]['L']['true_high'] = true_range_row['true_left_to']
-    seg_map[seg_id]['R']['true_low'] = true_range_row['true_right_from']
-    seg_map[seg_id]['R']['true_high'] = true_range_row['true_right_to']
-
-# TODO: redo curb stuff so it works with multiple parcel sources
-print('Reading curbs...')
-curb_rows = curb_table.read(to_srid=engine_srid)
-curb_map = {x['curb_id']: loads(x['geom']) for x in curb_rows}
-
-print('Reading parcel-curbs...')
-parcel_curb_map = {}  # parcel_source => parcel_id => curb_id
-parcel_curb_rows = parcel_curb_table.read()
-dor_parcel_curb_map = {str(x['parcel_row_id']): x['curb_id'] for x in parcel_curb_rows if x['parcel_source'] == 'dor'}
-pwd_parcel_curb_map = {str(x['parcel_row_id']): x['curb_id'] for x in parcel_curb_rows if x['parcel_source'] == 'pwd'}
-parcel_curb_map['dor'] = dor_parcel_curb_map
-parcel_curb_map['pwd'] = pwd_parcel_curb_map
-
-# with open("parcel_curb_map_output.txt", "w") as text_file:
-#     print("parcel_curb_map: {}".format(parcel_curb_map), file=text_file)
-
-print('Reading addresses from AIS...')
-address_rows = address_table.read(fields=address_fields, \
-                                  where=WHERE_STREET_NAME)
-# where='street_address = \'2653-55 N ORIANNA ST\'')
-
-addresses = []
-seg_side_map = {}
-
-# for address_row in address_rows:
-# 	street_address = address_row['street_address']
-# 	address = Address(street_address)
-# 	addresses.append(address)
-
-# addresses = Address.query.all()
-
-# TODO: index by seg ID, side (seg_side_map above)
-# For interpolating between parcel centroids
-# if address_row['seg_id']:
-# 	seg_id = address_row['seg_id']
-# 	seg_side = address_row['seg_side']
-# 	if seg_id in seg_side_map:
-# 		sides = seg_side_map[seg_id]
-# 		if seg_side in sides:
-# 			sides[seg_side].append(address)
-# 		else:
-# 			sides[seg_side] = [address]
-# 	else:
-# 		seg_side_map[seg_id] = {
-# 			seg_side: [address]
-# 		}
-
-print('Reading address-streets...')
-addr_street_rows = addr_street_table.read(where=WHERE_STREET_ADDRESS_IN)
-# Create map: street_address => address-street row
-addr_street_map = {x['street_address']: x for x in addr_street_rows}
-
-print('Reading address-parcels...')
-addr_parcel_rows = addr_parcel_table.read(where=WHERE_STREET_ADDRESS_IN)
-
-# Create map: street address => parcel source => [parcel object ids]
-addr_parcel_map = {}
-for addr_parcel_row in addr_parcel_rows:
-    street_address = addr_parcel_row['street_address']
-    parcel_source = addr_parcel_row['parcel_source']
-    parcel_row_id = addr_parcel_row['parcel_row_id']
-
-    addr_parcel_map.setdefault(street_address, {})
-    addr_parcel_map[street_address].setdefault(parcel_source, [])
-    addr_parcel_map[street_address][parcel_source].append(parcel_row_id)
-
-'''
-MAIN
-'''
-
-print('Geocoding addresses...')
-geocode_rows = []
-geocode_count = 0
-
-# address-parcels to insert from spatial match
-address_parcels = []
-
-for i, address_row in enumerate(address_rows):
-    try:
-        if i % 50000 == 0:
-            print(i)
-
-        if i % 150000 == 0:
-            geocode_table.write(geocode_rows)
-            geocode_count += len(geocode_rows)
-            geocode_rows = []
-
-        address_id = address_row['id']
-        street_address = address_row['street_address']
-        address_low = address_row['address_low']
-        address_high = address_row['address_high']
-
-        # Get mid-address of ranges
-        if address_high:
-            # This is not necessarily an integer, nor the right parity, but
-            # it shouldn't matter for interpolation.
-            address_mid_offset = (address_high - address_low) / 2
-            address_num = address_low + address_mid_offset
-        else:
-            address_num = address_low
-
-        # Get seg ID
-        try:
-            addr_street_row = addr_street_map[street_address]
-            seg_id = addr_street_row['seg_id']
-            seg_side = addr_street_row['seg_side']
-        except KeyError:
-            seg_id = None
-            seg_side = None
 
-        # Get seg XY
-        seg_shp = None  # use this in curbside later
-        if seg_id:
+def main():
+    start = datetime.now()
+    print('Starting...')
+
+    '''
+    SET UP
+    '''
+
+    config = app.config
+    engine_srid = config['ENGINE_SRID']
+    db = datum.connect(config['DATABASES']['engine'])
+    engine_srid = config['ENGINE_SRID']
+
+    parcel_layers = config['BASE_DATA_SOURCES']['parcels']
+    address_table = db['address']
+    address_fields = [
+        'id',
+        'street_address',
+        'address_low',
+        'address_high',
+    ]
+    seg_table = db['street_segment']
+    seg_fields = [
+        'seg_id',
+        'left_from',
+        'left_to',
+        'right_from',
+        'right_to',
+    ]
+    geocode_table = db['geocode']
+    parcel_curb_table = db['parcel_curb']
+    curb_table = db['curb']
+    addr_street_table = db['address_street']
+    addr_parcel_table = db['address_parcel']
+    true_range_view = db['true_range']
+    centerline_offset = config['GEOCODE']['centerline_offset']
+    centerline_end_buffer = config['GEOCODE']['centerline_end_buffer']
+    geocode_priority_map = config['ADDRESS_SUMMARY']['geocode_priority']
+    WRITE_OUT = True
+
+    # DEV - use this to work on only one street name at a time
+    # TODO move this to config somewhere
+    FILTER_STREET_NAME = ''
+    WHERE_STREET_NAME = None
+    WHERE_STREET_ADDRESS_IN = None
+    WHERE_SEG_ID_IN = None
+    if FILTER_STREET_NAME not in [None, '']:
+        WHERE_STREET_NAME = "street_name = '{}'".format(FILTER_STREET_NAME)
+        WHERE_STREET_ADDRESS_IN = "street_address in (select street_address from \
+            {} where {})".format(address_table.name, WHERE_STREET_NAME)
+        WHERE_SEG_ID_IN = "seg_id in (select seg_id from {} where {})" \
+            .format(seg_table.name, WHERE_STREET_NAME)
+
+    if WRITE_OUT:
+        print('Dropping indexes...')
+        geocode_table.drop_index('street_address')
+
+        print('Deleting existing XYs...')
+        geocode_table.delete()
+
+        print('Deleting spatial address-parcels...')
+        spatial_stmt = '''
+            DELETE FROM address_parcel
+                WHERE match_type = 'spatial'
+        '''
+        db.execute(spatial_stmt)
+
+    print('Reading streets from AIS...')
+    seg_rows = seg_table.read(fields=seg_fields, geom_field='geom', \
+                              where=WHERE_STREET_NAME)
+    seg_map = {}
+    seg_geom_field = seg_table.geom_field
+    for seg_row in seg_rows:
+        seg_id = seg_row['seg_id']
+        seg = {
+            'L': {
+                'low': seg_row['left_from'],
+                'high': seg_row['left_to'],
+            },
+            'R': {
+                'low': seg_row['right_from'],
+                'high': seg_row['right_to'],
+            },
+            'shape': loads(seg_row[seg_geom_field])
+        }
+        seg_map[seg_id] = seg
+
+    print('Reading parcels...')
+    parcel_xy_map = {}  # source name => parcel row id => centroid xy (Shapely)
+    parcel_geom_map = {} # parcel row id => geom
+
+    for parcel_layer_name, parcel_layer_def in parcel_layers.items():
+        source_table = parcel_layer_name + '_parcel'
+        print('  - {}'.format(parcel_layer_name))
+
+        # DEV
+        parcel_where = ''
+        if WHERE_STREET_NAME:
+            parcel_where = '{} and '.format(WHERE_STREET_NAME)
+        parcel_stmt = '''
+            select
+                id,
+                ST_AsText(geom) as geom,
+                st_astext(st_centroid(geom)) as centroid
+            from {source_table}
+            where {where} st_intersects(st_centroid(geom), geom)
+            union
+            select
+                id,
+                ST_AsText(geom) as geom,
+                st_astext(st_pointonsurface(geom)) as centroid
+            from {source_table}
+            where {where} not st_intersects(st_centroid(geom), geom)
+        '''.format(where=parcel_where, source_table=source_table)
+        parcel_rows = db.execute(parcel_stmt)
+        parcel_layer_xy_map = {}
+        parcel_layer_geom_map = {}
+        for parcel_row in parcel_rows:
+            parcel_id = parcel_row['id']
+            xy = loads(parcel_row['centroid'])
+            poly = loads(parcel_row['geom'])
+            parcel_layer_xy_map[parcel_id] = xy
+            parcel_layer_geom_map[str(parcel_id)] = poly
+        parcel_xy_map[parcel_layer_name] = parcel_layer_xy_map
+        parcel_geom_map[parcel_layer_name] = parcel_layer_geom_map
+
+    print('Reading true range...')
+    true_range_rows = true_range_view.read(where=WHERE_SEG_ID_IN)
+    for true_range_row in true_range_rows:
+        seg_id = true_range_row['seg_id']
+        seg_map[seg_id]['L']['true_low'] = true_range_row['true_left_from']
+        seg_map[seg_id]['L']['true_high'] = true_range_row['true_left_to']
+        seg_map[seg_id]['R']['true_low'] = true_range_row['true_right_from']
+        seg_map[seg_id]['R']['true_high'] = true_range_row['true_right_to']
+
+    # TODO: redo curb stuff so it works with multiple parcel sources
+    print('Reading curbs...')
+    curb_rows = curb_table.read(to_srid=engine_srid)
+    curb_map = {x['curb_id']: loads(x['geom']) for x in curb_rows}
+
+    print('Reading parcel-curbs...')
+    parcel_curb_map = {}  # parcel_source => parcel_id => curb_id
+    parcel_curb_rows = parcel_curb_table.read()
+    dor_parcel_curb_map = {str(x['parcel_row_id']): x['curb_id'] for x in parcel_curb_rows if x['parcel_source'] == 'dor'}
+    pwd_parcel_curb_map = {str(x['parcel_row_id']): x['curb_id'] for x in parcel_curb_rows if x['parcel_source'] == 'pwd'}
+    parcel_curb_map['dor'] = dor_parcel_curb_map
+    parcel_curb_map['pwd'] = pwd_parcel_curb_map
+
+    print('Reading addresses from AIS...')
+    address_rows = address_table.read(fields=address_fields, \
+                                      where=WHERE_STREET_NAME)
+
+    # TODO: index by seg ID, side (seg_side_map above)
+
+    print('Reading address-streets...')
+    addr_street_rows = addr_street_table.read(where=WHERE_STREET_ADDRESS_IN)
+    # Create map: street_address => address-street row
+    addr_street_map = {x['street_address']: x for x in addr_street_rows}
+
+    print('Reading address-parcels...')
+    addr_parcel_rows = addr_parcel_table.read(where=WHERE_STREET_ADDRESS_IN)
+
+    # Create map: street address => parcel source => [parcel object ids]
+    addr_parcel_map = {}
+    for addr_parcel_row in addr_parcel_rows:
+        street_address = addr_parcel_row['street_address']
+        parcel_source = addr_parcel_row['parcel_source']
+        parcel_row_id = addr_parcel_row['parcel_row_id']
+
+        addr_parcel_map.setdefault(street_address, {})
+        addr_parcel_map[street_address].setdefault(parcel_source, [])
+        addr_parcel_map[street_address][parcel_source].append(parcel_row_id)
+
+    '''
+    MAIN
+    '''
+
+    print('Geocoding addresses...')
+    geocode_rows = []
+    geocode_count = 0
+
+    # address-parcels to insert from spatial match
+    address_parcels = []
+
+    for i, address_row in enumerate(address_rows):
+        try:
+            if i % 50000 == 0:
+                print(i)
+
+            if i % 150000 == 0:
+                geocode_table.write(geocode_rows)
+                geocode_count += len(geocode_rows)
+                geocode_rows = []
+
+            address_id = address_row['id']
+            street_address = address_row['street_address']
+            address_low = address_row['address_low']
+            address_high = address_row['address_high']
+
+            # Get mid-address of ranges
+            if address_high:
+                # This is not necessarily an integer, nor the right parity, but
+                # it shouldn't matter for interpolation.
+                address_mid_offset = (address_high - address_low) / 2
+                address_num = address_low + address_mid_offset
+            else:
+                address_num = address_low
 
-            '''
-            CENTERLINE
-            '''
+            # Get seg ID
+            try:
+                addr_street_row = addr_street_map[street_address]
+                seg_id = addr_street_row['seg_id']
+                seg_side = addr_street_row['seg_side']
+            except KeyError:
+                seg_id = None
+                seg_side = None
 
             # Get seg XY
-            seg = seg_map[seg_id]
-            seg_shp = seg['shape']
-
-            # Interpolate using full range
-            low = seg[seg_side]['low']
-            high = seg[seg_side]['high']
-            side_delta = high - low
-            # seg_estimated = False
-
-            # If the there's no range
-            if side_delta == 0:
-                # print('No range: seg {}, {} - {}'.format(seg_id, low, high))
-                # continue
-                # Put it in the middle
-                distance_ratio = 0.5
-            # seg_estimated = True
-            else:
-                distance_ratio = (address_num - low) / side_delta
-            # print('Distance ratio: {}'.format(distance_ratio))
-
-            # Old method: just interpolate
-            # seg_xsect_xy_old = seg_shp.interpolate(distance_ratio, \
-            # 	normalized=True)
-            # print('Old intersect: {}'.format(seg_xsect_xy_old))
-
-            # New method: interpolate buffered
-            seg_xsect_xy = util.interpolate_buffered(seg_shp, distance_ratio, \
-                                                     centerline_end_buffer)
-            # print('Intersect: {}'.format(seg_xsect_xy))
-
-            seg_xy = util.offset(seg_shp, seg_xsect_xy, centerline_offset, \
-                                 seg_side)
-            # print('Offset to {}: {}'.format(seg_side, seg_xy))
-            geocode_rows.append({
-                # 'address_id': address_id,
-                'street_address': street_address,
-                'geocode_type': geocode_priority_map['centerline'],
-                # 'estimated': '1' if seg_estimated else '0',
-                'geom': dumps(seg_xy)
-            })
+            seg_shp = None  # use this in curbside later
+            if seg_id:
 
-            '''
-            TRUE RANGE
-            '''
+                '''
+                CENTERLINE
+                '''
 
-            true_low = seg[seg_side]['true_low']
-            true_high = seg[seg_side]['true_high']
-            true_side_delta = true_high - true_low
-            # true_estimated = False
+                # Get seg XY
+                seg = seg_map[seg_id]
+                seg_shp = seg['shape']
 
-            if true_side_delta == 0:
-                # print('No true range: {}, seg {}, {} - {}'.format(seg_id, true_low, true_high))
-                # continue
-                true_distance_ratio = 0.5
-            # true_estimated = True
-            else:
-                true_distance_ratio = (address_num - true_low) / true_side_delta
-
-            # true_xsect_xy = seg_shp.interpolate(true_distance_ratio, \
-            # 	normalized=True)
-            true_xsect_xy = util.interpolate_buffered(seg_shp, true_distance_ratio, \
-                                                      centerline_end_buffer)
-            true_seg_xy = util.offset(seg_shp, true_xsect_xy, centerline_offset, \
-                                      seg_side)
-            # print('true: {}'.format(true_seg_xy))
-            geocode_rows.append({
-                # 'address_id': address_id,
-                'street_address': street_address,
-                'geocode_type': geocode_priority_map['true_range'],
-                # 'estimated': '1' if true_estimated else '0',
-                'geom': dumps(true_seg_xy)
-            })
+                # Interpolate using full range
+                low = seg[seg_side]['low']
+                high = seg[seg_side]['high']
+                side_delta = high - low
+                # seg_estimated = False
 
-        '''
-        PARCELS
-        '''
+                # If the there's no range
+                if side_delta == 0:
+                    # Put it in the middle
+                    distance_ratio = 0.5
+                else:
+                    distance_ratio = (address_num - low) / side_delta
 
-        for parcel_layer_name, parcel_layer in parcel_layers.items():
-            source_table = parcel_layer_name + '_parcel'
-            # set these to None to avoid next address inheritance
-            parcel_ids = None
-            parcel_id = None
-            parcel_xy = None
-            try:
-                parcel_ids = addr_parcel_map[street_address][parcel_layer_name]
-            except KeyError as e:
-                # TODO: check if there's an address link that points to an
-                # address in address_parcel
-                parcel_ids = None
+                # Old method: just interpolate
+                # New method: interpolate buffered
+                seg_xsect_xy = util.interpolate_buffered(seg_shp, distance_ratio, \
+                                                         centerline_end_buffer)
+
+                seg_xy = util.offset(seg_shp, seg_xsect_xy, centerline_offset, \
+                                     seg_side)
+                geocode_rows.append({
+                    'street_address': street_address,
+                    'geocode_type': geocode_priority_map['centerline'],
+                    'geom': dumps(seg_xy)
+                })
 
-            # Get parcel XY
-            if parcel_ids:
-                # Single parcel match
-                if len(parcel_ids) == 1:
-                    parcel_id = parcel_ids[0]
-                    parcel_xy = parcel_xy_map[parcel_layer_name][parcel_id]
-                    geocode_rows.append({
-                        # 'address_id': address_id,
-                        'street_address': street_address,
-                        'geocode_type': geocode_priority_map[source_table],
-                        # 'estimated': 		'0',
-                        'geom': dumps(parcel_xy)
-                    })
-
-                # Multiple parcel matches
-                else:
-                    # TODO: could get the combined centroid of the parcels,
-                    # if they're adjacent
-                    # print('{}: {} parcel matches'.format(street_address, len(parcel_ids)))
-                    # num_multiple_parcel_matches += 1
-                    parcel_id = None
-                    parcel_xy = None
-
-            elif seg_id:
                 '''
-                SPATIAL MATCH
+                TRUE RANGE
                 '''
 
-                for test_offset in range(10, 50, 10):
-                    # Get test XY
-                    test_xy_shp = util.offset(seg_shp, true_xsect_xy, \
-                                              test_offset, seg_side)
-                    test_xy_wkt = dumps(test_xy_shp)
-
-                    parcel_match_stmt = '''
-						SELECT
-							id,
-							CASE
-								WHEN ST_Intersects(geom, ST_Centroid(geom))
-								THEN ST_AsText(ST_Centroid(geom))
-								ELSE ST_AsText(ST_PointOnSurface(geom))
-							END as wkt
-						FROM {source_table}
-						WHERE ST_Intersects(geom, ST_GeomFromText('{test_xy_wkt}', {engine_srid}))
-					'''.format(source_table=source_table, test_xy_wkt=test_xy_wkt, engine_srid=engine_srid)
-                    db.execute(parcel_match_stmt)
-                    parcel_match = db._c.fetchone()
-
-                    if parcel_match:
-                        parcel_id = parcel_match['id']
-                        # pwd_parcel_id = parcel_match['parcel_id']
-                        # pwd_parcel_xy = parcel_xy_map[parcel_layer_name][parcel_id]
-                        # print('Rematched {} to PWD parcel {}'.format(street_address, pwd_parcel_id))
-                        parcel_match_wkt = parcel_match['wkt']
-                        geocode_rows.append({
-                            'street_address': street_address,
-                            'geocode_type': geocode_priority_map[source_table + '_spatial'],
-                            # 'estimated': '1',
-                            # 'geometry': dumps(pwd_parcel_xy)
-                            'geom': parcel_match_wkt,
-                        })
-
-                        # Make estimated address-parcel
-                        address_parcels.append({
-                            'street_address': street_address,
-                            'parcel_source': parcel_layer_name,
-                            'parcel_row_id': parcel_id,
-                            'match_type': 'spatial',
-                        })
+                true_low = seg[seg_side]['true_low']
+                true_high = seg[seg_side]['true_high']
+                true_side_delta = true_high - true_low
 
-                        break
+                if true_side_delta == 0:
+                    true_distance_ratio = 0.5
+                else:
+                    true_distance_ratio = (address_num - true_low) / true_side_delta
+
+                true_xsect_xy = util.interpolate_buffered(seg_shp, true_distance_ratio, \
+                                                          centerline_end_buffer)
+                true_seg_xy = util.offset(seg_shp, true_xsect_xy, centerline_offset, \
+                                          seg_side)
+                geocode_rows.append({
+                    'street_address': street_address,
+                    'geocode_type': geocode_priority_map['true_range'],
+                    'geom': dumps(true_seg_xy)
+                })
 
             '''
-            CURBSIDE & IN_STREET (MIDPOINT B/T CURB & CENTERLINE)
+            PARCELS
             '''
 
-            if seg_id and parcel_id and parcel_xy:
-                # TODO: use pwd parcel if matched spatially
-                parcel_id = str(parcel_id)
-                if parcel_id in parcel_curb_map[parcel_layer_name] and seg_shp is not None:
-                    curb_id = parcel_curb_map[parcel_layer_name][parcel_id]
-                    curb_shp = curb_map[curb_id]
-
-                    # Project parcel centroid to centerline
-                    # if parcel_xy:
-                    proj_dist = seg_shp.project(parcel_xy)
-                    proj_xy = seg_shp.interpolate(proj_dist)
-                    proj_shp = LineString([parcel_xy, proj_xy])
-
-                    # Get point of intersection and add
-                    curb_xsect_line_shp = curb_shp.intersection(proj_shp)
-                    curb_xsect_pt = None
-                    if isinstance(curb_xsect_line_shp, LineString):
-                        curb_xsect_pt = curb_xsect_line_shp.coords[1]
-                    elif isinstance(curb_xsect_line_shp, MultiLineString):
-                        curb_xsect_pt = curb_xsect_line_shp[-1:][0].coords[1]
-                    if curb_xsect_pt:
-                        xy_on_curb_shp = Point(curb_xsect_pt)
-                        curb_geocode_row = {
-                            'street_address': street_address,
-                            'geocode_type': geocode_priority_map[parcel_layer_name + '_curb'],
-                            'geom': dumps(xy_on_curb_shp)
-                        }
-                        # Get midpoint between centerline and curb
-                        xy_in_street = (proj_xy.x + curb_xsect_pt[0]) / 2, (proj_xy.y + curb_xsect_pt[1]) / 2
-                        xy_in_st_shape = Point(xy_in_street)
-                        in_st_geocode_row = {
+            for parcel_layer_name, parcel_layer in parcel_layers.items():
+                source_table = parcel_layer_name + '_parcel'
+                # set these to None to avoid next address inheritance
+                parcel_ids = None
+                parcel_id = None
+                parcel_xy = None
+                try:
+                    parcel_ids = addr_parcel_map[street_address][parcel_layer_name]
+                except KeyError as e:
+                    # TODO: check if there's an address link that points to an
+                    # address in address_parcel
+                    parcel_ids = None
+
+                # Get parcel XY
+                if parcel_ids:
+                    # Single parcel match
+                    if len(parcel_ids) == 1:
+                        parcel_id = parcel_ids[0]
+                        parcel_xy = parcel_xy_map[parcel_layer_name][parcel_id]
+                        geocode_rows.append({
                             'street_address': street_address,
-                            'geocode_type': geocode_priority_map[parcel_layer_name + '_street'],
-                            'geom': dumps(xy_in_st_shape)
-                        }
-                        geocode_rows.append(curb_geocode_row)
-                        geocode_rows.append(in_st_geocode_row)
-
-                    # PWD centroid geocoded to front of building/parcel
-                    if parcel_id in parcel_geom_map[parcel_layer_name]:
-                        # print(parcel_id)
-                        parcel_geom = parcel_geom_map[parcel_layer_name][parcel_id]
-                        parcel_front_xsect_line_shp = parcel_geom.intersection(proj_shp)
-                        parcel_front_xsect_pt = None
-                        if isinstance(parcel_front_xsect_line_shp, LineString):
-                            parcel_front_xsect_pt = parcel_front_xsect_line_shp.coords[1]
-                        elif isinstance(parcel_front_xsect_line_shp, MultiLineString):
-                            parcel_front_xsect_pt = parcel_front_xsect_line_shp[-1:][0].coords[1]
-                        if parcel_front_xsect_pt:
-                            xy_on_parcel_front_shp = Point(parcel_front_xsect_pt)
-                            parcel_front_geocode_row = {
-                                'street_address': street_address,
-                                'geocode_type': geocode_priority_map[parcel_layer_name + '_parcel_front'],
-                                'geom': dumps(xy_on_parcel_front_shp)
-                            }
-                            geocode_rows.append(parcel_front_geocode_row)
-
-    except ValueError as e:
-        print(e)
-
-    except Exception:
-        print(traceback.format_exc())
-        sys.exit()
-
-if WRITE_OUT:
-    print('Writing XYs...')
-    geocode_table.write(geocode_rows, chunk_size=150000)
-
-    print('Writing address-parcels...')
-    # db.drop_index('address_parcel', 'street_address')
-    addr_parcel_table.write(address_parcels, chunk_size=150000)
-    # db.create_index('address_parcel', 'street_address')
-
-    # print('Creating index...')
-    # geocode_table.create_index('street_address')
+                            'geocode_type': geocode_priority_map[source_table],
+                            'geom': dumps(parcel_xy)
+                        })
 
-    print('Wrote {} rows'.format(len(geocode_rows) + geocode_count))
+                    # Multiple parcel matches
+                    else:
+                        # TODO: could get the combined centroid of the parcels,
+                        # if they're adjacent
+                        parcel_id = None
+                        parcel_xy = None
+
+                elif seg_id:
+                    '''
+                    SPATIAL MATCH
+                    '''
+
+                    for test_offset in range(10, 50, 10):
+                        # Get test XY
+                        test_xy_shp = util.offset(seg_shp, true_xsect_xy, \
+                                                  test_offset, seg_side)
+                        test_xy_wkt = dumps(test_xy_shp)
+
+                        parcel_match_stmt = '''
+                            SELECT
+                                id,
+                                CASE
+                                    WHEN ST_Intersects(geom, ST_Centroid(geom))
+                                    THEN ST_AsText(ST_Centroid(geom))
+                                    ELSE ST_AsText(ST_PointOnSurface(geom))
+                                END as wkt
+                            FROM {source_table}
+                            WHERE ST_Intersects(geom, ST_GeomFromText('{test_xy_wkt}', {engine_srid}))
+                        '''.format(source_table=source_table, test_xy_wkt=test_xy_wkt, engine_srid=engine_srid)
+                        db.execute(parcel_match_stmt)
+                        parcel_match = db._c.fetchone()
+
+                        if parcel_match:
+                            parcel_id = parcel_match['id']
+                            parcel_match_wkt = parcel_match['wkt']
+                            geocode_rows.append({
+                                'street_address': street_address,
+                                'geocode_type': geocode_priority_map[source_table + '_spatial'],
+                                'geom': parcel_match_wkt,
+                            })
 
-# Process source address point geocodes in batch:
-# geocodes:
-geocode_stmt = '''insert into geocode (street_address, geocode_type, geom)
-select street_address, {geocode_type} as geocode_type, geom
-from ng911_address_point
-'''.format(geocode_type=geocode_priority_map['ng911'])
-db.execute(geocode_stmt)
+                            # Make estimated address-parcel
+                            address_parcels.append({
+                                'street_address': street_address,
+                                'parcel_source': parcel_layer_name,
+                                'parcel_row_id': parcel_id,
+                                'match_type': 'spatial',
+                            })
 
-print('Creating index...')
-geocode_table.create_index('street_address')
+                            break
 
-db.close()
+                '''
+                CURBSIDE & IN_STREET (MIDPOINT B/T CURB & CENTERLINE)
+                '''
 
-print('Finished in {}'.format(datetime.now() - start))
+                if seg_id and parcel_id and parcel_xy:
+                    # TODO: use pwd parcel if matched spatially
+                    parcel_id = str(parcel_id)
+                    if parcel_id in parcel_curb_map[parcel_layer_name] and seg_shp is not None:
+                        curb_id = parcel_curb_map[parcel_layer_name][parcel_id]
+                        curb_shp = curb_map[curb_id]
+
+                        # Project parcel centroid to centerline
+                        proj_dist = seg_shp.project(parcel_xy)
+                        proj_xy = seg_shp.interpolate(proj_dist)
+                        proj_shp = LineString([parcel_xy, proj_xy])
+
+                        # Get point of intersection and add
+                        curb_xsect_line_shp = curb_shp.intersection(proj_shp)
+                        curb_xsect_pt = None
+                        if isinstance(curb_xsect_line_shp, LineString):
+                            try: 
+                                curb_xsect_pt = curb_xsect_line_shp.coords[1]
+                            # If no coords returned from our intersection, pass. 
+                            except IndexError: 
+                                pass
+                        elif isinstance(curb_xsect_line_shp, MultiLineString):
+                            curb_xsect_pt = curb_xsect_line_shp[-1:][0].coords[1]
+                        if curb_xsect_pt:
+                            xy_on_curb_shp = Point(curb_xsect_pt)
+                            curb_geocode_row = {
+                                'street_address': street_address,
+                                'geocode_type': geocode_priority_map[parcel_layer_name + '_curb'],
+                                'geom': dumps(xy_on_curb_shp)
+                            }
+                            # Get midpoint between centerline and curb
+                            xy_in_street = (proj_xy.x + curb_xsect_pt[0]) / 2, (proj_xy.y + curb_xsect_pt[1]) / 2
+                            xy_in_st_shape = Point(xy_in_street)
+                            in_st_geocode_row = {
+                                'street_address': street_address,
+                                'geocode_type': geocode_priority_map[parcel_layer_name + '_street'],
+                                'geom': dumps(xy_in_st_shape)
+                            }
+                            geocode_rows.append(curb_geocode_row)
+                            geocode_rows.append(in_st_geocode_row)
+
+                        # PWD centroid geocoded to front of building/parcel
+                        if parcel_id in parcel_geom_map[parcel_layer_name]:
+                            parcel_geom = parcel_geom_map[parcel_layer_name][parcel_id]
+                            parcel_front_xsect_line_shp = parcel_geom.intersection(proj_shp)
+                            parcel_front_xsect_pt = None
+                            if isinstance(parcel_front_xsect_line_shp, LineString):
+                                parcel_front_xsect_pt = parcel_front_xsect_line_shp.coords[1]
+                            elif isinstance(parcel_front_xsect_line_shp, MultiLineString):
+                                parcel_front_xsect_pt = parcel_front_xsect_line_shp[-1:][0].coords[1]
+                            if parcel_front_xsect_pt:
+                                xy_on_parcel_front_shp = Point(parcel_front_xsect_pt)
+                                parcel_front_geocode_row = {
+                                    'street_address': street_address,
+                                    'geocode_type': geocode_priority_map[parcel_layer_name + '_parcel_front'],
+                                    'geom': dumps(xy_on_parcel_front_shp)
+                                }
+                                geocode_rows.append(parcel_front_geocode_row)
+
+        except ValueError as e:
+            print(e)
+
+        except Exception as e:
+            print(traceback.format_exc())
+            raise e
+
+    if WRITE_OUT:
+        print('Writing XYs...')
+        geocode_table.write(geocode_rows, chunk_size=150000)
+        print('Writing address-parcels...')
+        addr_parcel_table.write(address_parcels, chunk_size=150000)
+        print('Wrote {} rows'.format(len(geocode_rows) + geocode_count))
+
+    # Process source address point geocodes in batch:
+    # geocodes:
+    geocode_stmt = '''insert into geocode (street_address, geocode_type, geom)
+    select street_address, {geocode_type} as geocode_type, geom
+    from ng911_address_point
+    '''.format(geocode_type=geocode_priority_map['ng911'])
+    db.execute(geocode_stmt)
+
+    print('Creating index...')
+    geocode_table.create_index('street_address')
+    db.close()
+    print('Finished in {}'.format(datetime.now() - start))
diff --git a/ais/engine/scripts/geocode_addresses_from_links.py b/ais/engine/scripts/geocode_addresses_from_links.py
index 0402c576..f49594c3 100644
--- a/ais/engine/scripts/geocode_addresses_from_links.py
+++ b/ais/engine/scripts/geocode_addresses_from_links.py
@@ -2,116 +2,118 @@
 import datum
 from ais import app
 
-start = datetime.now()
-print('Starting...')
+def main():
+    start = datetime.now()
+    print('Starting...')
 
-'''
-SET UP
-'''
+    '''
+    SET UP
+    '''
 
-config = app.config
-engine_srid = config['ENGINE_SRID']
-Parser = config['PARSER']
-parser = Parser()
-db = datum.connect(config['DATABASES']['engine'])
-WRITE_OUT = True
-geocode_table = db['geocode']
-address_tag_table = db['address_tag']
-geocode_tag_map = {
-    'pwd_parcel_id': (1, 3, 7, 11),
-    'dor_parcel_id': (2, 4, 8, 12)
-}
-new_geocode_rows = []
+    config = app.config
+    engine_srid = config['ENGINE_SRID']
+    Parser = config['PARSER']
+    parser = Parser()
+    db = datum.connect(config['DATABASES']['engine'])
+    WRITE_OUT = True
+    geocode_table = db['geocode']
+    address_tag_table = db['address_tag']
+    geocode_tag_map = {
+        'pwd_parcel_id': (1, 3, 7, 11),
+        'dor_parcel_id': (2, 4, 8, 12)
+    }
+    new_geocode_rows = []
 
-print('Reading geocode rows...')
-geocode_map = {}
-geocode_rows = geocode_table.read()
-print('Mapping geocode rows...')
-for geocode_row in geocode_rows:
-    street_address = geocode_row['street_address']
-    if not street_address in geocode_map:
-        geocode_map[street_address] = []
-    geocode_map[street_address].append(geocode_row)
+    print('Reading geocode rows...')
+    geocode_map = {}
+    geocode_rows = geocode_table.read()
+    print('Mapping geocode rows...')
+    for geocode_row in geocode_rows:
+        street_address = geocode_row['street_address']
+        if not street_address in geocode_map:
+            geocode_map[street_address] = []
+        geocode_map[street_address].append(geocode_row)
 
-print('Reading address tags...')
-tag_map = {}
-where = "linked_address != '' and key in ('pwd_parcel_id', 'dor_parcel_id')"
-tag_rows = address_tag_table.read(where=where)
-print('Mapping address tags...')
-for tag_row in tag_rows:
-    street_address = tag_row['street_address']
-    if not street_address in tag_map:
-        tag_map[street_address] = []
-    tag_map[street_address].append(tag_row)
+    print('Reading address tags...')
+    tag_map = {}
+    where = "linked_address != '' and key in ('pwd_parcel_id', 'dor_parcel_id')"
+    tag_rows = address_tag_table.read(where=where)
+    print('Mapping address tags...')
+    for tag_row in tag_rows:
+        street_address = tag_row['street_address']
+        if not street_address in tag_map:
+            tag_map[street_address] = []
+        tag_map[street_address].append(tag_row)
 
-for key, value in tag_map.items():
-    street_address = key
-    tags = value
-#    geocode_types = []
-    try:
-        geocode_types = [x['geocode_type'] for x in geocode_map[street_address]]
-    except:
-        geocode_types = []
-    for tag in tags:
-        linked_address = tag['linked_address']
-        linked_key = tag['key']
-        if linked_key == 'pwd_parcel_id' and 1 in geocode_types or linked_key == 'dor_parcel_id' and 2 in geocode_types:
-            continue
+    for key, value in tag_map.items():
+        street_address = key
+        tags = value
+    #    geocode_types = []
         try:
-            linked_geocode_rows = geocode_map[linked_address]
+            geocode_types = [x['geocode_type'] for x in geocode_map[street_address]]
         except:
-            linked_geocode_rows = []
-        if not linked_geocode_rows:
-            continue
-        for linked_row in linked_geocode_rows:
-            geocode_type = linked_row['geocode_type']
+            geocode_types = []
+        for tag in tags:
+            linked_address = tag['linked_address']
+            linked_key = tag['key']
+            if linked_key == 'pwd_parcel_id' and 1 in geocode_types or linked_key == 'dor_parcel_id' and 2 in geocode_types:
+                continue
+            try:
+                linked_geocode_rows = geocode_map[linked_address]
+            except:
+                linked_geocode_rows = []
+            if not linked_geocode_rows:
+                continue
+            for linked_row in linked_geocode_rows:
+                geocode_type = linked_row['geocode_type']
 
-            if geocode_type in geocode_tag_map[linked_key]:
-                geom = linked_row['geom']
-                new_geocode_row = {
-                    'street_address': street_address,
-                    'geocode_type': geocode_type,
-                    'geom': geom
-                }
-                new_geocode_rows.append(new_geocode_row)
+                if geocode_type in geocode_tag_map[linked_key]:
+                    geom = linked_row['geom']
+                    new_geocode_row = {
+                        'street_address': street_address,
+                        'geocode_type': geocode_type,
+                        'geom': geom
+                    }
+                    new_geocode_rows.append(new_geocode_row)
 
-# Remove any duplicate new_geocode_rows
-new_geocode_rows = [dict(t) for t in set([tuple(d.items()) for d in new_geocode_rows])]
+    # Remove any duplicate new_geocode_rows
+    new_geocode_rows = [dict(t) for t in set([tuple(d.items()) for d in new_geocode_rows])]
 
-if WRITE_OUT:
-    print('Dropping indexes...')
-    geocode_table.drop_index('street_address')
+    if WRITE_OUT:
+        print('Dropping indexes...')
+        geocode_table.drop_index('street_address')
 
-    print('Writing {num} new geocode rows...'.format(num=len(new_geocode_rows)))
-    # TODO: Use geopetl/datum instead of raw sql
-    i = 0
-    values = ''
-    for new_row in new_geocode_rows:
-        street_address = new_row['street_address']
-        geocode_type = new_row['geocode_type']
-        geom = new_row['geom']
-        new_vals = '''('{street_address}', {geocode_type}, ST_GeomFromText('{geom}',{engine_srid}))'''.format(street_address=street_address, geocode_type=geocode_type, geom=geom, engine_srid=engine_srid)
-        values = values + ', ' + new_vals if values else new_vals
-        i += 1
-        if i % 1000 == 0:
-            toi = i + 1000
-            #print("writing rows {i} to {toi}".format(i=i, toi=toi))
+        print('Writing {num} new geocode rows...'.format(num=len(new_geocode_rows)))
+        # TODO: Use geopetl/datum instead of raw sql
+        i = 0
+        values = ''
+        for new_row in new_geocode_rows:
+            street_address = new_row['street_address']
+            geocode_type = new_row['geocode_type']
+            geom = new_row['geom']
+            new_vals = '''('{street_address}', {geocode_type}, ST_GeomFromText('{geom}',{engine_srid}))'''.format(street_address=street_address, geocode_type=geocode_type, geom=geom, engine_srid=engine_srid)
+            values = values + ', ' + new_vals if values else new_vals
+            i += 1
+            if i % 1000 == 0:
+                toi = i + 1000
+                #print("writing rows {i} to {toi}".format(i=i, toi=toi))
+                write_stmt = '''
+                    INSERT INTO geocode (street_address, geocode_type, geom) VALUES {values}
+                '''.format(values=values)
+                db.execute(write_stmt)
+                db.save()
+                values = ''
+        if values: 
             write_stmt = '''
                 INSERT INTO geocode (street_address, geocode_type, geom) VALUES {values}
             '''.format(values=values)
             db.execute(write_stmt)
             db.save()
-            values = ''
-    if values: 
-        write_stmt = '''
-            INSERT INTO geocode (street_address, geocode_type, geom) VALUES {values}
-        '''.format(values=values)
-        db.execute(write_stmt)
-        db.save()        
 
-    print('Creating index...')
-    geocode_table.create_index('street_address')
+        print('Creating index...')
+        geocode_table.create_index('street_address')
 
-db.close()
+    db.close()
+
+    print('Finished in {}'.format(datetime.now() - start))
 
-print('Finished in {}'.format(datetime.now() - start))
diff --git a/ais/engine/scripts/get_pwd_matches_from_manual_opa_geocodes.py b/ais/engine/scripts/get_pwd_matches_from_manual_opa_geocodes.py
index 3cd5dd67..75d485e2 100644
--- a/ais/engine/scripts/get_pwd_matches_from_manual_opa_geocodes.py
+++ b/ais/engine/scripts/get_pwd_matches_from_manual_opa_geocodes.py
@@ -1,87 +1,93 @@
+from datetime import datetime
 import petl as etl
 import cx_Oracle
 import psycopg2
 import geopetl
 from ais import app
 
-config = app.config
 
-# read opa_active_accounts from databridge (oracle sde) and write to engine (postgis)
+def main():
+    start = datetime.now()
 
-# Get source table connection
-source_def = config['BASE_DATA_SOURCES']['opa_active_accounts']
-source_db_name = source_def['db']
-source_db_url = config['DATABASES'][source_db_name]
-source_table = source_def['table']
-source_field_map = source_def['field_map']
-source_fields = [field for field in list(source_field_map.values()) if field != 'shape']
-conn_dsn = source_db_url[source_db_url.index("//") + 2:]
-conn_user = conn_dsn[:conn_dsn.index(":")]
-conn_pw = conn_dsn[conn_dsn.index(":") + 1 : conn_dsn.index("@")]
-conn_db = conn_dsn[conn_dsn.index("@") + 1:]
-source_conn = cx_Oracle.connect(conn_user, conn_pw, conn_db)
+    config = app.config
 
-target_dsn = config['DATABASES']['engine']
-target_user = target_dsn[target_dsn.index("//") + 2:target_dsn.index(":", target_dsn.index("//"))]
-target_pw = target_dsn[target_dsn.index(":",target_dsn.index(target_user)) + 1:target_dsn.index("@")]
-target_name = target_dsn[target_dsn.index("/", target_dsn.index("@")) + 1:]
-target_conn = psycopg2.connect('dbname={db_name} user={db_user} password={db_pw} host=localhost'.format(db_name=target_name, db_user=target_user, db_pw=target_pw))
-target_cur = target_conn.cursor()
-target_table_name = 'public.t_opa_active_accounts'
+    # read opa_active_accounts from databridge (oracle sde) and write to engine (postgis)
 
-# Read source table:
-print("Reading rows from {}".format(source_table))
-rows = etl.fromoraclesde(source_conn, source_table, fields=source_fields)
-# Format fields
-rows = rows.rename({v:k for k,v in source_field_map.items()})
+    # Get source table connection
+    source_def = config['BASE_DATA_SOURCES']['opa_active_accounts']
+    source_db_name = source_def['db']
+    source_db_url = config['DATABASES'][source_db_name]
+    source_table = source_def['table']
+    source_field_map = source_def['field_map']
+    source_fields = [field for field in list(source_field_map.values()) if field != 'shape']
+    conn_dsn = source_db_url[source_db_url.index("//") + 2:]
+    conn_user = conn_dsn[:conn_dsn.index(":")]
+    conn_pw = conn_dsn[conn_dsn.index(":") + 1 : conn_dsn.index("@")]
+    conn_db = conn_dsn[conn_dsn.index("@") + 1:]
+    source_conn = cx_Oracle.connect(conn_user, conn_pw, conn_db)
 
-drop_stmt = '''drop table if exists {}'''.format(target_table_name)
-create_stmt = '''create table {} (
-               account_num text,
-               source_address text,
-               unit_num text,
-               geom geometry(Point,2272)
-)'''.format(target_table_name)
-# Create temp target table:
-print("Dropping temp table '{}' if already exists...".format(target_table_name))
-target_cur.execute(drop_stmt)
-print("Creating temp table '{}'...".format(target_table_name))
-target_cur.execute(create_stmt)
-target_conn.commit()
-# Write rows to target:
-print("Writing to temp table '{}'".format(target_table_name))
-rows.topostgis(target_conn, target_table_name)
+    target_dsn = config['DATABASES']['engine']
+    target_user = target_dsn[target_dsn.index("//") + 2:target_dsn.index(":", target_dsn.index("//"))]
+    target_pw = target_dsn[target_dsn.index(":",target_dsn.index(target_user)) + 1:target_dsn.index("@")]
+    target_name = target_dsn[target_dsn.index("/", target_dsn.index("@")) + 1:]
+    target_conn = psycopg2.connect('dbname={db_name} user={db_user} password={db_pw} host=localhost'.format(db_name=target_name, db_user=target_user, db_pw=target_pw))
+    target_cur = target_conn.cursor()
+    target_table_name = 'public.t_opa_active_accounts'
 
-# Update address_parcel by selecting address from opa_property associated with opa_account_num for opa_active_accounts record where opa_address doesn't have pwd_parcel in table, \
-# and add row_id of parcel based on associated pwd_parcel_id in opa_active_accounts table:
+    # Read source table:
+    print("Reading rows from {}".format(source_table))
+    rows = etl.fromoraclesde(source_conn, source_table, fields=source_fields) # Runs in 0:11:48
+    # Format fields
+    rows = rows.rename({v:k for k,v in source_field_map.items()})
 
-update_stmt = '''
-insert into address_parcel (street_address, parcel_source, parcel_row_id, match_type)
-select distinct opanopp.street_address, 'pwd' as parcel_source, pp.id, 'manual' as match_type
-from
-(
- select nopp.street_address, opaaa.account_num, opaaa.geom
- from opa_property opa
- inner join (
-		select street_address
-		from opa_property
-	except (
-		select street_address
-		from address_parcel
-		where parcel_source = 'pwd' and parcel_row_id is not null
-	)
- ) nopp on nopp.street_address = opa.street_address
- inner join t_opa_active_accounts opaaa on opaaa.account_num = opa.account_num
-) opanopp inner join pwd_parcel pp on st_intersects(opanopp.geom, pp.geom)
-'''
+    drop_stmt = '''drop table if exists {}'''.format(target_table_name)
+    create_stmt = '''create table {} (
+                   account_num text,
+                   source_address text,
+                   unit_num text,
+                   geom geometry(Point,2272)
+    )'''.format(target_table_name)
+    # Create temp target table:
+    print("Dropping temp table '{}' if already exists...".format(target_table_name))
+    target_cur.execute(drop_stmt)
+    print("Creating temp table '{}'...".format(target_table_name))
+    target_cur.execute(create_stmt)
+    target_conn.commit()
+    # Write rows to target:
+    print("Writing to temp table '{}'".format(target_table_name))
+    rows.topostgis(target_conn, target_table_name)
 
-print("Updating address_parcel table with manaual opa property geocodes intersecting pwd parcels...")
-target_cur.execute(update_stmt)
-target_conn.commit()
+    # Update address_parcel by selecting address from opa_property associated with opa_account_num for opa_active_accounts record where opa_address doesn't have pwd_parcel in table, \
+    # and add row_id of parcel based on associated pwd_parcel_id in opa_active_accounts table:
 
-print("Cleaning up...")
-# drop temp table:
-target_cur.execute(drop_stmt)
-target_conn.commit()
-# close db connection:
-target_conn.close()
+    update_stmt = '''
+    insert into address_parcel (street_address, parcel_source, parcel_row_id, match_type)
+    select distinct opanopp.street_address, 'pwd' as parcel_source, pp.id, 'manual' as match_type
+    from
+    (
+     select nopp.street_address, opaaa.account_num, opaaa.geom
+     from opa_property opa
+     inner join (
+            select street_address
+            from opa_property
+        except (
+            select street_address
+            from address_parcel
+            where parcel_source = 'pwd' and parcel_row_id is not null
+        )
+     ) nopp on nopp.street_address = opa.street_address
+     inner join t_opa_active_accounts opaaa on opaaa.account_num = opa.account_num
+    ) opanopp inner join pwd_parcel pp on st_intersects(opanopp.geom, pp.geom)
+    '''
+
+    print("Updating address_parcel table with manaual opa property geocodes intersecting pwd parcels...")
+    target_cur.execute(update_stmt)
+    target_conn.commit()
+
+    print("Cleaning up...")
+    # drop temp table:
+    target_cur.execute(drop_stmt)
+    target_conn.commit()
+    # close db connection:
+    target_conn.close()
+    print('Finished in {} seconds'.format(datetime.now() - start))
diff --git a/ais/engine/scripts/load_addresses.py b/ais/engine/scripts/load_addresses.py
index b1eebd08..938c9533 100644
--- a/ais/engine/scripts/load_addresses.py
+++ b/ais/engine/scripts/load_addresses.py
@@ -1,836 +1,800 @@
-import sys
-# import os
-# import csv
-# from copy import deepcopy
 from datetime import datetime
 import datum
 from ais import app
 from ais.models import Address
 from ais.util import parity_for_num, parity_for_range
 from passyunk.parser import PassyunkParser
-# DEV
-# import traceback
-# from pprint import pprint
-
-print('Starting...')
-start = datetime.now()
-
-config = app.config
-Parser = config['PARSER']
-
-parser_tags = config['ADDRESSES']['parser_tags']
-sources = config['ADDRESSES']['sources']
-db = datum.connect(config['DATABASES']['engine'])
-address_table = db['address']
-address_tag_table = db['address_tag']
-source_address_table = db['source_address']
-address_link_table = db['address_link']
-street_segment_table = db['street_segment']
-address_street_table = db['address_street']
-true_range_view_name = 'true_range'
-
-# TODO: something more elegant here.
-true_range_select_stmt = '''
-    select
-         coalesce(r.seg_id, l.seg_id) as seg_id,
-         r.low as true_right_from,
-         r.high as true_right_to,
-         l.low as true_left_from,
-         l.high as true_left_to
-    from (select
-              asr.seg_id,
-              min(a.address_low) as low,
-              greatest(max(a.address_low), max(a.address_high)) as high
-         from address a join address_street asr on a.street_address = asr.street_address
-         group by asr.seg_id, asr.seg_side
-         having asr.seg_id is not null and asr.seg_side = 'R') r
-    full outer join
-         (select
-              asl.seg_id,
-              min(a.address_low) as low,
-              greatest(max(a.address_low), max(a.address_high)) as high
-         from address a join address_street asl on a.street_address = asl.street_address
-         group by asl.seg_id, asl.seg_side
-         having asl.seg_id is not null and asl.seg_side = 'L') l
-    on r.seg_id = l.seg_id
-    order by r.seg_id
-'''
-parcel_layers = config['BASE_DATA_SOURCES']['parcels']
-address_parcel_table = db['address_parcel']
-address_property_table = db['address_property']
-address_error_table = db['address_error']
-WRITE_OUT = True
-
-DEV = False  # This will target a single address
-DEV_ADDRESS = "920-22 W GIRARD AVE"
-DEV_ADDRESS_COMPS = {
-    # 'base_address':      '1 FRANKLIN TOWN BLVD',
-    # 'address_high':     '4726',
-    # 'street_name':      'ALDEN',
-    # 'street_suffix':    'WALK',
-}
-DEV_STREET_NAME = 'W GIRARD AVE'
-
-# Logging stuff.
-address_errors = []
-# Use these maps to write out one error per source address/source name pair.
-source_map = {}  # source_address => [source_names]
-source_address_map = {}  # street_address => [source_addresses]
-
-"""MAIN"""
-
-addresses = []
-street_addresses_seen = set()
-address_tags = []
-parser_address_tags = []
-address_tag_strings = set()  # Pipe-joined addr/key/value triples
-source_addresses = []
-links = []  # dicts of address, relationship, address triples
-parser = PassyunkParser()
-parsed_addresses = {}
-
-if WRITE_OUT:
-    print('Dropping indexes...')
-    for table in (address_table, address_tag_table, source_address_table):
-        table.drop_index('street_address')
-    address_link_table.drop_index('address_1')
-    address_link_table.drop_index('address_2')
-
-    print('Deleting existing addresses...')
-    address_table.delete()
-    print('Deleting existing address tags...')
-    address_tag_table.delete()
-    print('Deleting existing source addresses...')
-    source_address_table.delete()
-    print('Deleting existing address links...')
-    address_link_table.delete()
-    print('Deleting address errors...')
-    address_error_table.delete()
-
-# Loop over address sources
-for source in sources:
-    source_name = source['name']
-
-    # Determine address field mapping: single or comps
-    address_fields = source['address_fields']
-    # Invert for aliases arg on read()
-    aliases = {value: key for key, value in address_fields.items()}
-    if len(address_fields) == 1:
-        if 'street_address' in address_fields:
-            source_type = 'single_field'
-        else:
-            raise ValueError('Unknown address field mapping')
-        preprocessor = source.get('preprocessor')
-    else:
-        source_type = 'comps'
-        # Check for necessary components
-        for field_name in ['address_low', 'street_name']:
-            if not field_name in address_fields and not all(x in address_fields for x in ['base_address', 'unit_num']):
-                raise ValueError('Missing required address field: {}' \
-                                 .format(field_name))
-        if 'preprocessor' not in source:
-            raise ValueError('No preprocessor specified for address source `{}`'.format(source_name))
-        preprocessor = source['preprocessor']
-
-    # Get other params
-    address_fields = source['address_fields']
-    source_fields = list(address_fields.values())
-    if 'tag_fields' in source:
-        # source_fields += [x['source_field'] for x in source['tag_fields']]
-        for tag_field in source['tag_fields']:
-            tag_source_fields = tag_field['source_fields']
-            tag_preprocessor = tag_field.get('preprocessor')
-            if len(tag_source_fields) > 1 and tag_preprocessor is None:
-                raise ValueError("Multiple tag source fields require a preprocessor.")
-            source_fields += tag_source_fields
-    # Make source_fields unique:
-    source_fields = list(set(source_fields))
-    source_db_name = source['db']
-    source_db = datum.connect(config['DATABASES'][source_db_name])
-    source_table = source_db[source['table']]
-    print('Reading from {}...'.format(source_name))
-
-    # Add source fields depending on address type
-    # Not using this since we implemented the `aliases` arg on .read()
-    # if source_type == 'single_field':
-    # source_fields.append('{} AS street_address'\
-    #     .format(address_fields['street_address']))
-    # source_fields.append(address_fields['street_address'])
-    # elif source_type == 'comps':
-    # for address_field_std, address_field in address_fields.items():
-    #     source_fields.append('{} AS {}'\
-    #         .format(address_field, address_field_std))
-    # source_fields.append(address_fields.values())
-
-    where = source['where'] if 'where' in source else None
-
-    # For debugging. (Only fetch a specific address.)
-    if DEV:
-        if source_type == 'single_field':
-            dev_where = "{} = '{}'" \
-                .format(address_fields['street_address'], DEV_ADDRESS)
-        elif source_type == 'comps':
-            clauses = ["{} = {}".format(address_fields[key], value) \
-                       for key, value in DEV_ADDRESS_COMPS.items()]
-            dev_where = ' AND '.join(clauses)
-        if where:
-            where += ' AND ' + dev_where
-        else:
-            where = dev_where
-        source_rows = source_table.read(fields=source_fields, \
-                                        aliases=aliases, where=where)
-    else:
-        if source_type == 'single_field':
-            source_rows = source_table.read(fields=source_fields, \
-                                            aliases=aliases, where=where, return_geom=False)
-        elif source_type == 'comps':
-            source_rows = source_table.read(fields=source_fields, \
-                                            aliases=aliases, where=where, return_geom=False)
-    if not DEV:
-        if not source_rows or len(source_rows) < 1:
-            print(source_rows)
-            raise Exception("Exiting because source table {source_table} is empty.".format(source_table=source_table))
-
-    # Loop over addresses
-    for i, source_row in enumerate(source_rows):
-        if i % 100000 == 0:
-            print(i)
 
-        # Get source address and add source to map. We do this outside the
-        # try statement so that source_address is always properly set for
-        # logging errors.
-        # if source_type == 'single_field':
-        #     source_address = source_row['street_address']
-        # else:
-        #     source_address = preprocessor(source_row)
-
-        # If there's a preprocessor, apply. This could be single field or comps.
-        if preprocessor:
-            source_address = preprocessor(source_row)
-        # Must be a single field
-        else:
-            source_address = source_row['street_address']
+# SET UP LOGGING / QC
+street_warning_map = {}  # street_address => [{reason, notes}]
+street_error_map = {}  # # street_address => {reason, notes}
 
-        if source_address is None:
-            # TODO: it might be helpful to log this, but right now we aren't
-            # logging object IDs so there would be no way to identify the
-            # null address in the source dataset. Just skipping for now.
-            continue
+def main():
+    print('Starting...')
+    start = datetime.now()
+
+    config = app.config
+
+    parser_tags = config['ADDRESSES']['parser_tags']
+    sources = config['ADDRESSES']['sources']
+    db = datum.connect(config['DATABASES']['engine'])
+    address_table = db['address']
+    address_tag_table = db['address_tag']
+    source_address_table = db['source_address']
+    address_link_table = db['address_link']
+    street_segment_table = db['street_segment']
+    address_street_table = db['address_street']
+    true_range_view_name = 'true_range'
+
+    # TODO: something more elegant here.
+    true_range_select_stmt = '''
+        select
+             coalesce(r.seg_id, l.seg_id) as seg_id,
+             r.low as true_right_from,
+             r.high as true_right_to,
+             l.low as true_left_from,
+             l.high as true_left_to
+        from (select
+                  asr.seg_id,
+                  min(a.address_low) as low,
+                  greatest(max(a.address_low), max(a.address_high)) as high
+             from address a join address_street asr on a.street_address = asr.street_address
+             group by asr.seg_id, asr.seg_side
+             having asr.seg_id is not null and asr.seg_side = 'R') r
+        full outer join
+             (select
+                  asl.seg_id,
+                  min(a.address_low) as low,
+                  greatest(max(a.address_low), max(a.address_high)) as high
+             from address a join address_street asl on a.street_address = asl.street_address
+             group by asl.seg_id, asl.seg_side
+             having asl.seg_id is not null and asl.seg_side = 'L') l
+        on r.seg_id = l.seg_id
+        order by r.seg_id
+    '''
+    parcel_layers = config['BASE_DATA_SOURCES']['parcels']
+    address_parcel_table = db['address_parcel']
+    address_property_table = db['address_property']
+    address_error_table = db['address_error']
+    WRITE_OUT = True
+
+    DEV = False  # This will target a single address
+    DEV_ADDRESS = "920-22 W GIRARD AVE"
+    DEV_ADDRESS_COMPS = {
+        # 'base_address':      '1 FRANKLIN TOWN BLVD',
+        # 'address_high':     '4726',
+        # 'street_name':      'ALDEN',
+        # 'street_suffix':    'WALK',
+    }
+    DEV_STREET_NAME = 'W GIRARD AVE'
 
-        source_map.setdefault(source_address, []).append(source_name)
+    # Logging stuff.
+    address_errors = []
+    # Use these maps to write out one error per source address/source name pair.
+    source_map = {}  # source_address => [source_names]
+    source_address_map = {}  # street_address => [source_addresses]
 
-        # Make sure this is reset on each run (also for logging)
-        street_address = None
+    """MAIN"""
 
-        try:
-            # Try parsing
-            parsed_address = parsed_addresses.get(source_address)
-            if parsed_address is None:
-                # Passyunk no longer raising errors
-                try:
-                    parsed_address = parser.parse(source_address)
-                    parsed_addresses[source_address] = parsed_address
+    addresses = []
+    street_addresses_seen = set()
+    address_tags = []
+    parser_address_tags = []
+    address_tag_strings = set()  # Pipe-joined addr/key/value triples
+    source_addresses = []
+    links = []  # dicts of address, relationship, address triples
+    parser = PassyunkParser()
+    parsed_addresses = {}
 
-                except:
-                    raise ValueError('Could not parse')
+    if WRITE_OUT:
+        print('Dropping indexes...')
+        for table in (address_table, address_tag_table, source_address_table):
+            table.drop_index('street_address')
+        address_link_table.drop_index('address_1')
+        address_link_table.drop_index('address_2')
+
+        print('Deleting existing addresses...')
+        address_table.delete()
+        print('Deleting existing address tags...')
+        address_tag_table.delete()
+        print('Deleting existing source addresses...')
+        source_address_table.delete()
+        print('Deleting existing address links...')
+        address_link_table.delete()
+        print('Deleting address errors...')
+        address_error_table.delete()
+
+    # Loop over address sources
+    for source in sources:
+        source_name = source['name']
+
+        # Determine address field mapping: single or comps
+        address_fields = source['address_fields']
+        # Invert for aliases arg on read()
+        aliases = {value: key for key, value in address_fields.items()}
+        if len(address_fields) == 1:
+            if 'street_address' in address_fields:
+                source_type = 'single_field'
+            else:
+                raise ValueError('Unknown address field mapping')
+            preprocessor = source.get('preprocessor')
+        else:
+            source_type = 'comps'
+            # Check for necessary components
+            for field_name in ['address_low', 'street_name']:
+                if not field_name in address_fields and not all(x in address_fields for x in ['base_address', 'unit_num']):
+                    raise ValueError('Missing required address field: {}' \
+                                     .format(field_name))
+            if 'preprocessor' not in source:
+                raise ValueError('No preprocessor specified for address source `{}`'.format(source_name))
+            preprocessor = source['preprocessor']
+
+        # Get other params
+        address_fields = source['address_fields']
+        source_fields = list(address_fields.values())
+        if 'tag_fields' in source:
+            for tag_field in source['tag_fields']:
+                tag_source_fields = tag_field['source_fields']
+                tag_preprocessor = tag_field.get('preprocessor')
+                if len(tag_source_fields) > 1 and tag_preprocessor is None:
+                    raise ValueError("Multiple tag source fields require a preprocessor.")
+                source_fields += tag_source_fields
+        # Make source_fields unique:
+        source_fields = list(set(source_fields))
+        source_db_name = source['db']
+        source_db = datum.connect(config['DATABASES'][source_db_name])
+        source_table = source_db[source['table']]
+        print('Reading from {}...'.format(source_name))
+
+        where = source['where'] if 'where' in source else None
+
+        # For debugging. (Only fetch a specific address.)
+        if DEV:
+            if source_type == 'single_field':
+                dev_where = "{} = '{}'" \
+                    .format(address_fields['street_address'], DEV_ADDRESS)
+            elif source_type == 'comps':
+                clauses = ["{} = '{}'".format(address_fields[key], value) \
+                           for key, value in DEV_ADDRESS_COMPS.items()]
+                dev_where = ' AND '.join(clauses)
+            if where:
+                where += ' AND ' + dev_where
+            else:
+                where = dev_where
+            source_rows = source_table.read(fields=source_fields, \
+                                            aliases=aliases, where=where)
+        else:
+            if source_type == 'single_field':
+                source_rows = source_table.read(fields=source_fields, \
+                                                aliases=aliases, where=where, return_geom=False)
+            elif source_type == 'comps':
+                source_rows = source_table.read(fields=source_fields, \
+                                                aliases=aliases, where=where, return_geom=False)
+
+        if not DEV: 
+            if not source_rows or len(source_rows) < 2:
+                raise Exception("Exiting because source table {source_table} is empty.".format(source_table=source_table))
+
+        # Loop over addresses
+        for i, source_row in enumerate(source_rows):
+            if i % 100000 == 0:
+                print(i)
+            # Get source address and add source to map. We do this outside the
+            # try statement so that source_address is always properly set for
+            # logging errors.
+
+            # If there's a preprocessor, apply. This could be single field or comps.
+            if preprocessor:
+                source_address = preprocessor(source_row)
+            # Must be a single field
+            else:
+                source_address = source_row['street_address']
+
+            if source_address is None:
+                # TODO: it might be helpful to log this, but right now we aren't
+                # logging object IDs so there would be no way to identify the
+                # null address in the source dataset. Just skipping for now.
+                continue
 
-            if parsed_address['type'] == "none":
-                raise ValueError('Unknown address type')
+            source_map.setdefault(source_address, []).append(source_name)
 
-            address = Address(parsed_address)
+            # Make sure this is reset on each run (also for logging)
+            street_address = None
 
-            # Get street address and map to source address
-            street_address = address.street_address
-            _source_addresses = source_address_map.setdefault(street_address, [])
-            if not source_address in _source_addresses:
-                _source_addresses.append(source_address)
-
-            # Check for zero address
-            if address.address_low == 0:
-                raise ValueError('Low number is zero')
-
-            # Add address
-            if not street_address in street_addresses_seen:
-                addresses.append(address)
-                street_addresses_seen.add(street_address)
-
-            # Make source address
-            source_address_dict = {
-                'source_name': source_name,
-                'source_address': source_address,
-                'street_address': street_address,
-            }
-            source_addresses.append(source_address_dict)
-
-            # Make address tags
-            for tag_field in source.get('tag_fields', []):
+            try:
+                # Try parsing
+                parsed_address = parsed_addresses.get(source_address)
+                if parsed_address is None:
+                    # Passyunk no longer raising errors
+                    try:
+                        parsed_address = parser.parse(source_address)
+                        parsed_addresses[source_address] = parsed_address
+
+                    except:
+                        raise ValueError('Could not parse')
+
+                if parsed_address['type'] == "none":
+                    raise ValueError('Unknown address type')
+
+                address = Address(parsed_address)
+
+                # Get street address and map to source address
+                street_address = address.street_address
+                _source_addresses = source_address_map.setdefault(street_address, [])
+                if not source_address in _source_addresses:
+                    _source_addresses.append(source_address)
+
+                # Check for zero address
+                if address.address_low == 0:
+                    raise ValueError('Low number is zero')
+
+                # Add address
+                if not street_address in street_addresses_seen:
+                    addresses.append(address)
+                    street_addresses_seen.add(street_address)
+
+                # Make source address
+                source_address_dict = {
+                    'source_name': source_name,
+                    'source_address': source_address,
+                    'street_address': street_address,
+                }
+                source_addresses.append(source_address_dict)
 
-                tag_preprocessor = tag_field.get('preprocessor')
-                source_fields = tag_field['source_fields']
-                if tag_preprocessor:
-                    value = tag_preprocessor(source_row)
-                else:
-                    source_field = source_fields[0]
-                    value = source_row[source_field]
-
-                # source_field = tag_field['source_field']
-                # value = source_row[source_field]
-
-                # Skip empty tags
-                if value is None or len(str(value).strip()) == 0:
-                    continue
+                # Make address tags
+                for tag_field in source.get('tag_fields', []):
 
-                # Make uppercase
-                if isinstance(value, str):
-                    value = value.upper()
+                    tag_preprocessor = tag_field.get('preprocessor')
+                    source_fields = tag_field['source_fields']
+                    if tag_preprocessor:
+                        value = tag_preprocessor(source_row)
+                    else:
+                        source_field = source_fields[0]
+                        value = source_row[source_field]
+
+                    # Skip empty tags
+                    if value is None or len(str(value).strip()) == 0:
+                        continue
+
+                    # Make uppercase
+                    if isinstance(value, str):
+                        value = value.upper()
+
+                    key = tag_field['key']
+
+                    # Make address tag string to check if we already added this tag
+                    address_tag_string_vals = [
+                        street_address,
+                        key,
+                        value,
+                    ]
+                    address_tag_string = '|'.join([str(x) for x in \
+                                                   address_tag_string_vals])
+
+                    if str(value).strip() and not address_tag_string in address_tag_strings:
+                        address_tag = {
+                            'street_address': street_address,
+                            'key': key,
+                            'value': value
+                        }
+                        address_tags.append(address_tag)
+                        address_tag_strings.add(address_tag_string)
+
+                # If it's a unit or low num suffix, make sure we have the base
+                # address
+                if not address.is_base:
+                    base_address = address.base_address
+                    if not base_address in street_addresses_seen:
+                        # Reparse:
+                        base_address_obj = Address(base_address)
+
+                        # TEMP: a small handful of addresses aren't reparsable,
+                        # meaning they give different results on the second round
+                        # of parsing.
+                        # https://github.com/CityOfPhiladelphia/passyunk/issues/11
+                        # TODO: write a parser test for this.
+                        if base_address != base_address_obj.street_address:
+                            raise ValueError('Base address is not reparsable')
+
+                        addresses.append(base_address_obj)
+                        street_addresses_seen.add(base_address)
+                        # Make source address
+                        source_address_dict = {
+                            'source_name': 'AIS',
+                            'source_address': base_address,
+                            'street_address': base_address,
+                        }
+                        # Add base AIS created addresses to source_address table
+                        source_addresses.append(source_address_dict)
+                        # Add to source address map
+                        _source_addresses = source_address_map.setdefault(base_address, [])
+                        if not source_address in _source_addresses:
+                            _source_addresses.append(source_address)
+
+                # # If it's a range, make sure we have all the child addresses
+
+            except ValueError as e:
+                address_error = {
+                    'source_name': source_name,
+                    'source_address': source_address,
+                    'street_address': street_address or '',
+                    'level': 'error',
+                    'reason': str(e),
+                    # TODO: haven't needed these so far, but they should be passed
+                    # in with the exception if we ever do.
+                    'notes': '',
+                }
+                address_errors.append(address_error)
 
-                key = tag_field['key']
-                # value = source_row[source_field]
+        if WRITE_OUT:
+            print('Writing {} address tags...'.format(len(address_tags)))
+            address_tag_table.write(address_tags, chunk_size=150000)
+            address_tags = []
+            address_tag_strings = set()
 
-                # Make address tag string to check if we already added this tag
-                address_tag_string_vals = [
-                    street_address,
-                    key,
-                    value,
-                ]
-                address_tag_string = '|'.join([str(x) for x in \
-                                               address_tag_string_vals])
+            print('Writing {} source addresses...'.format(len(source_addresses)))
+            source_address_table.write(source_addresses, chunk_size=150000)
+            source_addresses = []
 
-                if str(value).strip() and not address_tag_string in address_tag_strings:
-                    address_tag = {
-                        'street_address': street_address,
-                        # 'key':              tag_field['key'],
-                        'key': key,
-                        # 'value':            source_row[source_field]
-                        'value': value
-                    }
-                    address_tags.append(address_tag)
-                    address_tag_strings.add(address_tag_string)
-
-            # If it's a unit or low num suffix, make sure we have the base
-            # address
-            if not address.is_base:
-                base_address = address.base_address
-                if not base_address in street_addresses_seen:
-                    # Reparse:
-                    base_address_obj = Address(base_address)
-
-                    # TEMP: a small handful of addresses aren't reparsable,
-                    # meaning they give different results on the second round
-                    # of parsing.
-                    # https://github.com/CityOfPhiladelphia/passyunk/issues/11
-                    # TODO: write a parser test for this.
-                    if base_address != base_address_obj.street_address:
-                        raise ValueError('Base address is not reparsable')
-
-                    addresses.append(base_address_obj)
-                    street_addresses_seen.add(base_address)
-                    # Make source address
-                    source_address_dict = {
-                        'source_name': 'AIS',
-                        'source_address': base_address,
-                        'street_address': base_address,
-                    }
-                    # Add base AIS created addresses to source_address table
-                    source_addresses.append(source_address_dict)
-                    # Add to source address map
-                    _source_addresses = source_address_map.setdefault(base_address, [])
-                    if not source_address in _source_addresses:
-                        _source_addresses.append(source_address)
-
-            # # If it's a range, make sure we have all the child addresses
-            # for child_obj in address.child_addresses:
-            #     child_street_address = child_obj.street_address
-            #     if not child_street_address in street_addresses_seen:
-            #         addresses.append(child_obj)
-            #         street_addresses_seen.add(child_street_address)
-            #
-            #         # Add to source address map
-            #         _source_addresses = source_address_map.setdefault(child_street_address, [])
-            #         if not source_address in _source_addresses:
-            #             _source_addresses.append(source_address)
-
-        except ValueError as e:
-            address_error = {
-                'source_name': source_name,
-                'source_address': source_address,
-                'street_address': street_address or '',
-                'level': 'error',
-                'reason': str(e),
-                # TODO: haven't needed these so far, but they should be passed
-                # in with the exception if we ever do.
-                'notes': '',
-            }
-            address_errors.append(address_error)
+    insert_rows = [dict(x) for x in addresses]
 
     if WRITE_OUT:
-        print('Writing {} address tags...'.format(len(address_tags)))
-        address_tag_table.write(address_tags, chunk_size=150000)
-        address_tags = []
-        address_tag_strings = set()
-
-        print('Writing {} source addresses...'.format(len(source_addresses)))
-        source_address_table.write(source_addresses, chunk_size=150000)
-        source_addresses = []
-
-    #source_db.close()
-
-insert_rows = [dict(x) for x in addresses]
-
-if WRITE_OUT:
-    print('Writing {} addresses...'.format(len(addresses)))
-    address_table.write(insert_rows, chunk_size=150000)
-del insert_rows
-
-print('Making {} parser_address_tags...'.format(len(parsed_addresses)))
-for source_address, comps in parsed_addresses.items():
-    comps = comps.get('components')
-    street_address = comps.get('output_address', '')
-    if comps and comps.get('address') and comps.get('address')['addrnum_type'] == 'Range':
-        continue
-    for tag_field, path in parser_tags.items():
-        value = comps
-        key = tag_field
-        for item in path:
-            if __name__ == '__main__':
+        print('Writing {} addresses...'.format(len(addresses)))
+        address_table.write(insert_rows, chunk_size=150000)
+    del insert_rows
+
+    print('Making {} parser_address_tags...'.format(len(parsed_addresses)))
+    for source_address, comps in parsed_addresses.items():
+        comps = comps.get('components')
+        street_address = comps.get('output_address', '')
+        if comps and comps.get('address') and comps.get('address')['addrnum_type'] == 'Range':
+            continue
+        for tag_field, path in parser_tags.items():
+            value = comps
+            key = tag_field
+            for item in path:
                 value = value.get(item)
-        value = str(value).strip()
-        value = '' if value == 'None' else value
-        parser_address_tag_string_vals = [
-            street_address,
-            key,
-            value,
-        ]
-        parser_address_tag_string = '|'.join([str(x) for x in \
-                                              parser_address_tag_string_vals])
-        if value and not parser_address_tag_string in address_tag_strings:
-            parser_address_tag = {
-                'street_address': street_address,
-                'key': key,
-                'value': value
-            }
-            address_tag_strings.add(parser_address_tag_string)
-            parser_address_tags.append(parser_address_tag)
-
-if WRITE_OUT:
-    print("Writing tags")
-    address_tag_table.write(parser_address_tags, chunk_size=150000)
-del parser_address_tags
-address_tag_strings = set()
-
-# spatial parcel id tags from address points:
-print("Adding dor_parcel_id tags for address points via spatial join...")
-dor_parcel_id_tag_stmt = '''
-insert into address_tag (street_address, key, value)
-select ng911.street_address, 'dor_parcel_id', dor.parcel_id 
-from ng911_address_point ng911 
-join (
-    select street_address from address_tag 
-    except (
-        select street_address from address_tag where key = 'dor_parcel_id'
-        )
-    ) atag on atag.street_address = ng911.street_address
-join dor_parcel dor on st_dwithin(dor.geom, ng911.geom,2)
-'''
-db.execute(dor_parcel_id_tag_stmt)
-
-print("Adding pwd_parcel_id tags for address points via spatial join...")
-pwd_parcel_id_tag_stmt = '''
-insert into address_tag (street_address, key, value)
-select ng911.street_address, 'pwd_parcel_id', pwd.parcel_id 
-from ng911_address_point ng911 
-join (
-    select street_address from address_tag 
-    except (
-        select street_address from address_tag where key = 'pwd_parcel_id'
-        )
-    ) atag on atag.street_address = ng911.street_address
-join pwd_parcel pwd on st_dwithin(pwd.geom, ng911.geom,2)
+            value = str(value).strip()
+            value = '' if value == 'None' else value
+            parser_address_tag_string_vals = [
+                street_address,
+                key,
+                value,
+            ]
+            parser_address_tag_string = '|'.join([str(x) for x in \
+                                                  parser_address_tag_string_vals])
+            if value and not parser_address_tag_string in address_tag_strings:
+                parser_address_tag = {
+                    'street_address': street_address,
+                    'key': key,
+                    'value': value
+                }
+                address_tag_strings.add(parser_address_tag_string)
+                parser_address_tags.append(parser_address_tag)
+
+    if WRITE_OUT:
+        print("Writing tags")
+        address_tag_table.write(parser_address_tags, chunk_size=150000)
+    del parser_address_tags
+    address_tag_strings = set()
+
+    # spatial parcel id tags from address points:
+    print("Adding dor_parcel_id tags for address points via spatial join...")
+    dor_parcel_id_tag_stmt = '''
+    insert into address_tag (street_address, key, value)
+    select ng911.street_address, 'dor_parcel_id', dor.parcel_id 
+    from ng911_address_point ng911 
+    join (
+        select street_address from address_tag 
+        except (
+            select street_address from address_tag where key = 'dor_parcel_id'
+            )
+        ) atag on atag.street_address = ng911.street_address
+    join dor_parcel dor on st_dwithin(dor.geom, ng911.geom,2)
 '''
-db.execute(pwd_parcel_id_tag_stmt)
-db.save()
-###############################################################################
-# ADDRESS LINKS
-###############################################################################
-print('** ADDRESS LINKS **')
-print('Indexing addresses...')
-street_address_map = {}  # street_full => [addresses]
-street_range_map = {}  # street_full => [range addresses]
-base_address_map = {}  # base_address => [unit addresses]
-for i, address in enumerate(addresses):
-    if i % 100000 == 0:
-        print(i)
-
-    street_full = address.street_full
-    if not street_full in street_address_map:
-        street_address_map[street_full] = []
-        street_range_map[street_full] = []
-    street_address_map[street_full].append(address)
-
-    # TODO: Include addresses with units in street range map? - base address or include unit?
-    if address.address_high is not None and address.unit_type is None:
-        street_range_map[street_full].append(address)
-
-    base_address = address.base_address  # TODO: handle addresses with number suffixes using base_address_no_suffix
-    # # Get 'has_base' link for addresses with units
-    if address.unit_type is not None:
-        if not base_address in base_address_map:
-            base_address_map[base_address] = []
-        base_address_map[base_address].append(address)
-
-# Loop over addresses
-print('Making address links...')
-
-generic_unit_types = set(['#', 'APT', 'UNIT'])
-apt_unit_types = set(['APT', 'UNIT'])
-
-new_addresses = []
-addresses_seen = set()
-for i, address in enumerate(addresses):
-    if i % 100000 == 0:
-        print(i)
-
-    # # Include address with unit_type in 'has_base' link set
-    if address.unit_type is not None:
-    # if address.unit_type is not None or address.address_low_suffix is not None:
-        # Base link
-        base_link = {
-            'address_1': address.street_address,
-            'relationship': 'has base',
-            'address_2':       address.base_address,
-            # 'address_2': address.base_address_no_suffix,
-        }
-        links.append(base_link)
+    db.execute(dor_parcel_id_tag_stmt)
+
+    print("Adding pwd_parcel_id tags for address points via spatial join...")
+    pwd_parcel_id_tag_stmt = '''
+    insert into address_tag (street_address, key, value)
+    select ng911.street_address, 'pwd_parcel_id', pwd.parcel_id 
+    from ng911_address_point ng911 
+    join (
+        select street_address from address_tag 
+        except (
+            select street_address from address_tag where key = 'pwd_parcel_id'
+            )
+        ) atag on atag.street_address = ng911.street_address
+    join pwd_parcel pwd on st_dwithin(pwd.geom, ng911.geom,2)
+    '''
+    db.execute(pwd_parcel_id_tag_stmt)
+    db.save()
+    ###############################################################################
+    # ADDRESS LINKS
+    ###############################################################################
+    print('** ADDRESS LINKS **')
+    print('Indexing addresses...')
+    street_address_map = {}  # street_full => [addresses]
+    street_range_map = {}  # street_full => [range addresses]
+    base_address_map = {}  # base_address => [unit addresses]
+    for i, address in enumerate(addresses):
+        if i % 100000 == 0:
+            print(i)
 
-        # Sibling generic unit links
-        # These relate unit addresses to all other addresses that share the same
-        # generic unit. Bidirectional.
-        if address.unit_type in generic_unit_types:
-            base_address = address.base_address
-            base_matches = base_address_map[base_address]
-            for base_match in base_matches:
-                if address.unit_num == base_match.unit_num and \
-                                address.unit_type != base_match.unit_type and \
-                                base_match.unit_type in generic_unit_types:
-                    matches_unit_link_1 = {
-                        'address_1': address.street_address,
-                        'relationship': 'matches unit',
-                        'address_2': base_match.street_address,
-                    }
-                    matches_unit_link_2 = {
-                        'address_1': base_match.street_address,
-                        'relationship': 'matches unit',
-                        'address_2': address.street_address,
-                    }
-                    links.append(matches_unit_link_1)
-                    links.append(matches_unit_link_2)
-
-            # Parent generic unit link
-            # ex. 902 PINE ST APT 2R => 902 PINE ST # 2R
-            # We only want to create these if the generic (#) address was seen
-            # in the data. Don't make like we make child addresses for ranges.
-            if address.unit_type in apt_unit_types:
-                generic_unit = address.generic_unit
-                if generic_unit in street_addresses_seen:
-                    parent_unit_link = {
-                        'address_1': address.street_address,
-                        'relationship': 'has generic unit',
-                        'address_2': generic_unit,
-                    }
-                    links.append(parent_unit_link)
-
-    # Child link
-    elif address.address_high is None:
-        address_low = address.address_low
-        address_suffix = address.address_low_suffix
-        parity = address.parity
-        ranges_on_street = street_range_map[address.street_full]
-
-        for range_on_street in ranges_on_street:
-            if (range_on_street.address_low <= address_low <= \
-                        range_on_street.address_high) and \
-                            range_on_street.parity == parity and \
-                            range_on_street.address_low_suffix == address_suffix:
-                child_link = {
-                    'address_1': address.street_address,
-                    'relationship': 'in range',
-                    'address_2': range_on_street.street_address,
-                }
-                links.append(child_link)
-                break
-
-    # New method: create links (and address objects) for all possible address in range of every ranged address
-    if address.address_high:
-        address_low = address.address_low
-        address_high = address.address_high
-        address_suffix = address.address_low_suffix
         street_full = address.street_full
-        street_address = address.street_address
-        unit_type = address.unit_type
-        unit_num = address.unit_num
-        base_address = address.base_address
-        base_address_no_suffix = address.base_address_no_suffix
-        parity = address.parity
+        if not street_full in street_address_map:
+            street_address_map[street_full] = []
+            street_range_map[street_full] = []
+        street_address_map[street_full].append(address)
+
+        # TODO: Include addresses with units in street range map? - base address or include unit?
+        if address.address_high is not None and address.unit_type is None:
+            street_range_map[street_full].append(address)
+
+        base_address = address.base_address  # TODO: handle addresses with number suffixes using base_address_no_suffix
+        # # Get 'has_base' link for addresses with units
+        if address.unit_type is not None:
+            if not base_address in base_address_map:
+                base_address_map[base_address] = []
+            base_address_map[base_address].append(address)
 
-        for x in range(address_low, address_high + 1, 2):
+    # Loop over addresses
+    print('Making address links...')
 
-            child_address_low_full = str(x) + address_suffix if address_suffix else str(x)
-            child_address_comps = (child_address_low_full, street_full)
-            child_address = " ".join(filter(None, child_address_comps))
-            try:
-                child_obj = Address(child_address)
-                child_street_address = child_obj.street_address
-                if not child_street_address in street_addresses_seen:
-                    # Check for zero address
-                    if child_obj.address_low == 0:
-                        raise ValueError('Low number is zero')
-                    new_addresses.append(child_obj)
-                    # Make source address
-                    source_address_dict = {
-                        'source_name': 'AIS',
-                        'source_address': child_street_address,
-                        'street_address': child_street_address,
-                    }
-                    # Add in-range AIS created addresses to source_address table
-                    source_addresses.append(source_address_dict)
-                    street_addresses_seen.add(child_street_address)
-                    child_link = {
-                        'address_1': child_street_address,
-                        'relationship': 'in range',
-                        'address_2': address.base_address,
-                        # 'address_2': base_address_no_suffix,
-                    }
-                    links.append(child_link)
-            except ValueError:
-                print('Could not parse new address: {}'.format(child_address))
-                continue
+    generic_unit_types = set(['#', 'APT', 'UNIT'])
+    apt_unit_types = set(['APT', 'UNIT'])
+
+    new_addresses = []
+    for i, address in enumerate(addresses):
+        if i % 100000 == 0:
+            print(i)
+
+        # # Include address with unit_type in 'has_base' link set
+        if address.unit_type is not None:
+            # if address.unit_type is not None or address.address_low_suffix is not None:
+            # Base link
+            base_link = {
+                'address_1': address.street_address,
+                'relationship': 'has base',
+                'address_2':       address.base_address,
+            }
+            links.append(base_link)
+
+            # Sibling generic unit links
+            # These relate unit addresses to all other addresses that share the same
+            # generic unit. Bidirectional.
+            if address.unit_type in generic_unit_types:
+                base_address = address.base_address
+                base_matches = base_address_map[base_address]
+                for base_match in base_matches:
+                    if address.unit_num == base_match.unit_num and \
+                                    address.unit_type != base_match.unit_type and \
+                                    base_match.unit_type in generic_unit_types:
+                        matches_unit_link_1 = {
+                            'address_1': address.street_address,
+                            'relationship': 'matches unit',
+                            'address_2': base_match.street_address,
+                        }
+                        matches_unit_link_2 = {
+                            'address_1': base_match.street_address,
+                            'relationship': 'matches unit',
+                            'address_2': address.street_address,
+                        }
+                        links.append(matches_unit_link_1)
+                        links.append(matches_unit_link_2)
+
+                # Parent generic unit link
+                # ex. 902 PINE ST APT 2R => 902 PINE ST # 2R
+                # We only want to create these if the generic (#) address was seen
+                # in the data. Don't make like we make child addresses for ranges.
+                if address.unit_type in apt_unit_types:
+                    generic_unit = address.generic_unit
+                    if generic_unit in street_addresses_seen:
+                        parent_unit_link = {
+                            'address_1': address.street_address,
+                            'relationship': 'has generic unit',
+                            'address_2': generic_unit,
+                        }
+                        links.append(parent_unit_link)
+
+        # Child link
+        elif address.address_high is None:
+            address_low = address.address_low
+            address_suffix = address.address_low_suffix
+            parity = address.parity
+            ranges_on_street = street_range_map[address.street_full]
 
-        # Overlap link
-        if street_full in street_range_map:
-            ranges_on_street = street_range_map[street_full]
             for range_on_street in ranges_on_street:
-                if street_address != range_on_street.street_address \
-                        and range_on_street.parity == parity \
-                        and base_address != range_on_street.base_address \
-                        and base_address_no_suffix != range_on_street.base_address_no_suffix \
-                        and unit_type == range_on_street.unit_type \
-                        and unit_num == range_on_street.unit_num \
-                        and (
-                                            range_on_street.address_low <= address_low <= range_on_street.address_high
-                                or range_on_street.address_low <= address_high <= range_on_street.address_high
-                        ):
+                if (range_on_street.address_low <= address_low <= \
+                            range_on_street.address_high) and \
+                                range_on_street.parity == parity and \
+                                range_on_street.address_low_suffix == address_suffix:
                     child_link = {
-                        'address_1': street_address,
-                        'relationship': 'overlaps',
+                        'address_1': address.street_address,
+                        'relationship': 'in range',
                         'address_2': range_on_street.street_address,
                     }
-                    # 'overlaps' links are bi-directional
-                    child_link_rev = {
-                        'address_1': range_on_street.street_address,
-                        'relationship': 'overlaps',
-                        'address_2': street_address,
-                    }
                     links.append(child_link)
-                    links.append(child_link_rev)
+                    break
 
-# Remove any duplicates in link list
-links = [dict(t) for t in set([tuple(d.items()) for d in links])]
+        # New method: create links (and address objects) for all possible address in range of every ranged address
+        if address.address_high:
+            address_low = address.address_low
+            address_high = address.address_high
+            address_suffix = address.address_low_suffix
+            street_full = address.street_full
+            street_address = address.street_address
+            unit_type = address.unit_type
+            unit_num = address.unit_num
+            base_address = address.base_address
+            base_address_no_suffix = address.base_address_no_suffix
+            parity = address.parity
 
-if WRITE_OUT:
-    print('Writing address links...')
-    address_link_table.write(links, chunk_size=150000)
-    print('Created {} address links'.format(len(links)))
+            for x in range(address_low, address_high + 1, 2):
 
-del links
+                child_address_low_full = str(x) + address_suffix if address_suffix else str(x)
+                child_address_comps = (child_address_low_full, street_full)
+                child_address = " ".join(filter(None, child_address_comps))
+                try:
+                    child_obj = Address(child_address)
+                    child_street_address = child_obj.street_address
+                    if not child_street_address in street_addresses_seen:
+                        # Check for zero address
+                        if child_obj.address_low == 0:
+                            raise ValueError('Low number is zero')
+                        new_addresses.append(child_obj)
+                        # Make source address
+                        source_address_dict = {
+                            'source_name': 'AIS',
+                            'source_address': child_street_address,
+                            'street_address': child_street_address,
+                        }
+                        # Add in-range AIS created addresses to source_address table
+                        source_addresses.append(source_address_dict)
+                        street_addresses_seen.add(child_street_address)
+                        child_link = {
+                            'address_1': child_street_address,
+                            'relationship': 'in range',
+                            'address_2': address.base_address,
+                        }
+                        links.append(child_link)
+                except ValueError:
+                    continue
 
-insert_rows = [dict(x) for x in new_addresses]
-if WRITE_OUT:
-    print("Writing {} new addresses... ".format(len(new_addresses)))
-    address_table.write(insert_rows, chunk_size=150000)
+            # Overlap link
+            if street_full in street_range_map:
+                ranges_on_street = street_range_map[street_full]
+                for range_on_street in ranges_on_street:
+                    if street_address != range_on_street.street_address \
+                            and range_on_street.parity == parity \
+                            and base_address != range_on_street.base_address \
+                            and base_address_no_suffix != range_on_street.base_address_no_suffix \
+                            and unit_type == range_on_street.unit_type \
+                            and unit_num == range_on_street.unit_num \
+                            and (
+                                                range_on_street.address_low <= address_low <= range_on_street.address_high
+                                    or range_on_street.address_low <= address_high <= range_on_street.address_high
+                            ):
+                        child_link = {
+                            'address_1': street_address,
+                            'relationship': 'overlaps',
+                            'address_2': range_on_street.street_address,
+                        }
+                        # 'overlaps' links are bi-directional
+                        child_link_rev = {
+                            'address_1': range_on_street.street_address,
+                            'relationship': 'overlaps',
+                            'address_2': street_address,
+                        }
+                        links.append(child_link)
+                        links.append(child_link_rev)
+
+    # Remove any duplicates in link list
+    links = [dict(t) for t in set([tuple(d.items()) for d in links])]
 
-    print('Writing {} base and in-range AIS source addresses...'.format(len(source_addresses)))
-    source_address_table.write(source_addresses, chunk_size=150000)
-    source_addresses = []
+    if WRITE_OUT:
+        print('Writing address links...')
+        address_link_table.write(links, chunk_size=150000)
+        print('Created {} address links'.format(len(links)))
 
-source_addresses = []
-del insert_rows
-del street_addresses_seen
-# ###############################################################################
-# # ADDRESS-STREETS
-# ###############################################################################
+    del links
 
-print('** ADDRESS-STREETS **')
+    insert_rows = [dict(x) for x in new_addresses]
+    if WRITE_OUT:
+        print("Writing {} new addresses... ".format(len(new_addresses)))
+        address_table.write(insert_rows, chunk_size=150000)
 
-# SET UP LOGGING / QC
-street_warning_map = {}  # street_address => [{reason, notes}]
-street_error_map = {}  # # street_address => {reason, notes}
+        print('Writing {} base and in-range AIS source addresses...'.format(len(source_addresses)))
+        source_address_table.write(source_addresses, chunk_size=150000)
+        source_addresses = []
 
+    source_addresses = []
+    del insert_rows
+    del street_addresses_seen
+    # ###############################################################################
+    # # ADDRESS-STREETS
+    # ###############################################################################
 
-class ContinueIteration(Exception):
-    pass
+    print('** ADDRESS-STREETS **')
 
+    class ContinueIteration(Exception):
+        pass
 
-def had_street_warning(street_address, reason, notes=None):
-    '''
-    Convenience function to log street warnings as they happen.
-    '''
-    global street_warning_map
-    address_warnings = street_warning_map.setdefault(street_address, [])
-    warning = {
-        'reason': reason,
-        'notes': notes or '',
-    }
-    address_warnings.append(warning)
+    def had_street_warning(street_address, reason, notes=None):
+        '''
+        Convenience function to log street warnings as they happen.
+        '''
+        global street_warning_map
+        address_warnings = street_warning_map.setdefault(street_address, [])
+        warning = {
+            'reason': reason,
+            'notes': notes or '',
+        }
+        address_warnings.append(warning)
+
+    def had_street_error(street_address, reason, notes=None):
+        '''
+        This is a wrapper around had_street_warning that raises an error.
+        Technically these are written out as warnings since they are non-fatal.
+        '''
+        global street_error_map
+        street_error_map[street_address] = {'reason': reason, 'notes': notes}
+        had_street_warning(street_address, reason, notes=notes)
+        raise ContinueIteration
+
+    # START WORK
+    if WRITE_OUT:
+        print('Deleting existing address-streets...')
+        address_street_table.delete()
+
+    print('Reading street segments...')
+    seg_fields = [
+        'seg_id',
+        'street_full',
+        'left_from',
+        'left_to',
+        'right_from',
+        'right_to'
+    ]
+    seg_map = {}
+    seg_rows = street_segment_table.read(
+        fields=seg_fields, sort=["STREET_FULL", "LEFT_FROM", "RIGHT_FROM"]
+    )
+    for seg_row in seg_rows:
+        street_full = seg_row['street_full']
+        street_full_segs = seg_map.setdefault(street_full, [])
+        street_full_segs.append(seg_row)
+
+    address_streets = []
+    base_address_map = {}  # base_address => {seg_id, seg_side, had_alias}
+
+    print('Making address-streets...')
+    addresses = addresses + new_addresses
+    for address in addresses:
+        try:
+            street_address = address.street_address
+            base_address = address.base_address
+            # If the base address already had an error, raise it again
+            if base_address in street_error_map:
+                error = street_error_map[base_address]
+                reason = error['reason']
+                notes = error['notes']
+                had_street_error(street_address, reason, notes=notes)
+
+            match = None
+            had_alias = None
+
+            # If we've already seen the base address before, used cached values
+            if base_address in base_address_map:
+                # Make match, also
+                match = base_address_map[base_address]
+                seg_id = match['seg_id']
+                seg_side = match['seg_side']
+                had_alias = match['had_alias']
+
+                # If there were warnings, raise them again
+                for warning in street_warning_map.get(base_address, []):
+                    had_street_warning(
+                        street_address,
+                        warning['reason'],
+                        notes=warning.get('notes')
+                    )
+
+            # Otherwise this is a new address
+            else:
+                # There are some types of warnings we only want to write out if a
+                # more serious error isn't raised first. Keep these here, append
+                # during the seg loop, then call had_street_warning for each one
+                # if we get to that point.
+                deferred_warnings = []
+
+                # Check for valid street
+                address_low = address.address_low
+                address_high = address.address_high
+                street_full = address.street_full
+                address_parity = address.parity
+                if not street_full in seg_map:
+                    had_street_error(street_address, 'Not a valid street')
 
+                matching_segs = seg_map[street_full]
+                matching_seg = None
+                matching_side = None
 
-def had_street_error(street_address, reason, notes=None):
-    '''
-    This is a wrapper around had_street_warning that raises an error.
-    Technically these are written out as warnings since they are non-fatal.
-    '''
-    global street_error_map
-    street_error_map[street_address] = {'reason': reason, 'notes': notes}
-    had_street_warning(street_address, reason, notes=notes)
-    raise ContinueIteration
-
-
-# START WORK
-if WRITE_OUT:
-    print('Deleting existing address-streets...')
-    address_street_table.delete()
-
-print('Reading street segments...')
-seg_fields = [
-    'seg_id',
-    'street_full',
-    'left_from',
-    'left_to',
-    'right_from',
-    'right_to'
-]
-seg_map = {}
-seg_rows = street_segment_table.read(fields=seg_fields, sort=['STREET_FULL', 'LEFT_FROM', 'RIGHT_FROM'])
-for seg_row in seg_rows:
-    street_full = seg_row['street_full']
-    street_full_segs = seg_map.setdefault(street_full, [])
-    street_full_segs.append(seg_row)
-
-address_streets = []
-base_address_map = {}  # base_address => {seg_id, seg_side, had_alias}
-
-print('Making address-streets...')
-addresses = addresses + new_addresses
-for address in addresses:
-    try:
-        street_address = address.street_address
-        base_address = address.base_address
-        # If the base address already had an error, raise it again
-        if base_address in street_error_map:
-            error = street_error_map[base_address]
-            reason = error['reason']
-            notes = error['notes']
-            had_street_error(street_address, reason, notes=notes)
-
-        match = None
-        had_alias = None
-
-        # If we've already seen the base address before, used cached values
-        if base_address in base_address_map:
-            # Make match, also
-            match = base_address_map[base_address]
-            seg_id = match['seg_id']
-            seg_side = match['seg_side']
-            had_alias = match['had_alias']
+                # Loop through segs for that street full
+                for seg in matching_segs:
+                    left_from = seg['left_from']
+                    left_to = seg['left_to']
+                    right_from = seg['right_from']
+                    right_to = seg['right_to']
 
-            # If there were warnings, raise them again
-            for warning in street_warning_map.get(base_address, []):
-                had_street_warning(
-                    street_address,
-                    warning['reason'],
-                    notes=warning.get('notes')
-                )
+                    left_parity = parity_for_range(left_from, left_to)
+                    right_parity = parity_for_range(right_from, right_to)
 
-        # Otherwise this is a new address
-        else:
-            # There are some types of warnings we only want to write out if a
-            # more serious error isn't raised first. Keep these here, append
-            # during the seg loop, then call had_street_warning for each one
-            # if we get to that point.
-            deferred_warnings = []
+                    # Match to side of street based on parity
+                    check_from = None
+                    check_to = None
 
-            # Check for valid street
-            address_low = address.address_low
-            address_high = address.address_high
-            street_full = address.street_full
-            address_parity = address.parity
-            if not street_full in seg_map:
-                had_street_error(street_address, 'Not a valid street')
-
-            matching_segs = seg_map[street_full]
-            matching_seg = None
-            matching_side = None
-
-            # Loop through segs for that street full
-            for seg in matching_segs:
-                left_from = seg['left_from']
-                left_to = seg['left_to']
-                right_from = seg['right_from']
-                right_to = seg['right_to']
-
-                left_parity = parity_for_range(left_from, left_to)
-                right_parity = parity_for_range(right_from, right_to)
-
-                # Match to side of street based on parity
-                check_from = None
-                check_to = None
-
-                if left_parity in [address_parity, 'B'] and right_parity not in [address_parity, 'B']:
-                    check_from = left_from
-                    check_to = left_to
-                    matching_side = 'L'
-                elif right_parity in [address_parity, 'B'] and left_parity not in (address_parity, 'B'):
-                    check_from = right_from
-                    check_to = right_to
-                    matching_side = 'R'
-
-                # If seg has same parity on both sides, match to closest side:
-                elif left_parity in [address_parity, 'B'] and right_parity in [address_parity, 'B']:
-                    if abs(address_low - left_from) < abs(address_low - right_from):
+                    if left_parity in [address_parity, 'B'] and right_parity not in [address_parity, 'B']:
                         check_from = left_from
                         check_to = left_to
                         matching_side = 'L'
-                    else:
+                    elif right_parity in [address_parity, 'B'] and left_parity not in (address_parity, 'B'):
                         check_from = right_from
                         check_to = right_to
                         matching_side = 'R'
-                else:
-                    continue
 
-                # SINGLE ADDRESS
-                if address_high is None:
-                    if check_from <= address_low <= check_to:
-                        matching_seg = seg
-                        break
-
-                # RANGE ADDRESS
-                else:
-                    # If the low num is in range
-                    if check_from <= address_low <= check_to:
-                        # Check for a previously matched seg
-                        if matching_seg:
-                            seg_ids = sorted([x['seg_id'] for x in \
-                                              [matching_seg, seg]])
-                            notes = ', '.join([str(x) for x in seg_ids])
-                            had_street_error(street_address, 'Range address matches multiple street segments',
-                                             notes=notes)
-                        # Match it
-                        matching_seg = seg
-                        # If the high num is out of range
-                        if check_to < address_high:
-                            # Warn
+                    # If seg has same parity on both sides, match to closest side:
+                    elif left_parity in [address_parity, 'B'] and right_parity in [address_parity, 'B']:
+                        if abs(address_low - left_from) < abs(address_low - right_from):
+                            check_from = left_from
+                            check_to = left_to
+                            matching_side = 'L'
+                        else:
+                            check_from = right_from
+                            check_to = right_to
+                            matching_side = 'R'
+                    else:
+                        continue
+
+                    # SINGLE ADDRESS
+                    if address_high is None:
+                        if check_from <= address_low <= check_to:
+                            matching_seg = seg
+                            break
+
+                    # RANGE ADDRESS
+                    else:
+                        # If the low num is in range
+                        if check_from <= address_low <= check_to:
+                            # Check for a previously matched seg
+                            if matching_seg:
+                                seg_ids = sorted([x['seg_id'] for x in \
+                                                  [matching_seg, seg]])
+                                notes = ', '.join([str(x) for x in seg_ids])
+                                had_street_error(street_address, 'Range address matches multiple street segments',
+                                                 notes=notes)
+                            # Match it
+                            matching_seg = seg
+                            # If the high num is out of range
+                            if check_to < address_high:
+                                # Warn
+                                deferred_warnings.append({
+                                    'street_address': street_address,
+                                    'reason': 'High address out of range',
+                                    'notes': 'Seg {}: {} to {}'.format(
+                                        matching_seg['seg_id'],
+                                        check_from,
+                                        check_to
+                                    )
+                                })
+
+                        # If only the high address is in range (unlikely)
+                        elif check_from <= address_high <= check_to:
+                            # Match and warn
+                            matching_seg = seg
                             deferred_warnings.append({
                                 'street_address': street_address,
-                                'reason': 'High address out of range',
+                                'reason': 'Low address out of range',
                                 'notes': 'Seg {}: {} to {}'.format(
                                     matching_seg['seg_id'],
                                     check_from,
@@ -838,440 +802,410 @@ def had_street_error(street_address, reason, notes=None):
                                 )
                             })
 
-                    # If only the high address is in range (unlikely)
-                    elif check_from <= address_high <= check_to:
-                        # Match and warn
-                        matching_seg = seg
-                        deferred_warnings.append({
-                            'street_address': street_address,
-                            'reason': 'Low address out of range',
-                            'notes': 'Seg {}: {} to {}'.format(
-                                matching_seg['seg_id'],
-                                check_from,
-                                check_to
-                            )
-                        })
-
-            # Store the match
-            if matching_seg:
-                match = {
-                    'seg_id': matching_seg['seg_id'],
-                    'seg_side': matching_side,
-                    'had_alias': had_alias,
-                }
-                base_address_map[base_address] = match
+                # Store the match
+                if matching_seg:
+                    match = {
+                        'seg_id': matching_seg['seg_id'],
+                        'seg_side': matching_side,
+                        'had_alias': had_alias,
+                    }
+                    base_address_map[base_address] = match
+
+                # Handle deferred warnings
+                for warning in deferred_warnings:
+                    had_street_warning(
+                        warning['street_address'],
+                        warning['reason'],
+                        notes=warning.get('notes')
+                    )
 
-            # Handle deferred warnings
-            for warning in deferred_warnings:
-                had_street_warning(
-                    warning['street_address'],
-                    warning['reason'],
-                    notes=warning.get('notes')
-                )
+            if match is None:
+                had_street_error(street_address, 'Out of street range')
 
-        if match is None:
-            had_street_error(street_address, 'Out of street range')
+            if had_alias:
+                # TODO: check against aliases; raise warning if alias used
+                pass
+            seg_id = match['seg_id']
+            seg_side = match['seg_side']
+
+            address_street = {
+                'street_address': street_address,
+                'seg_id': seg_id,
+                'seg_side': seg_side,
+            }
+            address_streets.append(address_street)
 
-        if had_alias:
-            # TODO: check against aliases; raise warning if alias used
+        except ContinueIteration:
             pass
-        seg_id = match['seg_id']
-        seg_side = match['seg_side']
 
-        address_street = {
-            'street_address': street_address,
-            'seg_id': seg_id,
-            'seg_side': seg_side,
-        }
-        address_streets.append(address_street)
+    if WRITE_OUT:
+        print('Writing address-streets...')
+        address_street_table.write(address_streets, chunk_size=150000)
+    del address_streets
+
+    # Handle errors
+    for street_address, warnings in street_warning_map.items():
+        for warning_dict in warnings:
+            reason = warning_dict['reason']
+            notes = warning_dict['notes']
+
+            # Get source addresses
+            # TEMP: put this in a try statement until some base address parsing
+            # issues in Passyunk are resolved
+            try:
+                _source_addresses = source_address_map[street_address]
+            except KeyError:
+                continue
 
-    except ContinueIteration:
-        pass
+            for source_address in _source_addresses:
+                # Get sources
+                sources = source_map[source_address]
+                for source in sources:
+                    address_errors.append({
+                        'source_name': source,
+                        'source_address': source_address,
+                        'street_address': street_address,
+                        'level': 'warning',
+                        'reason': reason,
+                        'notes': notes,
+                    })
+
+    ################################################################################
+    # ADDRESS-PARCELS
+    ################################################################################
+
+    print('** ADDRESS-PARCELS **')
+
+    # This maps address variant names to AddressParcel match types
+    ADDRESS_VARIANT_MATCH_TYPE = {
+        'base_address': 'base',
+        'base_address_no_suffix': 'base_no_suffix',
+        'generic_unit': 'generic_unit'
+    }
 
-if WRITE_OUT:
-    print('Writing address-streets...')
-    address_street_table.write(address_streets, chunk_size=150000)
-del address_streets
+    match_counts = {x: 0 for x in ADDRESS_VARIANT_MATCH_TYPE.values()}
+    match_counts['address_in_parcel_range'] = 0
+    match_counts['parcel_in_address_range'] = 0
 
-# Handle errors
-for street_address, warnings in street_warning_map.items():
-    for warning_dict in warnings:
-        reason = warning_dict['reason']
-        notes = warning_dict['notes']
+    address_parcels = []
 
-        # Get source addresses
-        # TEMP: put this in a try statement until some base address parsing
-        # issues in Passyunk are resolved
-        try:
-            _source_addresses = source_address_map[street_address]
-        except KeyError:
-            continue
+    if WRITE_OUT:
+        print('Dropping index on address-parcels...')
+        address_parcel_table.drop_index('street_address')
+        print('Deleting existing address-parcels...')
+        address_parcel_table.delete()
+
+    for parcel_layer in parcel_layers:
+        source_table_name = parcel_layer + '_parcel'
+        source_table = db[source_table_name]
+        print('Reading from {}...'.format(parcel_layer))
+
+        if DEV:
+            where = "street_name = '{}'".format(DEV_STREET_NAME)
+            parcel_rows = source_table.read(fields=['street_address', 'id'], \
+                                            where=where)
+        else:
+            parcel_rows = source_table.read(fields=['street_address', 'id'])
 
-        for source_address in _source_addresses:
-            # Get sources
-            sources = source_map[source_address]
-            for source in sources:
-                address_errors.append({
-                    'source_name': source,
-                    'source_address': source_address,
-                    'street_address': street_address,
-                    'level': 'warning',
-                    'reason': reason,
-                    'notes': notes,
-                })
-
-################################################################################
-# ADDRESS-PARCELS
-################################################################################
-
-print('** ADDRESS-PARCELS **')
-
-# This maps address variant names to AddressParcel match types
-ADDRESS_VARIANT_MATCH_TYPE = {
-    'base_address': 'base',
-    'base_address_no_suffix': 'base_no_suffix',
-    'generic_unit': 'generic_unit'
-}
-
-match_counts = {x: 0 for x in ADDRESS_VARIANT_MATCH_TYPE.values()}
-match_counts['address_in_parcel_range'] = 0
-match_counts['parcel_in_address_range'] = 0
-
-address_parcels = []
-
-if WRITE_OUT:
-    print('Dropping index on address-parcels...')
-    address_parcel_table.drop_index('street_address')
-    print('Deleting existing address-parcels...')
-    address_parcel_table.delete()
-
-for parcel_layer in parcel_layers:
-    source_table_name = parcel_layer + '_parcel'
-    source_table = db[source_table_name]
-    print('Reading from {}...'.format(parcel_layer))
-
-    if DEV:
-        where = "street_name = '{}'".format(DEV_STREET_NAME)
-        parcel_rows = source_table.read(fields=['street_address', 'id'], \
-                                        where=where)
-    else:
-        parcel_rows = source_table.read(fields=['street_address', 'id'])
-
-    print('Building indexes...')
-    # Index: street_address => [row_ids]
-    parcel_map = {}
-    # Index: street full => hundred block =>
-    #   {row_id/address low/high/suffix dicts}
-    block_map = {}
-
-    for parcel_row in parcel_rows:
-        street_address = parcel_row['street_address']
-        row_id = parcel_row['id']
-
-        # Get address components
-        try:
-            parcel_address = Address(street_address)
-        except ValueError:
-            # TODO: this should never happen
-            print('Could not parse parcel address: {}'.format(street_address))
-            continue
+        print('Building indexes...')
+        # Index: street_address => [row_ids]
+        parcel_map = {}
+        # Index: street full => hundred block =>
+        #   {row_id/address low/high/suffix dicts}
+        block_map = {}
 
-        street_full = parcel_address.street_full
-        address_low = parcel_address.address_low
-        address_low_suffix = parcel_address.address_low_suffix
-        address_high = parcel_address.address_high
+        for parcel_row in parcel_rows:
+            street_address = parcel_row['street_address']
+            row_id = parcel_row['id']
 
-        parcel_map.setdefault(street_address, [])
-        parcel_map[street_address].append(row_id)
+            # Get address components
+            try:
+                parcel_address = Address(street_address)
+            except ValueError:
+                # TODO: this should never happen
+                # print('Could not parse parcel address: {}'.format(street_address))
+                continue
 
-        block_map.setdefault(street_full, {})
-        hundred_block = parcel_address.hundred_block
-        block_map[street_full].setdefault(hundred_block, [])
+            street_full = parcel_address.street_full
+            address_low = parcel_address.address_low
+            address_low_suffix = parcel_address.address_low_suffix
+            address_high = parcel_address.address_high
 
-        # Add a few address components used below
-        parcel_row['address_low'] = parcel_address.address_low
-        parcel_row['address_high'] = parcel_address.address_high
-        parcel_row['address_low_suffix'] = parcel_address.address_low_suffix
-        parcel_row['unit_full'] = parcel_address.unit_full
-        block_map[street_full][hundred_block].append(parcel_row)
+            parcel_map.setdefault(street_address, [])
+            parcel_map[street_address].append(row_id)
 
-    print('Relating addresses to parcels...')
-    for address in addresses:
-        address_low = address.address_low
-        address_low_suffix = address.address_low_suffix
-        address_unit_full = address.unit_full
-        street_address = address.street_address
-        matches = []  # dicts of {row_id, match_type}
+            block_map.setdefault(street_full, {})
+            hundred_block = parcel_address.hundred_block
+            block_map[street_full].setdefault(hundred_block, [])
 
-        # EXACT
-        if street_address in parcel_map:
-            for row_id in parcel_map[street_address]:
-                matches.append({
-                    'parcel_row_id': row_id,
-                    'match_type': 'exact',
-                })
-        else:
-            try:
-                # BASE, BASE NO SUFFIX, GENERIC UNIT
-                for variant_type in ['base_address', 'base_address_no_suffix', \
-                                     'generic_unit']:
-                    variant = getattr(address, variant_type)
-
-                    if variant in parcel_map:
-                        row_ids = parcel_map[variant]
-                        for row_id in row_ids:
-                            match_type = ADDRESS_VARIANT_MATCH_TYPE[variant_type]
-                            matches.append({
-                                'parcel_row_id': row_id,
-                                'match_type': match_type,
-                            })
-                            match_counts[match_type] += 1
-                            raise ContinueIteration
+            # Add a few address components used below
+            parcel_row['address_low'] = parcel_address.address_low
+            parcel_row['address_high'] = parcel_address.address_high
+            parcel_row['address_low_suffix'] = parcel_address.address_low_suffix
+            parcel_row['unit_full'] = parcel_address.unit_full
+            block_map[street_full][hundred_block].append(parcel_row)
 
-            except ContinueIteration:
-                pass
+        print('Relating addresses to parcels...')
+        for address in addresses:
+            address_low = address.address_low
+            address_low_suffix = address.address_low_suffix
+            address_unit_full = address.unit_full
+            street_address = address.street_address
+            matches = []  # dicts of {row_id, match_type}
+
+            # EXACT
+            if street_address in parcel_map:
+                for row_id in parcel_map[street_address]:
+                    matches.append({
+                        'parcel_row_id': row_id,
+                        'match_type': 'exact',
+                    })
+            else:
+                try:
+                    # BASE, BASE NO SUFFIX, GENERIC UNIT
+                    for variant_type in ['base_address', 'base_address_no_suffix', \
+                                         'generic_unit']:
+                        variant = getattr(address, variant_type)
+
+                        if variant in parcel_map:
+                            row_ids = parcel_map[variant]
+                            for row_id in row_ids:
+                                match_type = ADDRESS_VARIANT_MATCH_TYPE[variant_type]
+                                matches.append({
+                                    'parcel_row_id': row_id,
+                                    'match_type': match_type,
+                                })
+                                match_counts[match_type] += 1
+                                raise ContinueIteration
+
+                except ContinueIteration:
+                    pass
+
+                # RANGES
+                if len(matches) == 0:
+                    address_high = address.address_high
+                    street_full = address.street_full
+                    hundred_block = address.hundred_block
+                    address_parity = address.parity
+
+                    # ADDRESS IN PARCEL RANGE
+                    if street_full in block_map:
+                        street_map = block_map[street_full]
+                        parcels_on_block = street_map.get(hundred_block, [])
+
+                        for parcel_row in parcels_on_block:
+                            # Check low/high, suffix
+                            parcel_low = parcel_row['address_low']
+                            parcel_high = parcel_row['address_high']
+
+                            # Check unit
+                            parcel_unit_full = parcel_row['unit_full']
+                            if address_unit_full != parcel_unit_full:
+                                continue
 
-            # RANGES
-            if len(matches) == 0:
-                address_high = address.address_high
-                street_full = address.street_full
-                hundred_block = address.hundred_block
-                address_parity = address.parity
+                            # Check suffix
+                            parcel_low_suffix = parcel_row['address_low_suffix']
+                            if address_low_suffix != parcel_low_suffix:
+                                continue
+
+                            # SINGLE ADDRESS => RANGE PARCEL
+                            if address_high is None:
+                                # Skip single addresses
+                                if parcel_high is None:
+                                    continue
+
+                                check_low = parcel_low
+                                check_mid = address_low
+                                check_high = parcel_high
+
+                                parcel_parity = \
+                                    parity_for_range(parcel_low, parcel_high)
+
+                            # RANGE ADDRESS => SINGLE PARCEL(S)
+                            else:
+                                check_low = address_low
+                                check_mid = parcel_low
+                                check_high = address_high
 
-                # ADDRESS IN PARCEL RANGE
-                if street_full in block_map:
-                    street_map = block_map[street_full]
-                    parcels_on_block = street_map.get(hundred_block, [])
-
-                    for parcel_row in parcels_on_block:
-                        # Check low/high, suffix
-                        parcel_low = parcel_row['address_low']
-                        parcel_high = parcel_row['address_high']
-
-                        # Check unit
-                        parcel_unit_full = parcel_row['unit_full']
-                        if address_unit_full != parcel_unit_full:
-                            continue
-
-                        # Check suffix
-                        parcel_low_suffix = parcel_row['address_low_suffix']
-                        if address_low_suffix != parcel_low_suffix:
-                            continue
-
-                        # SINGLE ADDRESS => RANGE PARCEL
-                        if address_high is None:
-                            # Skip single addresses
-                            if parcel_high is None:
+                                parcel_parity = parity_for_num(parcel_low)
+
+                            if parcel_parity != address_parity:
                                 continue
 
-                            check_low = parcel_low
-                            check_mid = address_low
-                            check_high = parcel_high
+                            # If it's in range
+                            if check_low <= check_mid <= check_high:
+                                row_id = parcel_row['id']
+
+                                if address_high is None:
+                                    match_type = 'address_in_parcel_range'
+                                else:
+                                    match_type = 'parcel_in_address_range'
+                                match_counts[match_type] += 1
+                                matches.append({
+                                    'parcel_row_id': row_id,
+                                    'match_type': match_type,
+                                })
+
+            # Handle matches
+            for match in matches:
+                address_parcel = {
+                    'street_address': street_address,
+                    'parcel_source': parcel_layer,
+                }
+                address_parcel.update(match)
+                address_parcels.append(address_parcel)
 
-                            parcel_parity = \
-                                parity_for_range(parcel_low, parcel_high)
+                # Rework this to support multiple matches
 
-                        # RANGE ADDRESS => SINGLE PARCEL(S)
-                        else:
-                            check_low = address_low
-                            check_mid = parcel_low
-                            check_high = address_high
+    if WRITE_OUT:
+        print('Writing address-parcels...')
+        address_parcel_table.write(address_parcels, chunk_size=150000)
+        print('Indexing address-parcels...')
+        address_parcel_table.create_index('street_address')
 
-                            parcel_parity = parity_for_num(parcel_low)
+    for variant_type, count in match_counts.items():
+        print('{} matched on {}'.format(count, variant_type))
 
-                        if parcel_parity != address_parity:
-                            continue
+    del address_parcels
 
-                        # If it's in range
-                        if check_low <= check_mid <= check_high:
-                            row_id = parcel_row['id']
+    ################################################################################
+    # ADDRESS-PROPERTIES
+    ################################################################################
 
-                            if address_high is None:
-                                match_type = 'address_in_parcel_range'
-                            else:
-                                match_type = 'parcel_in_address_range'
-                            match_counts[match_type] += 1
-                            matches.append({
-                                'parcel_row_id': row_id,
-                                'match_type': match_type,
-                            })
+    print('** ADDRESS-PROPERTIES **')
 
-        # Handle matches
-        for match in matches:
-            address_parcel = {
-                'street_address': street_address,
-                'parcel_source': parcel_layer,
-            }
-            address_parcel.update(match)
-            address_parcels.append(address_parcel)
-
-            # Rework this to support multiple matches
-            # if parcel_id is not None:
-            #   address_parcel = {
-            #       'street_address':   street_address,
-            #       'parcel_source':    parcel_layer,
-            #       'parcel_id':        parcel_id,
-            #       'match_type':       match_type,
-            #   }
-            #   address_parcels.append(address_parcel)
-
-if WRITE_OUT:
-    print('Writing address-parcels...')
-    address_parcel_table.write(address_parcels, chunk_size=150000)
-    print('Indexing address-parcels...')
-    address_parcel_table.create_index('street_address')
-
-for variant_type, count in match_counts.items():
-    print('{} matched on {}'.format(count, variant_type))
-
-del address_parcels
-
-################################################################################
-# ADDRESS-PROPERTIES
-################################################################################
-
-print('** ADDRESS-PROPERTIES **')
-
-if WRITE_OUT:
-    print('Dropping index on address-properties...')
-    address_property_table.drop_index('street_address')
-    print('Deleting existing address-properties...')
-    address_property_table.delete()
-
-# Read properties in
-print('Reading properties from AIS...')
-# TODO: clean this up, move config to config
-prop_rows = db['opa_property'].read(fields=['street_address', 'account_num', 'address_low', 'address_high', 'unit_num'])
-prop_map = {x['street_address']: x for x in prop_rows}
-
-print('Indexing range properties...')
-range_rows = [x for x in prop_rows if x['address_high'] is not None]
-range_map = {}  # street_full => [range props]
-for range_row in range_rows:
-    try:
-        street_address = range_row['street_address']
-        street_full = Address(street_address).street_full
-        prop_list = range_map.setdefault(street_full, [])
-        prop_list.append(range_row)
-    except ValueError:
-        print('Unrecognized format for range address: {}'.format(street_address))
-        continue
-
-address_props = []
-
-print('Relating addresses to properties...')
-for i, address in enumerate(addresses):
-    if i % 100000 == 0:
-        print(i)
-
-    street_address = address.street_address
-    unit_num = address.unit_num
-    prop = None
-    match_type = None
-
-    # EXACT
-    if street_address in prop_map:
-        prop = prop_map[street_address]
-        match_type = 'exact'
-
-    # BASE
-    elif unit_num and address.base_address in prop_map:
-        prop = prop_map[address.base_address]
-        match_type = 'base'
-
-    # BASE NO SUFFIX
-    elif address.base_address_no_suffix in prop_map:
-        prop = prop_map[address.base_address_no_suffix]
-        match_type = 'base_no_suffix'
-
-    # GENERIC UNIT
-    elif unit_num and address.generic_unit in prop_map:
-        prop = prop_map[address.generic_unit]
-        match_type = 'generic_unit'
-
-    # RANGE
-    elif address.address_high is None and address.street_full in range_map:
-        range_props_on_street = range_map[address.street_full]
-        address_parity = address.parity
-
-        for range_prop in range_props_on_street:
-            range_prop_address_low = range_prop['address_low']
-            range_prop_address_high = range_prop['address_high']
-            range_prop_parity = parity_for_range(range_prop_address_low, \
-                                                 range_prop_address_high)
-
-            # Check parity
-            if address_parity != range_prop_parity:
-                continue
+    if WRITE_OUT:
+        print('Dropping index on address-properties...')
+        address_property_table.drop_index('street_address')
+        print('Deleting existing address-properties...')
+        address_property_table.delete()
+
+    # Read properties in
+    print('Reading properties from AIS...')
+    # TODO: clean this up, move config to config
+    prop_rows = db['opa_property'].read(fields=['street_address', 'account_num', 'address_low', 'address_high', 'unit_num'])
+    prop_map = {x['street_address']: x for x in prop_rows}
+
+    print('Indexing range properties...')
+    range_rows = [x for x in prop_rows if x['address_high'] is not None]
+    range_map = {}  # street_full => [range props]
+    for range_row in range_rows:
+        try:
+            street_address = range_row['street_address']
+            street_full = Address(street_address).street_full
+            prop_list = range_map.setdefault(street_full, [])
+            prop_list.append(range_row)
+        except ValueError:
+            print('Unrecognized format for range address: {}'.format(street_address))
+            continue
 
-            if range_prop['address_low'] <= address.address_low <= range_prop['address_high']:
-                # If there's a unit num we have to make sure that matches too
-                if (address.unit_num or range_prop['unit_num']) and \
-                                address.unit_num != range_prop['unit_num']:
+    address_props = []
+
+    print('Relating addresses to properties...')
+    for i, address in enumerate(addresses):
+        if i % 100000 == 0:
+            print(i)
+
+        street_address = address.street_address
+        unit_num = address.unit_num
+        prop = None
+        match_type = None
+
+        # EXACT
+        if street_address in prop_map:
+            prop = prop_map[street_address]
+            match_type = 'exact'
+
+        # BASE
+        elif unit_num and address.base_address in prop_map:
+            prop = prop_map[address.base_address]
+            match_type = 'base'
+
+        # BASE NO SUFFIX
+        elif address.base_address_no_suffix in prop_map:
+            prop = prop_map[address.base_address_no_suffix]
+            match_type = 'base_no_suffix'
+
+        # GENERIC UNIT
+        elif unit_num and address.generic_unit in prop_map:
+            prop = prop_map[address.generic_unit]
+            match_type = 'generic_unit'
+
+        # RANGE
+        elif address.address_high is None and address.street_full in range_map:
+            range_props_on_street = range_map[address.street_full]
+            address_parity = address.parity
+
+            for range_prop in range_props_on_street:
+                range_prop_address_low = range_prop['address_low']
+                range_prop_address_high = range_prop['address_high']
+                range_prop_parity = parity_for_range(range_prop_address_low, \
+                                                     range_prop_address_high)
+
+                # Check parity
+                if address_parity != range_prop_parity:
                     continue
 
-                prop = range_prop
-                match_type = 'range'
-                break
+                if range_prop['address_low'] <= address.address_low <= range_prop['address_high']:
+                    # If there's a unit num we have to make sure that matches too
+                    if (address.unit_num or range_prop['unit_num']) and \
+                                    address.unit_num != range_prop['unit_num']:
+                        continue
 
-    if prop:
-        address_prop = {
-            'street_address': street_address,
-            'opa_account_num': prop['account_num'],
-            'match_type': match_type,
-        }
-        address_props.append(address_prop)
+                    prop = range_prop
+                    match_type = 'range'
+                    break
 
-if WRITE_OUT:
-    address_property_table.write(address_props)
-    print('Indexing address-properties...')
-    address_property_table.create_index('street_address')
-del address_props
+        if prop:
+            address_prop = {
+                'street_address': street_address,
+                'opa_account_num': prop['account_num'],
+                'match_type': match_type,
+            }
+            address_props.append(address_prop)
 
-################################################################################
-# TRUE RANGE
-################################################################################
+    if WRITE_OUT:
+        address_property_table.write(address_props)
+        print('Indexing address-properties...')
+        address_property_table.create_index('street_address')
+    del address_props
 
-print('** TRUE RANGE **')
+    ################################################################################
+    # TRUE RANGE
+    ################################################################################
 
-if WRITE_OUT:
-    print('Creating true range view...')
-    db.drop_view(true_range_view_name)
-    db.create_view(true_range_view_name, true_range_select_stmt)
+    print('** TRUE RANGE **')
 
-################################################################################
-# ERRORS
-################################################################################
+    if WRITE_OUT:
+        print('Creating true range view...')
+        db.drop_view(true_range_view_name)
+        db.create_view(true_range_view_name, true_range_select_stmt)
 
-print('** ERRORS **')
+    ################################################################################
+    # ERRORS
+    ################################################################################
 
-if WRITE_OUT:
-    print('Writing errors...')
-    address_error_table.write(address_errors, chunk_size=150000)
-del address_errors
+    print('** ERRORS **')
 
-# print('{} errors'.format(error_count))
-# print('{} warnings'.format(warning_count))
+    if WRITE_OUT:
+        print('Writing errors...')
+        address_error_table.write(address_errors, chunk_size=150000)
+    del address_errors
 
-################################################################################
-# FINISH
-################################################################################
+    ################################################################################
+    # FINISH
+    ################################################################################
 
-print('** FINISHING **')
+    print('** FINISHING **')
 
-if WRITE_OUT:
-    print('Creating indexes...')
-    #     index_tables = (
-    #         address_table,
-    #         address_tag_table,
-    #         source_address_table,
-    #     )
-    for table in (address_table, address_tag_table, source_address_table):
-        table.create_index('street_address')
-    address_link_table.create_index('address_1')
-    address_link_table.create_index('address_2')
-    address_street_table.create_index('street_address')
+    if WRITE_OUT:
+        print('Creating indexes...')
+        for table in (address_table, address_tag_table, source_address_table):
+            table.create_index('street_address')
+        address_link_table.create_index('address_1')
+        address_link_table.create_index('address_2')
+        address_street_table.create_index('street_address')
 
-db.close()
+    db.close()
 
-print('Finished in {} seconds'.format(datetime.now() - start))
+    print('Finished in {} seconds'.format(datetime.now() - start))
diff --git a/ais/engine/scripts/load_curbs.py b/ais/engine/scripts/load_curbs.py
index c5716bb8..61246931 100644
--- a/ais/engine/scripts/load_curbs.py
+++ b/ais/engine/scripts/load_curbs.py
@@ -1,71 +1,65 @@
-import sys
+from datetime import datetime
 import datum
 from ais import app
-# DEV
-import traceback
-from pprint import pprint
 
-
-"""SET UP"""
-
-config = app.config
-source_def = config['BASE_DATA_SOURCES']['curbs']
-source_db = datum.connect(config['DATABASES'][source_def['db']])
-source_table = source_db[source_def['table']]
-field_map = source_def['field_map']
-db = datum.connect(config['DATABASES']['engine'])
-curb_table = db['curb']
-parcel_curb_table = db['parcel_curb']
-
-
-"""MAIN"""
-
-# print('Dropping parcel-curb view...')
-# db.drop_mview('parcel_curb')
-
-print('Dropping indexes...')
-curb_table.drop_index('curb_id')
-parcel_curb_table.drop_index('curb_id')
-parcel_curb_table.drop_index('parcel_source', 'parcel_row_id')
-
-print('Deleting existing curbs...')
-curb_table.delete()
-
-print('Deleting existing parcel curbs...')
-parcel_curb_table.delete()
-
-print('Reading curbs from source...')
-source_rows = source_table.read()
-curbs = []
-for source_row in source_rows:
-	curb = {x: source_row[field_map[x]] for x in field_map}
-	# curb['geom'] = source_row[wkt_field]
-	curbs.append(curb)
-
-print('Writing curbs...')
-curb_table.write(curbs)
-
-print('Making parcel-curbs...')
-for agency in config['BASE_DATA_SOURCES']['parcels']:
-    print('  - ' + agency)
-    # table_name = parcel_source_def['table']
-    stmt = '''
-        insert into parcel_curb (parcel_source, parcel_row_id, curb_id) (
-          select distinct on (p.id)
-            '{agency}',
-            p.id,
-            c.curb_id
-          from {agency}_parcel p
-          join curb c
-          on ST_Intersects(p.geom, c.geom)
-          order by p.id, st_area(st_intersection(p.geom, c.geom)) desc
-        )
-    '''.format(agency=agency)
-    db.execute(stmt)
-    db.save()
-
-print('Creating indexes...')
-
-
-
-db.close()
+def main():
+    print('Starting...')
+    start = datetime.now()
+
+    """SET UP"""
+
+    config = app.config
+    source_def = config['BASE_DATA_SOURCES']['curbs']
+    source_db = datum.connect(config['DATABASES'][source_def['db']])
+    source_table = source_db[source_def['table']]
+    field_map = source_def['field_map']
+    db = datum.connect(config['DATABASES']['engine'])
+    curb_table = db['curb']
+    parcel_curb_table = db['parcel_curb']
+
+    """MAIN"""
+
+    print('Dropping indexes...')
+    curb_table.drop_index('curb_id')
+    parcel_curb_table.drop_index('curb_id')
+    parcel_curb_table.drop_index('parcel_source', 'parcel_row_id')
+
+    print('Deleting existing curbs...')
+    curb_table.delete()
+
+    print('Deleting existing parcel curbs...')
+    parcel_curb_table.delete()
+
+    print('Reading curbs from source...')
+    source_rows = source_table.read()
+    curbs = []
+    for source_row in source_rows:
+        curb = {x: source_row[field_map[x]] for x in field_map}
+        curbs.append(curb)
+
+    print('Writing curbs...')
+    curb_table.write(curbs)
+
+    print('Making parcel-curbs...')
+    for agency in config['BASE_DATA_SOURCES']['parcels']:
+        print('  - ' + agency)
+        stmt = '''
+            insert into parcel_curb (parcel_source, parcel_row_id, curb_id) (
+              select distinct on (p.id)
+                '{agency}',
+                p.id,
+                c.curb_id
+              from {agency}_parcel p
+              join curb c
+              on ST_Intersects(p.geom, c.geom)
+              order by p.id, st_area(st_intersection(p.geom, c.geom)) desc
+         )
+        '''.format(agency=agency)
+        db.execute(stmt)
+        db.save()
+
+    print('Creating indexes...')
+
+    db.close()
+    print('Finished in {} seconds'.format(datetime.now() - start))
+    
diff --git a/ais/engine/scripts/load_dor_condos.py b/ais/engine/scripts/load_dor_condos.py
index e2e64ce9..36ba71c3 100644
--- a/ais/engine/scripts/load_dor_condos.py
+++ b/ais/engine/scripts/load_dor_condos.py
@@ -4,84 +4,84 @@
 from datetime import datetime
 from ais import app
 
+def main():
+    start = datetime.now()
+    print('Starting...')
 
-start = datetime.now()
-print('Starting...')
+    """SET UP"""
+    DEV = False
+    config = app.config
+    engine_dsn = config['DATABASES']['engine']
+    db_user = engine_dsn[engine_dsn.index("//") + 2:engine_dsn.index(":", engine_dsn.index("//"))]
+    db_pw = engine_dsn[engine_dsn.index(":",engine_dsn.index(db_user)) + 1:engine_dsn.index("@")]
+    db_name = engine_dsn[engine_dsn.index("/", engine_dsn.index("@")) + 1:]
+    pg_db = psycopg2.connect('dbname={db_name} user={db_user} password={db_pw} host=localhost'.format(db_name=db_name, db_user=db_user, db_pw=db_pw))
+    source_def = config['BASE_DATA_SOURCES']['condos']['dor']
+    source_db_name = source_def['db']
+    source_db_url = config['DATABASES'][source_db_name]
+    dsn = source_db_url.split('//')[1].replace(':','/')
+    dbo = cx_Oracle.connect(dsn)
+    source_table_name = source_def['table']
+    source_field_map = source_def['field_map']
+    source_field_map_upper = {}
 
-"""SET UP"""
-DEV = False
-config = app.config
-engine_dsn = config['DATABASES']['engine']
-db_user = engine_dsn[engine_dsn.index("//") + 2:engine_dsn.index(":", engine_dsn.index("//"))]
-db_pw = engine_dsn[engine_dsn.index(":",engine_dsn.index(db_user)) + 1:engine_dsn.index("@")]
-db_name = engine_dsn[engine_dsn.index("/", engine_dsn.index("@")) + 1:]
-pg_db = psycopg2.connect('dbname={db_name} user={db_user} password={db_pw} host=localhost'.format(db_name=db_name, db_user=db_user, db_pw=db_pw))
-source_def = config['BASE_DATA_SOURCES']['condos']['dor']
-source_db_name = source_def['db']
-source_db_url = config['DATABASES'][source_db_name]
-dsn = source_db_url.split('//')[1].replace(':','/')
-dbo = cx_Oracle.connect(dsn)
-source_table_name = source_def['table']
-source_field_map = source_def['field_map']
-source_field_map_upper = {}
+    for k,v in source_field_map.items():
+        source_field_map_upper[k] = v.upper()
 
-for k,v in source_field_map.items():
-    source_field_map_upper[k] = v.upper()
+    # Read DOR CONDO rows from source
+    print("Reading condos...")
+    # TODO: get fieldnames from source_field_map
+    dor_condo_read_stmt = '''
+        select condounit, objectid, mapref from {dor_condo_table}
+        where status in (1,3)
+    '''.format(dor_condo_table = source_table_name)
+    source_dor_condo_rows = etl.fromdb(dbo, dor_condo_read_stmt).fieldmap(source_field_map_upper)
+    if DEV:
+        print(etl.look(source_dor_condo_rows))
 
-# Read DOR CONDO rows from source
-print("Reading condos...")
-# TODO: get fieldnames from source_field_map
-dor_condo_read_stmt = '''
-    select condounit, objectid, mapref from {dor_condo_table}
-    where status in (1,3)
-'''.format(dor_condo_table = source_table_name)
-source_dor_condo_rows = etl.fromdb(dbo, dor_condo_read_stmt).fieldmap(source_field_map_upper)
-if DEV:
-    print(etl.look(source_dor_condo_rows))
+    # Read DOR Parcel rows from engine db
+    print("Reading parcels...")
+    dor_parcel_read_stmt = '''
+        select parcel_id, street_address, address_low, address_low_suffix, address_low_frac, address_high, street_predir, 
+        street_name, street_suffix, street_postdir, street_full from {dor_parcel_table}
+        '''.format(dor_parcel_table='dor_parcel')
+    engine_dor_parcel_rows = etl.fromdb(pg_db, dor_parcel_read_stmt)
+    if DEV:
+        print(etl.look(engine_dor_parcel_rows))
 
-# Read DOR Parcel rows from engine db
-print("Reading parcels...")
-dor_parcel_read_stmt = '''
-    select parcel_id, street_address, address_low, address_low_suffix, address_low_frac, address_high, street_predir, 
-    street_name, street_suffix, street_postdir, street_full from {dor_parcel_table}
-    '''.format(dor_parcel_table='dor_parcel')
-engine_dor_parcel_rows = etl.fromdb(pg_db, dor_parcel_read_stmt)
-if DEV:
-    print(etl.look(engine_dor_parcel_rows))
+    # Get duplicate parcel_ids:
+    non_unique_parcel_id_rows = engine_dor_parcel_rows.duplicates(key='parcel_id')
+    unique_parcel_id_rows = etl.complement(engine_dor_parcel_rows, non_unique_parcel_id_rows)
 
-# Get duplicate parcel_ids:
-non_unique_parcel_id_rows = engine_dor_parcel_rows.duplicates(key='parcel_id')
-unique_parcel_id_rows = etl.complement(engine_dor_parcel_rows, non_unique_parcel_id_rows)
+    # Get address comps for condos by joining to dor_parcel with unique parcel_id on parcel_id:
+    print("Relating condos to parcels...")
+    joined = etl.join(source_dor_condo_rows, unique_parcel_id_rows, key='parcel_id') \
+        .convert('street_address', lambda a, row: row.street_address + ' # ' + row.unit_num, pass_row=True)
+    print("joined rowcount: ", etl.nrows(joined))
+    if DEV:
+        print(etl.look(joined))
 
-# Get address comps for condos by joining to dor_parcel with unique parcel_id on parcel_id:
-print("Relating condos to parcels...")
-joined = etl.join(source_dor_condo_rows, unique_parcel_id_rows, key='parcel_id') \
-    .convert('street_address', lambda a, row: row.street_address + ' # ' + row.unit_num, pass_row=True)
-print("joined rowcount: ", etl.nrows(joined))
-if DEV:
-    print(etl.look(joined))
+    # Calculate errors
+    print("Calculating errors...")
+    unjoined = etl.antijoin(source_dor_condo_rows, joined, key='source_object_id')
+    print("unjoined rowcount: ", etl.nrows(unjoined))
+    dor_condos_unjoined_unmatched = etl.antijoin(unjoined, non_unique_parcel_id_rows, key='parcel_id').addfield('reason', 'non-active/remainder mapreg')
+    print("non-active/remainder mapreg error rowcount: ", etl.nrows(dor_condos_unjoined_unmatched))
+    if DEV:
+        print(etl.look(dor_condos_unjoined_unmatched))
+    dor_condos_unjoined_duplicates = etl.antijoin(unjoined, dor_condos_unjoined_unmatched, key='source_object_id').addfield('reason', 'non-unique active/remainder mapreg')
+    print("non-unique active/remainder mapreg error rowcount: ", etl.nrows(dor_condos_unjoined_duplicates))
+    if DEV:
+        print(etl.look(dor_condos_unjoined_duplicates))
+    error_table = etl.cat(dor_condos_unjoined_unmatched, dor_condos_unjoined_duplicates)
+    if DEV:
+        print(etl.look(error_table))
 
-# Calculate errors
-print("Calculating errors...")
-unjoined = etl.antijoin(source_dor_condo_rows, joined, key='source_object_id')
-print("unjoined rowcount: ", etl.nrows(unjoined))
-dor_condos_unjoined_unmatched = etl.antijoin(unjoined, non_unique_parcel_id_rows, key='parcel_id').addfield('reason', 'non-active/remainder mapreg')
-print("non-active/remainder mapreg error rowcount: ", etl.nrows(dor_condos_unjoined_unmatched))
-if DEV:
-    print(etl.look(dor_condos_unjoined_unmatched))
-dor_condos_unjoined_duplicates = etl.antijoin(unjoined, dor_condos_unjoined_unmatched, key='source_object_id').addfield('reason', 'non-unique active/remainder mapreg')
-print("non-unique active/remainder mapreg error rowcount: ", etl.nrows(dor_condos_unjoined_duplicates))
-if DEV:
-    print(etl.look(dor_condos_unjoined_duplicates))
-error_table = etl.cat(dor_condos_unjoined_unmatched, dor_condos_unjoined_duplicates)
-if DEV:
-    print(etl.look(error_table))
+    # Write to engine db
+    if not DEV:
+        print('Writing condos...')
+        joined.todb(pg_db, 'dor_condominium')
+        print('Writing errors...')
+        error_table.todb(pg_db, 'dor_condominium_error')
 
-# Write to engine db
-if not DEV:
-    print('Writing condos...')
-    joined.todb(pg_db, 'dor_condominium')
-    print('Writing errors...')
-    error_table.todb(pg_db, 'dor_condominium_error')
-
-print("Completed in ", datetime.now() - start, " minutes.")
\ No newline at end of file
+    print("Completed in ", datetime.now() - start, " minutes.")
diff --git a/ais/engine/scripts/load_dor_parcels.py b/ais/engine/scripts/load_dor_parcels.py
index 5d3670ce..7afc3d7e 100644
--- a/ais/engine/scripts/load_dor_parcels.py
+++ b/ais/engine/scripts/load_dor_parcels.py
@@ -1,6 +1,4 @@
 import sys
-import os
-import csv
 import re
 from datetime import datetime
 from passyunk.data import DIRS_STD, SUFFIXES_STD
@@ -8,561 +6,549 @@
 from ais.models import Address
 from ais.util import parity_for_num, parity_for_range
 from ais import app
-from config import VALID_ADDRESS_LOW_SUFFIXES
 # DEV
-from pprint import pprint
 import traceback
-
-start = datetime.now()
-print('Starting...')
-
-"""SET UP"""
-
-config = app.config
-db = datum.connect(config['DATABASES']['engine'])
-
-source_def = config['BASE_DATA_SOURCES']['parcels']['dor']
-source_db_name = source_def['db']
-source_db_url = config['DATABASES'][source_db_name]
-source_db = datum.connect(source_db_url)
-source_field_map = source_def['field_map']
-source_table_name = source_def['table']
-source_table = source_db[source_table_name]
-source_geom_field = source_table.geom_field
-field_map = source_def['field_map']
-
-street_table = db['street_segment']
-parcel_table = db['dor_parcel']
-parcel_error_table = db['dor_parcel_error']
-parcel_error_polygon_table = db['dor_parcel_error_polygon']
-error_exempt_fields = ['frac',]
-WRITE_OUT = True
-
-# Regex
-street_name_re = re.compile('^[A-Z0-9 ]+$')
-unit_num_re = re.compile('^[A-Z0-9\-]+$')
-parcel_id_re = re.compile('^\d{3}(N|S)\d{6}$')
-geometry_re = re.compile('^(MULTI)?POLYGON')
-
-"""MAIN"""
-
-if WRITE_OUT:
-    print('Dropping indexes...')
-    parcel_table.drop_index('street_address')
-    print('Deleting existing parcels...')
-    parcel_table.delete()
-    print('Deleting existing parcel errors...')
-    parcel_error_table.delete()
-    print('Deleting existing parcel error polygons...')
-    parcel_error_polygon_table.delete()
-
-print('Reading streets...')
-street_stmt = '''
-    select street_full, seg_id, street_code, left_from, left_to, right_from, right_to
-    from {}
-'''.format(street_table.name)
-street_rows = db.execute(street_stmt)
-
-street_code_map = {}  # street_full => street_code
-street_full_map = {}  # street_code => street_full
-seg_map = {}  # street_full => [seg rows]
-
-for street_row in street_rows:
-    street_code = street_row['street_code']
-    street_full = street_row['street_full']
-
-    seg_map.setdefault(street_full, [])
-    seg_map[street_full].append(street_row)
-
-    street_code_map[street_full] = street_code
-    street_full_map[street_code] = street_full
-
-# TODO: currently there's a problem with parsing street names where a
-# single street_full will map to more than one street code. (It's dropping the
-# RAMP suffix where it shouldn't be. Only a few instances of this and so just 
-# override for now.
-street_code_map.update({
-    'VINE ST':      80120,
-    'MARKET ST':    53560,
-    'COMMERCE ST':  24500,
-})
-street_full_map.update({
-    80120:          'VINE ST',
-    53560:          'MARKET ST',
-    24500:          'COMMERCE ST',
-})
-
-print('Reading parcels from source...')
-# Get field names
-source_where = source_def['where']
-
-# DEV
-# source_table += ' SAMPLE(1)'
-# source_where += " AND mapreg = '001S050134'"
-# source_where += " AND objectid = 540985"
-#source_where += " AND rownum < 100"
-
-source_fields = list(field_map.values())
-source_parcels = source_table.read(where=source_where)
-source_parcel_map = {x['objectid']: x for x in source_parcels}
-
-parcels = []
-parcel_map = {}             # object ID => parcel object
-
-# QC
-error_map = {}              # object ID => error string
-warning_map = {}            # object ID => [warning strings]
-object_id = None            # Make this global so the error functions work
-should_add_parcel = None    # Declare this here for scope reasons
-bad_geom_parcels = []       # Object IDs
-
-address_counts = {}         # street_address => count
-parcel_id_counts = {}       # parcel_id => count
-
-# Use this to continue working on a parcel if one part of validation fails
-# class KeepGoing(Exception):
-#   pass
-
-def had_warning(reason, note=None):
-    global warning_map
-    global object_id
-    parcel_warnings = warning_map.setdefault(object_id, [])
-    warning = {
-        'reason':   reason,
-        'note':     note if note else '',
-    }
-    parcel_warnings.append(warning)
-
-def had_error(reason, note=None):
-    global error_map
-    global object_id
-    global should_add_parcel
-    parcel_errors = error_map.setdefault(object_id, [])
-    error = {
-        'reason':   reason,
-        'note':     note if note else '',
-    }
-    parcel_errors.append(error)
-    should_add_parcel = False
-
-# Loop over source parcels
-for i, source_parcel in enumerate(source_parcels):
-    try:
+import psycopg2
+import petl as etl
+import geopetl
+
+def main():
+    start = datetime.now()
+    print('Starting...')
+
+    """SET UP"""
+
+    config = app.config
+    db = datum.connect(config['DATABASES']['engine'])
+
+    VALID_ADDRESS_LOW_SUFFIXES = config['VALID_ADDRESS_LOW_SUFFIXES']
+
+    source_def = config['BASE_DATA_SOURCES']['parcels']['dor']
+    source_db_name = source_def['db']
+    source_db_url = config['DATABASES'][source_db_name]
+    source_db = datum.connect(source_db_url)
+    source_table_name = source_def['table']
+    source_table = source_db[source_table_name]
+    source_geom_field = source_table.geom_field
+    field_map = source_def['field_map']
+
+    street_table = db['street_segment']
+    parcel_table = db['dor_parcel']
+    parcel_error_table = db['dor_parcel_error']
+    parcel_error_polygon_table = db['dor_parcel_error_polygon']
+    error_exempt_fields = ['frac',]
+    WRITE_OUT = True
+
+    # Regex
+    unit_num_re = re.compile('^[A-Z0-9\-]+$')
+    parcel_id_re = re.compile('^\d{3}(N|S)\d{6}$')
+    geometry_re = re.compile('^(MULTI)?POLYGON')
+
+    """MAIN"""
+
+    if WRITE_OUT:
+        print('Dropping indexes...')
+        parcel_table.drop_index('street_address')
+        print('Deleting existing parcels...')
+        parcel_table.delete()
+        print('Deleting existing parcel errors...')
+        parcel_error_table.delete()
+        print('Deleting existing parcel error polygons...')
+        parcel_error_polygon_table.delete()
+
+    print('Reading streets...')
+    street_stmt = '''
+        select street_full, seg_id, street_code, left_from, left_to, right_from, right_to
+        from {}
+    '''.format(street_table.name)
+    street_rows = db.execute(street_stmt)
+
+    street_code_map = {}  # street_full => street_code
+    street_full_map = {}  # street_code => street_full
+    seg_map = {}  # street_full => [seg rows]
+
+    for street_row in street_rows:
+        street_code = street_row['street_code']
+        street_full = street_row['street_full']
+
+        seg_map.setdefault(street_full, [])
+        seg_map[street_full].append(street_row)
+
+        street_code_map[street_full] = street_code
+        street_full_map[street_code] = street_full
+
+    # TODO: currently there's a problem with parsing street names where a
+    # single street_full will map to more than one street code. (It's dropping the
+    # RAMP suffix where it shouldn't be. Only a few instances of this and so just 
+    # override for now.
+    street_code_map.update({
+        'VINE ST':      80120,
+        'MARKET ST':    53560,
+        'COMMERCE ST':  24500,
+    })
+    street_full_map.update({
+        80120:          'VINE ST',
+        53560:          'MARKET ST',
+        24500:          'COMMERCE ST',
+    })
+
+    # Get field names
+    source_where = source_def['where']
+
+    # DEV
+    # source_table += ' SAMPLE(1)'
+    # source_where += " AND rownum <= 20000"
+
+    source_fields = list(field_map.values())
+    print(f'Reading parcels from {source_table_name} at db {source_db_url}...')
+    source_parcels = source_table.read(where=source_where)
+    print(f'Read in {len(source_parcels)} rows.')
+    source_parcel_map = {x['objectid']: x for x in source_parcels}
+
+    parcels = []
+    parcel_map = {}             # object ID => parcel object
+
+    # QC
+    error_map = {}              # object ID => error string
+    warning_map = {}            # object ID => [warning strings]
+    object_id = None            # Make this global so the error functions work
+    should_add_parcel = None    # Declare this here for scope reasons
+    bad_geom_parcels = []       # Object IDs
+
+    address_counts = {}         # street_address => count
+    parcel_id_counts = {}       # parcel_id => count
+
+    def had_warning(reason, note=None):
+        parcel_warnings = warning_map.setdefault(object_id, [])
+        warning = {
+            'reason':   reason,
+            'note':     note if note else '',
+        }
+        parcel_warnings.append(warning)
+
+    def had_error(reason, note=None):
+        parcel_errors = error_map.setdefault(object_id, [])
+        error = {
+            'reason':   reason,
+            'note':     note if note else '',
+        }
+        parcel_errors.append(error)
+
+    # Loop over source parcels
+    for i, source_parcel in enumerate(source_parcels):
         if i % 50000 == 0:
             print(i)
-
-        should_add_parcel = True
-
-        # Strip whitespace, null out empty strings, zeroes
-        for field, value in source_parcel.items():
-            if isinstance(value, str):
-                value = value.strip()
-                if len(value) == 0 or value == '0':
-                    value = None
-                source_parcel[field] = value
-            elif value == 0:
-                source_parcel[field] = None
-
-        # Get attributes
-        object_id = source_parcel[field_map['source_object_id']]
-        address_low = source_parcel[field_map['address_low']]
-        address_low_suffix = source_parcel[field_map['address_low_suffix']]
-        address_low_fractional = source_parcel[field_map['address_low_frac']]
-        address_high = source_parcel[field_map['address_high']]
-        street_predir = source_parcel[field_map['street_predir']]
-        street_name = source_parcel[field_map['street_name']]
-        street_suffix = source_parcel[field_map['street_suffix']]
-        street_postdir = source_parcel[field_map['street_postdir']]
-        unit_num = source_parcel[field_map['unit_num']]
-        street_code = source_parcel[field_map['street_code']]
-        parcel_id = source_parcel[field_map['parcel_id']]
-        geometry = source_parcel[source_geom_field]
-
-        # Declare this here so the except clause doesn't bug out
-        source_address = None
-        
-        # Set this flag to false if we handle any specific address errors.
-        # If no specific errors are found, compare the parsed address to
-        # the source address to flag parser modifications.
-        should_check_street_full = True
-
-        # QC: Check address components
-        if street_predir and street_predir not in DIRS_STD:
-            had_warning('Non-standard predir')
-            should_check_street_full = False
-        if street_postdir and street_postdir not in DIRS_STD:
-            had_warning('Non-standard postdir')
-            should_check_street_full = False
-        if street_suffix and street_suffix not in SUFFIXES_STD:
-            had_warning('Non-standard suffix')
-            should_check_street_full = False
-        if unit_num and unit_num_re and not unit_num_re.match(unit_num):
-            had_warning('Invalid unit num')
-            should_check_street_full = False
-        #if address_low_fractional and address_low_fractional not in ('1/4', '1/3', '1/2'):
-        #    had_warning('Invalid address_low_frac')
-        #    should_check_street_full = False
-
-
-
-
-        # QC: Check street components
-        if street_name is None:
-            had_error('No street name')
-        if street_code is None:
-            had_warning('No street code')
-
-        # Make street full
-        if street_name:
-            street_comps = [street_predir, street_name, street_suffix, \
-                street_postdir]
-            street_full = ' '.join([x for x in street_comps if x])
-
-            # QC: Check if street full exists
-            found_street_full = True
-            if street_full not in street_code_map:
-                found_street_full = False
-                note = 'Unknown street: {}'.format(street_full)
-                # had_error('Unknown street', note=note)
-                had_warning('Unknown street', note=note)
-
-            if street_code:
-                # QC: Check if street code exists
-                if street_code not in street_full_map:
-                    had_warning('Unknown street code')
-
-                # QC: Check if street full matches street code
-                elif found_street_full and \
-                    street_code_map[street_full] != street_code:
-                    actual_street = street_full_map[street_code]
-                    note = 'Street code {} => {}'.format(street_code, actual_street)
-                    had_warning('Incorrect street code', note=note)
-
-        # QC: Check for low address number
-        if address_low is None:
-            had_error('No address number')
-
-        # Clean up
-        if address_high == 0:
-            address_high = None
-
-        if address_low_suffix not in VALID_ADDRESS_LOW_SUFFIXES:
-            address_low_suffix = None
-        if address_low_suffix == '2':
-            address_low_fractional = '1/2'
-            address_low_suffix = None
-        # Handle ranges
-        if address_low and address_high:
-            address_low_str = str(address_low)
-            address_high_str = str(address_high)
-            len_address_low = len(address_low_str)
-            len_address_high = len(address_high_str)
-            address_high_full = None
-
-            if len(address_high_str) != 2:
-                had_warning('High address should be two digits')
-
-            if not address_high_str.isnumeric():
+        try:
+
+            should_add_parcel = True
+
+            # Strip whitespace, null out empty strings, zeroes
+            for field, value in source_parcel.items():
+                if isinstance(value, str):
+                    value = value.strip()
+                    if len(value) == 0 or value == '0':
+                        value = None
+                    source_parcel[field] = value
+                elif value == 0:
+                    source_parcel[field] = None
+
+            # Get attributes
+            object_id = source_parcel[field_map['source_object_id']]
+            address_low = source_parcel[field_map['address_low']]
+            address_low_suffix = source_parcel[field_map['address_low_suffix']]
+            address_low_fractional = source_parcel[field_map['address_low_frac']]
+            address_high = source_parcel[field_map['address_high']]
+            street_predir = source_parcel[field_map['street_predir']]
+            street_name = source_parcel[field_map['street_name']]
+            street_suffix = source_parcel[field_map['street_suffix']]
+            street_postdir = source_parcel[field_map['street_postdir']]
+            unit_num = source_parcel[field_map['unit_num']]
+            street_code = source_parcel[field_map['street_code']]
+            parcel_id = source_parcel[field_map['parcel_id']]
+            geometry = source_parcel[source_geom_field]
+
+            # Declare this here so the except clause doesn't bug out
+            source_address = None
+            
+            # Set this flag to false if we handle any specific address errors.
+            # If no specific errors are found, compare the parsed address to
+            # the source address to flag parser modifications.
+            should_check_street_full = True
+
+            # QC: Check address components
+            if street_predir and street_predir not in DIRS_STD:
+                had_warning('Non-standard predir')
+                should_check_street_full = False
+            if street_postdir and street_postdir not in DIRS_STD:
+                had_warning('Non-standard postdir')
+                should_check_street_full = False
+            if street_suffix and street_suffix not in SUFFIXES_STD:
+                had_warning('Non-standard suffix')
+                should_check_street_full = False
+            if unit_num and unit_num_re and not unit_num_re.match(unit_num):
+                had_warning('Invalid unit num')
+                should_check_street_full = False
+
+            # QC: Check street components
+            if street_name is None:
+                should_add_parcel = False
+                had_error('No street name')
+            if street_code is None:
+                had_warning('No street code')
+
+            # Make street full
+            if street_name:
+                street_comps = [street_predir, street_name, street_suffix, \
+                    street_postdir]
+                street_full = ' '.join([x for x in street_comps if x])
+
+                # QC: Check if street full exists
+                found_street_full = True
+                if street_full not in street_code_map:
+                    found_street_full = False
+                    note = 'Unknown street: {}'.format(street_full)
+                    had_warning('Unknown street', note=note)
+
+                if street_code:
+                    # QC: Check if street code exists
+                    if street_code not in street_full_map:
+                        had_warning('Unknown street code')
+
+                    # QC: Check if street full matches street code
+                    elif found_street_full and \
+                        street_code_map[street_full] != street_code:
+                        actual_street = street_full_map[street_code]
+                        note = 'Street code {} => {}'.format(street_code, actual_street)
+                        had_warning('Incorrect street code', note=note)
+
+            # QC: Check for low address number
+            if address_low is None:
+                should_add_parcel = False
+                had_error('No address number')
+
+            # Clean up
+            if address_high == 0:
                 address_high = None
-                had_warning('Invalid high address')
-
-            if address_high:
-                # Case: 1234-36 or 1234-6
-                if len_address_high < len_address_low:
-                    # Make address high full and compare to address low
-                    address_high_prefix = address_low_str[:-len_address_high]
-                    address_high_full = int(address_high_prefix + address_high_str)
-                # Cases: 1234-1236 or 2-12
-                elif len_address_low == len_address_high or \
-                    (len_address_low == 1 and len_address_high == 2):
-                    address_high_full = address_high
-                else:
-                    had_error('Address spans multiple hundred blocks')
-
-                # Case: 317-315
-                if address_high_full:
-                    if address_high_full < address_low:
-                        # print(address_low, address_high_full)
-                        had_error('Inverted range address')
-
-                    # Make sure both addresses are on the same hundred block
-                    hun_block_low = address_low - (address_low % 100)
-                    hun_block_high = address_high_full - (address_high_full % 100)
-                    if hun_block_low != hun_block_high:
-                        # print(hun_block_low, hun_block_high)
+
+            if address_low_suffix not in VALID_ADDRESS_LOW_SUFFIXES:
+                address_low_suffix = None
+            if address_low_suffix == '2':
+                address_low_fractional = '1/2'
+                address_low_suffix = None
+            # Handle ranges
+            if address_low and address_high:
+                address_low_str = str(address_low)
+                address_high_str = str(address_high)
+                len_address_low = len(address_low_str)
+                len_address_high = len(address_high_str)
+                address_high_full = None
+
+                if len(address_high_str) != 2:
+                    had_warning('High address should be two digits')
+
+                if not address_high_str.isnumeric():
+                    address_high = None
+                    had_warning('Invalid high address')
+
+                if address_high:
+                    # Case: 1234-36 or 1234-6
+                    if len_address_high < len_address_low:
+                        # Make address high full and compare to address low
+                        address_high_prefix = address_low_str[:-len_address_high]
+                        address_high_full = int(address_high_prefix + address_high_str)
+                    # Cases: 1234-1236 or 2-12
+                    elif len_address_low == len_address_high or \
+                        (len_address_low == 1 and len_address_high == 2):
+                        address_high_full = address_high
+                    else:
+                        should_add_parcel = False
                         had_error('Address spans multiple hundred blocks')
 
-                    address_high = str(address_high_full)[-2:]
-
-        # Make address full
-        address_full = None
-        if address_low:
-            address_full = str(address_low)
-            if address_low_suffix:
-                address_full += address_low_suffix
-            if address_low_fractional:
-                address_full += ' ' + address_low_fractional
-            if address_high:
-                address_full += '-' + str(address_high)
-
-        # Get unit
-        unit_full = None
-        if unit_num:
-            unit_full = '# {}'.format(unit_num)
-        
-        address = None
-        
-        if address_full and street_full:
-            source_address_comps = [address_full, street_full, unit_full]
-            source_address = ' '.join([x for x in source_address_comps if x])
-
-            # Try to parse
-            try:
-                address = Address(source_address)
-
-                # QC: check for miscellaneous parcel modifications
-                street_address = address.street_address
-                if should_check_street_full and source_address != street_address:
-                    note = 'Parser changes: {} => {}'.format(source_address, street_address)
-                    had_warning('Parser changes', note=note)
-
-                # QC: check for duplicate address
-                address_counts.setdefault(street_address, 0)
-                address_counts[street_address] += 1
+                    # Case: 317-315
+                    if address_high_full:
+                        if address_high_full < address_low:
+                            should_add_parcel = False
+                            had_error('Inverted range address')
+
+                        # Make sure both addresses are on the same hundred block
+                        hun_block_low = address_low - (address_low % 100)
+                        hun_block_high = address_high_full - (address_high_full % 100)
+                        if hun_block_low != hun_block_high:
+                            should_add_parcel = False
+                            had_error('Address spans multiple hundred blocks')
+
+                        address_high = str(address_high_full)[-2:]
+
+            # Make address full
+            address_full = None
+            if address_low:
+                address_full = str(address_low)
+                if address_low_suffix:
+                    address_full += address_low_suffix
+                if address_low_fractional:
+                    address_full += ' ' + address_low_fractional
+                if address_high:
+                    address_full += '-' + str(address_high)
+
+            # Get unit
+            unit_full = None
+            if unit_num:
+                unit_full = '# {}'.format(unit_num)
             
-            except Exception as e:
-                print(source_address)
-                had_error('Could not parse')
-
-        # QC: parcel ID (aka mapreg)
-        if parcel_id is None:
-            had_error('No parcel ID')
-        else:
-            # Check for duplicate
-            parcel_id_counts.setdefault(parcel_id, 0)
-            parcel_id_counts[parcel_id] += 1
-
-            if not parcel_id_re.match(parcel_id):
-                had_warning('Invalid parcel ID')
-
-        # QC: geometry
-        if not geometry_re.match(geometry):
-            had_error('Invalid geometry')
-            bad_geom_parcels.append(object_id)
-
-        '''
-        STREET MATCH
-        '''
-
-        if address:
-            # Get the parsed street_full
-            street_full = address.street_full
-
-            if street_full in seg_map:
-                address_low = address.address_low
-                address_high = address.address_high
+            address = None
+            
+            if address_full and street_full:
+                source_address_comps = [address_full, street_full, unit_full]
+                source_address = ' '.join([x for x in source_address_comps if x])
+
+                # Try to parse
+                try:
+                    address = Address(source_address)
+
+                    # QC: check for miscellaneous parcel modifications
+                    street_address = address.street_address
+                    if should_check_street_full and source_address != street_address:
+                        note = 'Parser changes: {} => {}'.format(source_address, street_address)
+                        had_warning('Parser changes', note=note)
+
+                    # QC: check for duplicate address
+                    address_counts.setdefault(street_address, 0)
+                    address_counts[street_address] += 1
+                
+                except Exception as e:
+                    should_add_parcel = False
+                    had_error('Could not parse')
+
+            # QC: parcel ID (aka mapreg)
+            if parcel_id is None:
+                should_add_parcel = False
+                had_error('No parcel ID')
+            else:
+                # Check for duplicate
+                parcel_id_counts.setdefault(parcel_id, 0)
+                parcel_id_counts[parcel_id] += 1
+
+                if not parcel_id_re.match(parcel_id):
+                    had_warning('Invalid parcel ID')
+
+            # QC: geometry
+            if not geometry_re.match(geometry):
+                should_add_parcel = False
+                had_error('Invalid geometry')
+                bad_geom_parcels.append(object_id)
+
+            '''
+            STREET MATCH
+            '''
+
+            if address:
+                # Get the parsed street_full
                 street_full = address.street_full
-                address_parity = parity_for_num(address_low)
-                matching_segs = seg_map[street_full]
-                matching_seg = None
-                matching_side = None
-                had_alias = False  # TODO: check for aliases
-
-                # Loop through segs for that street full
-                for seg in matching_segs:
-                    left_from = seg['left_from']
-                    left_to = seg['left_to']
-                    right_from = seg['right_from']
-                    right_to = seg['right_to']
-
-                    left_parity = parity_for_range(left_from, left_to)
-                    right_parity = parity_for_range(right_from, right_to)
-
-                    # Match to side of street based on parity
-                    check_from = None
-                    check_to = None
-
-                    if left_parity in [address_parity, 'B']:
-                        check_from = left_from
-                        check_to = left_to
-                        matching_side = 'L'
-                    elif right_parity in [address_parity, 'B']:
-                        check_from = right_from
-                        check_to = right_to
-                        matching_side = 'R'
-                    else:
-                        continue
-
-                    # If it's in range
-                    if check_from <= address_low <= check_to:
-                        # And it's a single address
-                        if address_high is None:
-                            matching_seg = seg
-                            break
-                        # Otherwise if it's a range address
+
+                if street_full in seg_map:
+                    address_low = address.address_low
+                    address_high = address.address_high
+                    street_full = address.street_full
+                    address_parity = parity_for_num(address_low)
+                    matching_segs = seg_map[street_full]
+                    matching_seg = None
+                    matching_side = None
+                    had_alias = False  # TODO: check for aliases
+
+                    # Loop through segs for that street full
+                    for seg in matching_segs:
+                        left_from = seg['left_from']
+                        left_to = seg['left_to']
+                        right_from = seg['right_from']
+                        right_to = seg['right_to']
+
+                        left_parity = parity_for_range(left_from, left_to)
+                        right_parity = parity_for_range(right_from, right_to)
+
+                        # Match to side of street based on parity
+                        check_from = None
+                        check_to = None
+
+                        if left_parity in [address_parity, 'B']:
+                            check_from = left_from
+                            check_to = left_to
+                            matching_side = 'L'
+                        elif right_parity in [address_parity, 'B']:
+                            check_from = right_from
+                            check_to = right_to
+                            matching_side = 'R'
                         else:
-                            # If we already had a match, flag multiple matches
-                            if matching_seg:
-                                seg_ids = sorted([x['seg_id'] for x in [matching_seg, seg]])
-                                note = ','.join([str(x) for x in seg_ids])
-                                had_warning('Range address matches multiple street segments', note=note)
+                            continue
+
+                        # If it's in range
+                        if check_from <= address_low <= check_to:
+                            # And it's a single address
+                            if address_high is None:
+                                matching_seg = seg
+                                break
+                            # Otherwise if it's a range address
+                            else:
+                                # If we already had a match, flag multiple matches
+                                if matching_seg:
+                                    seg_ids = sorted([x['seg_id'] for x in [matching_seg, seg]])
+                                    note = ','.join([str(x) for x in seg_ids])
+                                    had_warning('Range address matches multiple street segments', note=note)
+                                    
+                                # Check if the high address is greater than the street max
+                                if check_to < address_high:
+                                    # Otherwise, make the match and keep looking (in case
+                                    # it matches to multiple segments)
+                                    had_warning('High address out of street range')
                                 
-                            # Check if the high address is greater than the street max
-                            if check_to < address_high:
-                                # Otherwise, make the match and keep looking (in case
-                                # it matches to multiple segments)
-                                had_warning('High address out of street range')
-                            
-                            matching_seg = seg
-
-                if matching_seg is None:
-                    # had_error('Out of street range')
-                    # should_add_parcel = False
-                    had_warning('Out of street range')
-                    should_add_parcel = True
-
-
-        '''
-        END STREET MATCH
-        '''
-
-        # Make parcel object
-        if should_add_parcel:
-            parcel = dict(address)
-            #Remove fields not in parcel tables:
-            parcel.pop('zip_code', None)
-            parcel.pop('zip_4', None)
-            #Add fields:
-            parcel.update({
-                'parcel_id':        parcel_id,
-                'source_object_id': object_id,
-                'source_address':   source_address,
-                'geom':             geometry,
-            })
-            parcels.append(parcel)
-            parcel_map[object_id] = parcel
-
-    # except ValueError as e:
-    #   # print('Parcel {}: {}'.format(parcel_id, e))
-    #   reason = str(e)
-    #   had_error(reason)
-
-    except Exception as e:
-        print('{}: Unhandled error'.format(source_parcel))
-        print(parcel_id)
-        print(traceback.format_exc())
-        sys.exit()
-
-print('Checking for duplicates...')
-
-# Check for duplicate parcel IDs. Use source parcels for most results.
-for source_parcel in source_parcels:
-    # Set object ID here so logging function works
-    object_id = source_parcel['objectid']
-    parcel_id = source_parcel['mapreg']
-    count = parcel_id_counts.get(parcel_id, 0)
-    if count > 1:
-        note = 'Parcel ID count: {}'.format(count)
-        had_warning('Duplicate parcel ID', note=note)
-
-# Check for duplicate addresses. Use parcels since source parcels don't have
-# an address.
-for parcel in parcels:
-    # Set object ID here so logging function works
-    object_id = parcel['source_object_id']
-    street_address = parcel['street_address']
-    count = address_counts.get(street_address, 0)
-    if count > 1:
-        note = 'Address count: {}'.format(count)
-        had_warning('Duplicate address', note=note)
-
-# Remember how many parcels we went through before we delete them all
-parcel_count = len(parcels)
-
-if WRITE_OUT:
-    print('Writing parcels...')
-    parcel_table.write(parcels, chunk_size=50000)
-
-    print('Writing parcel errors...')
-    errors = []
-    source_non_geom_fields = [x for x in source_fields if x != source_geom_field]
-
-    for level in ['error', 'warning']:
-        issue_map = error_map if level == 'error' else warning_map
-        for object_id, issues in issue_map.items():
-            for issue in issues:
-                reason = issue['reason']
-                note = issue['note']
+                                matching_seg = seg
+
+                    if matching_seg is None:
+                        had_warning('Out of street range')
+                        should_add_parcel = True
+
+
+            '''
+            END STREET MATCH
+            '''
+
+            # Make parcel object
+            if should_add_parcel:
+                parcel = dict(address)
+                #Remove fields not in parcel tables:
+                parcel.pop('zip_code', None)
+                parcel.pop('zip_4', None)
+                #Add fields:
+                parcel.update({
+                    'parcel_id':        parcel_id,
+                    'source_object_id': object_id,
+                    'source_address':   source_address,
+                    'geom':             geometry,
+                })
+                parcels.append(parcel)
+                parcel_map[object_id] = parcel
+
+        except Exception as e:
+            print('{}: Unhandled error'.format(source_parcel))
+            print(parcel_id)
+            print(traceback.format_exc())
+            raise e
+
+    print('Checking for duplicates...')
+
+    # Check for duplicate parcel IDs. Use source parcels for most results.
+    for source_parcel in source_parcels:
+        # Set object ID here so logging function works
+        object_id = source_parcel['objectid']
+        parcel_id = source_parcel['mapreg']
+        count = parcel_id_counts.get(parcel_id, 0)
+        if count > 1:
+            note = 'Parcel ID count: {}'.format(count)
+            had_warning('Duplicate parcel ID', note=note)
+
+    # Check for duplicate addresses. Use parcels since source parcels don't have
+    # an address.
+    for parcel in parcels:
+        # Set object ID here so logging function works
+        object_id = parcel['source_object_id']
+        street_address = parcel['street_address']
+        count = address_counts.get(street_address, 0)
+        if count > 1:
+            note = 'Address count: {}'.format(count)
+            had_warning('Duplicate address', note=note)
+
+    # Remember how many parcels we went through before we delete them all
+    parcel_count = len(parcels)
+
+    if WRITE_OUT:
+        print('Writing parcels...')
+        parcel_table.write(parcels, chunk_size=50000)
+
+        print('Writing parcel errors...')
+        errors = []
+        source_non_geom_fields = [x for x in source_fields if x != source_geom_field]
+
+        for level in ['error', 'warning']:
+            issue_map = error_map if level == 'error' else warning_map
+            for object_id, issues in issue_map.items():
+                for issue in issues:
+                    reason = issue['reason']
+                    note = issue['note']
+                    source_parcel = source_parcel_map[object_id]
+                    
+                    # Make error row
+                    error = {x: source_parcel[x] if source_parcel[x] is not None \
+                        else '' for x in source_non_geom_fields if x not in error_exempt_fields}
+
+                    # Make this work for integer fields
+                    if error['house'] == '':
+                        error['house'] = None
+                    if error['stcod'] == '':
+                        error['stcod'] = None
+
+                    error.update({
+                        'level':    level,
+                        'reason':   reason,
+                        'notes':    note,
+                    })
+
+                    errors.append(error)
+
+        parcel_error_table.write(errors, chunk_size=150000)
+        del errors
+
+        print('Writing parcel error polygons...')
+        error_polygons = []
+
+        for level in ['error', 'warning']:
+            issue_map = error_map if level == 'error' else warning_map
+
+            for object_id, issues in issue_map.items():
+                # If this object had a geometry error, skip
+                if object_id in bad_geom_parcels:
+                    continue
+
+                # Roll up reasons, notes
+                reasons = [x['reason'] for x in issues]
+                reasons_joined = '; '.join(sorted(reasons))
+                notes = [x['note'] for x in issues if x['note'] != '']
+                notes_joined = '; '.join(notes)
                 source_parcel = source_parcel_map[object_id]
                 
                 # Make error row
-                error = {x: source_parcel[x] if source_parcel[x] is not None \
-                    else '' for x in source_non_geom_fields if x not in error_exempt_fields}
-
-                # Make this work for integer fields
-                if error['house'] == '':
-                    error['house'] = None
-                if error['stcod'] == '':
-                    error['stcod'] = None
-
-                error.update({
-                    'level':    level,
-                    'reason':   reason,
-                    'notes':    note,
+                error_polygon = {x: source_parcel[x] if x is not None \
+                    else '' for x in source_fields if x not in error_exempt_fields}
+
+                # Add/clean up fields
+                if error_polygon['house'] == '':
+                    error_polygon['house'] = None
+                if error_polygon['stcod'] == '':
+                    error_polygon['stcod'] = None
+
+                error_polygon.update({
+                    'reasons':      reasons_joined,
+                    'reason_count': len(reasons),
+                    'notes':        notes_joined,
                 })
+                error_polygons.append(error_polygon)
 
-                errors.append(error)
-
-    parcel_error_table.write(errors, chunk_size=150000)
-    del errors
+        target_dsn = config['DATABASES']['engine']
+        target_user = target_dsn[target_dsn.index("//") + 2:target_dsn.index(":", target_dsn.index("//"))]
+        target_pw = target_dsn[target_dsn.index(":",target_dsn.index(target_user)) + 1:target_dsn.index("@")]
+        target_name = target_dsn[target_dsn.index("/", target_dsn.index("@")) + 1:]
+        target_conn = psycopg2.connect('dbname={db_name} user={db_user} password={db_pw} host=localhost'.format(db_name=target_name, db_user=target_user, db_pw=target_pw))
+        target_table_name = 'public.dor_parcel_error_polygon'
+        error_polygon_rows = etl.fromdicts(error_polygons)
+        error_polygon_rows.topostgis(target_conn, target_table_name)
 
-    print('Writing parcel error polygons...')
-    error_polygons = []
+        del error_polygons
 
-    for level in ['error', 'warning']:
-        issue_map = error_map if level == 'error' else warning_map
+        print('Creating indexes...')
+        parcel_table.create_index('street_address')
+        # TODO: index error tables?
 
-        for object_id, issues in issue_map.items():
-            # If this object had a geometry error, skip
-            if object_id in bad_geom_parcels:
-                continue
+    db.close()
 
-            # Roll up reasons, notes
-            reasons = [x['reason'] for x in issues]
-            reasons_joined = '; '.join(sorted(reasons))
-            notes = [x['note'] for x in issues if x['note'] != '']
-            notes_joined = '; '.join(notes)
-            source_parcel = source_parcel_map[object_id]
-            
-            # Make error row
-            error_polygon = {x: source_parcel[x] if x is not None \
-                else '' for x in source_fields if x not in error_exempt_fields}
-
-            # Add/clean up fields
-            if error_polygon['house'] == '':
-                error_polygon['house'] = None
-            if error_polygon['stcod'] == '':
-                error_polygon['stcod'] = None
-
-            error_polygon.update({
-                'reasons':      reasons_joined,
-                'reason_count': len(reasons),
-                'notes':        notes_joined,
-                # 'shape':        source_parcel[wkt_field],
-            })
-            error_polygons.append(error_polygon)
-
-    parcel_error_polygon_table.write(error_polygons, chunk_size=50000)
-    del error_polygons
-
-    print('Creating indexes...')
-    parcel_table.create_index('street_address')
-    # TODO: index error tables?
-
-#source_db.close()
-db.close()
-
-print('Finished in {} seconds'.format(datetime.now() - start))
-print('Processed {} parcels'.format(parcel_count))
-print('{} errors'.format(len(error_map)))
-print('{} warnings'.format(len(warning_map)))
+    print('Finished in {} seconds'.format(datetime.now() - start))
+    print('Processed {} parcels'.format(parcel_count))
+    print('{} errors'.format(len(error_map)))
+    print('{} warnings'.format(len(warning_map)))
diff --git a/ais/engine/scripts/load_dor_parcels_dev.py b/ais/engine/scripts/load_dor_parcels_dev.py
new file mode 100644
index 00000000..081c8c5e
--- /dev/null
+++ b/ais/engine/scripts/load_dor_parcels_dev.py
@@ -0,0 +1,590 @@
+import sys
+import os
+import csv
+import re
+from datetime import datetime
+from passyunk.data import DIRS_STD, SUFFIXES_STD
+import datum
+from ais.models import Address
+from ais.util import parity_for_num, parity_for_range
+from ais import app
+# DEV
+from pprint import pprint
+import traceback
+import psycopg2
+import petl as etl
+import geopetl
+
+def main():
+    start = datetime.now()
+    print('Starting...')
+
+    """SET UP"""
+
+    config = app.config
+    db = datum.connect(config['DATABASES']['engine'])
+
+    VALID_ADDRESS_LOW_SUFFIXES = config['VALID_ADDRESS_LOW_SUFFIXES']
+
+    source_def = config['BASE_DATA_SOURCES']['parcels']['dor']
+    source_db_name = source_def['db']
+    source_db_url = config['DATABASES'][source_db_name]
+    source_db = datum.connect(source_db_url)
+    source_field_map = source_def['field_map']
+    source_table_name = source_def['table']
+    source_table = source_db[source_table_name]
+    source_geom_field = source_table.geom_field
+    field_map = source_def['field_map']
+
+    street_table = db['street_segment']
+    parcel_table = db['dor_parcel']
+    parcel_error_table = db['dor_parcel_error']
+    parcel_error_polygon_table = db['dor_parcel_error_polygon']
+    error_exempt_fields = ['frac',]
+    WRITE_OUT = True
+
+    # Regex
+    street_name_re = re.compile('^[A-Z0-9 ]+$')
+    unit_num_re = re.compile('^[A-Z0-9\-]+$')
+    parcel_id_re = re.compile('^\d{3}(N|S)\d{6}$')
+    geometry_re = re.compile('^(MULTI)?POLYGON')
+
+    """MAIN"""
+
+    if WRITE_OUT:
+        print('Dropping indexes...')
+        parcel_table.drop_index('street_address')
+        print('Deleting existing parcels...')
+        parcel_table.delete()
+        print('Deleting existing parcel errors...')
+        parcel_error_table.delete()
+        print('Deleting existing parcel error polygons...')
+        parcel_error_polygon_table.delete()
+
+    print('Reading streets...')
+    street_stmt = '''
+        select street_full, seg_id, street_code, left_from, left_to, right_from, right_to
+        from {}
+    '''.format(street_table.name)
+    street_rows = db.execute(street_stmt)
+
+    street_code_map = {}  # street_full => street_code
+    street_full_map = {}  # street_code => street_full
+    seg_map = {}  # street_full => [seg rows]
+
+    for street_row in street_rows:
+        street_code = street_row['street_code']
+        street_full = street_row['street_full']
+
+        seg_map.setdefault(street_full, [])
+        seg_map[street_full].append(street_row)
+
+        street_code_map[street_full] = street_code
+        street_full_map[street_code] = street_full
+
+    # TODO: currently there's a problem with parsing street names where a
+    # single street_full will map to more than one street code. (It's dropping the
+    # RAMP suffix where it shouldn't be. Only a few instances of this and so just 
+    # override for now.
+    street_code_map.update({
+        'VINE ST':      80120,
+        'MARKET ST':    53560,
+        'COMMERCE ST':  24500,
+    })
+    street_full_map.update({
+        80120:          'VINE ST',
+        53560:          'MARKET ST',
+        24500:          'COMMERCE ST',
+    })
+
+    # Get field names
+    source_where = source_def['where']
+
+    # DEV
+    # source_table += ' SAMPLE(1)'
+    # source_where += " AND mapreg = '001S050134'"
+    # source_where += " AND objectid = 540985"
+    #source_where += " AND rownum < 100"
+
+    source_fields = list(field_map.values())
+    print(f'Reading parcels from {source_table_name} at db {source_db_url}...')
+    source_parcels = source_table.read(where=source_where)
+    print(f'Read in {len(source_parcels)} rows.')
+    source_parcel_map = {x['objectid']: x for x in source_parcels}
+
+    parcels = []
+    parcel_map = {}             # object ID => parcel object
+
+    # QC
+    error_map = {}              # object ID => error string
+    warning_map = {}            # object ID => [warning strings]
+    object_id = None            # Make this global so the error functions work
+    should_add_parcel = None    # Declare this here for scope reasons
+    bad_geom_parcels = []       # Object IDs
+
+    address_counts = {}         # street_address => count
+    parcel_id_counts = {}       # parcel_id => count
+
+    # Use this to continue working on a parcel if one part of validation fails
+    # class KeepGoing(Exception):
+    #   pass
+
+    def had_warning(reason, note=None):
+        try:
+            global warning_map
+            global object_id
+            parcel_warnings = warning_map.setdefault(object_id, [])
+            warning = {
+                'reason':   reason,
+                'note':     note if note else '',
+            }
+            parcel_warnings.append(warning)
+        except Exception as e:
+            pass
+
+    def had_error(reason, note=None):
+        try:
+            global error_map
+            global object_id
+            global should_add_parcel
+            parcel_errors = error_map.setdefault(object_id, [])
+            error = {
+                'reason':   reason,
+                'note':     note if note else '',
+            }
+            parcel_errors.append(error)
+            should_add_parcel = False
+        except Exception as e:
+            pass
+
+    # Loop over source parcels
+    for i, source_parcel in enumerate(source_parcels):
+        try:
+            if i % 50000 == 0:
+                print(i)
+
+            should_add_parcel = True
+
+            # Strip whitespace, null out empty strings, zeroes
+            for field, value in source_parcel.items():
+                if isinstance(value, str):
+                    value = value.strip()
+                    if len(value) == 0 or value == '0':
+                        value = None
+                    source_parcel[field] = value
+                elif value == 0:
+                    source_parcel[field] = None
+
+            # Get attributes
+            object_id = source_parcel[field_map['source_object_id']]
+            address_low = source_parcel[field_map['address_low']]
+            address_low_suffix = source_parcel[field_map['address_low_suffix']]
+            address_low_fractional = source_parcel[field_map['address_low_frac']]
+            address_high = source_parcel[field_map['address_high']]
+            street_predir = source_parcel[field_map['street_predir']]
+            street_name = source_parcel[field_map['street_name']]
+            street_suffix = source_parcel[field_map['street_suffix']]
+            street_postdir = source_parcel[field_map['street_postdir']]
+            unit_num = source_parcel[field_map['unit_num']]
+            street_code = source_parcel[field_map['street_code']]
+            parcel_id = source_parcel[field_map['parcel_id']]
+            geometry = source_parcel[source_geom_field]
+
+            # Declare this here so the except clause doesn't bug out
+            source_address = None
+            
+            # Set this flag to false if we handle any specific address errors.
+            # If no specific errors are found, compare the parsed address to
+            # the source address to flag parser modifications.
+            should_check_street_full = True
+
+            # QC: Check address components
+            if street_predir and street_predir not in DIRS_STD:
+                had_warning('Non-standard predir')
+                should_check_street_full = False
+            if street_postdir and street_postdir not in DIRS_STD:
+                had_warning('Non-standard postdir')
+                should_check_street_full = False
+            if street_suffix and street_suffix not in SUFFIXES_STD:
+                had_warning('Non-standard suffix')
+                should_check_street_full = False
+            if unit_num and unit_num_re and not unit_num_re.match(unit_num):
+                had_warning('Invalid unit num')
+                should_check_street_full = False
+            #if address_low_fractional and address_low_fractional not in ('1/4', '1/3', '1/2'):
+            #    had_warning('Invalid address_low_frac')
+            #    should_check_street_full = False
+
+
+
+
+            # QC: Check street components
+            if street_name is None:
+                had_error('No street name')
+            if street_code is None:
+                had_warning('No street code')
+
+            # Make street full
+            if street_name:
+                street_comps = [street_predir, street_name, street_suffix, \
+                    street_postdir]
+                street_full = ' '.join([x for x in street_comps if x])
+
+                # QC: Check if street full exists
+                found_street_full = True
+                if street_full not in street_code_map:
+                    found_street_full = False
+                    note = 'Unknown street: {}'.format(street_full)
+                    # had_error('Unknown street', note=note)
+                    had_warning('Unknown street', note=note)
+
+                if street_code:
+                    # QC: Check if street code exists
+                    if street_code not in street_full_map:
+                        had_warning('Unknown street code')
+
+                    # QC: Check if street full matches street code
+                    elif found_street_full and \
+                        street_code_map[street_full] != street_code:
+                        actual_street = street_full_map[street_code]
+                        note = 'Street code {} => {}'.format(street_code, actual_street)
+                        had_warning('Incorrect street code', note=note)
+
+            # QC: Check for low address number
+            if address_low is None:
+                had_error('No address number')
+
+            # Clean up
+            if address_high == 0:
+                address_high = None
+
+            if address_low_suffix not in VALID_ADDRESS_LOW_SUFFIXES:
+                address_low_suffix = None
+            if address_low_suffix == '2':
+                address_low_fractional = '1/2'
+                address_low_suffix = None
+            # Handle ranges
+            if address_low and address_high:
+                address_low_str = str(address_low)
+                address_high_str = str(address_high)
+                len_address_low = len(address_low_str)
+                len_address_high = len(address_high_str)
+                address_high_full = None
+
+                if len(address_high_str) != 2:
+                    had_warning('High address should be two digits')
+
+                if not address_high_str.isnumeric():
+                    address_high = None
+                    had_warning('Invalid high address')
+
+                if address_high:
+                    # Case: 1234-36 or 1234-6
+                    if len_address_high < len_address_low:
+                        # Make address high full and compare to address low
+                        address_high_prefix = address_low_str[:-len_address_high]
+                        address_high_full = int(address_high_prefix + address_high_str)
+                    # Cases: 1234-1236 or 2-12
+                    elif len_address_low == len_address_high or \
+                        (len_address_low == 1 and len_address_high == 2):
+                        address_high_full = address_high
+                    else:
+                        had_error('Address spans multiple hundred blocks')
+
+                    # Case: 317-315
+                    if address_high_full:
+                        if address_high_full < address_low:
+                            # print(address_low, address_high_full)
+                            had_error('Inverted range address')
+
+                        # Make sure both addresses are on the same hundred block
+                        hun_block_low = address_low - (address_low % 100)
+                        hun_block_high = address_high_full - (address_high_full % 100)
+                        if hun_block_low != hun_block_high:
+                            # print(hun_block_low, hun_block_high)
+                            had_error('Address spans multiple hundred blocks')
+
+                        address_high = str(address_high_full)[-2:]
+
+            # Make address full
+            address_full = None
+            if address_low:
+                address_full = str(address_low)
+                if address_low_suffix:
+                    address_full += address_low_suffix
+                if address_low_fractional:
+                    address_full += ' ' + address_low_fractional
+                if address_high:
+                    address_full += '-' + str(address_high)
+
+            # Get unit
+            unit_full = None
+            if unit_num:
+                unit_full = '# {}'.format(unit_num)
+            
+            address = None
+            
+            if address_full and street_full:
+                source_address_comps = [address_full, street_full, unit_full]
+                source_address = ' '.join([x for x in source_address_comps if x])
+
+                # Try to parse
+                try:
+                    address = Address(source_address)
+
+                    # QC: check for miscellaneous parcel modifications
+                    street_address = address.street_address
+                    if should_check_street_full and source_address != street_address:
+                        note = 'Parser changes: {} => {}'.format(source_address, street_address)
+                        had_warning('Parser changes', note=note)
+
+                    # QC: check for duplicate address
+                    address_counts.setdefault(street_address, 0)
+                    address_counts[street_address] += 1
+                
+                except Exception as e:
+                    #print(source_address)
+                    had_error('Could not parse')
+
+            # QC: parcel ID (aka mapreg)
+            if parcel_id is None:
+                had_error('No parcel ID')
+            else:
+                # Check for duplicate
+                parcel_id_counts.setdefault(parcel_id, 0)
+                parcel_id_counts[parcel_id] += 1
+
+                if not parcel_id_re.match(parcel_id):
+                    had_warning('Invalid parcel ID')
+
+            # QC: geometry
+            if not geometry_re.match(geometry):
+                had_error('Invalid geometry')
+                bad_geom_parcels.append(object_id)
+
+            '''
+            STREET MATCH
+            '''
+
+            if address:
+                # Get the parsed street_full
+                street_full = address.street_full
+
+                if street_full in seg_map:
+                    address_low = address.address_low
+                    address_high = address.address_high
+                    street_full = address.street_full
+                    address_parity = parity_for_num(address_low)
+                    matching_segs = seg_map[street_full]
+                    matching_seg = None
+                    matching_side = None
+                    had_alias = False  # TODO: check for aliases
+
+                    # Loop through segs for that street full
+                    for seg in matching_segs:
+                        left_from = seg['left_from']
+                        left_to = seg['left_to']
+                        right_from = seg['right_from']
+                        right_to = seg['right_to']
+
+                        left_parity = parity_for_range(left_from, left_to)
+                        right_parity = parity_for_range(right_from, right_to)
+
+                        # Match to side of street based on parity
+                        check_from = None
+                        check_to = None
+
+                        if left_parity in [address_parity, 'B']:
+                            check_from = left_from
+                            check_to = left_to
+                            matching_side = 'L'
+                        elif right_parity in [address_parity, 'B']:
+                            check_from = right_from
+                            check_to = right_to
+                            matching_side = 'R'
+                        else:
+                            continue
+
+                        # If it's in range
+                        if check_from <= address_low <= check_to:
+                            # And it's a single address
+                            if address_high is None:
+                                matching_seg = seg
+                                break
+                            # Otherwise if it's a range address
+                            else:
+                                # If we already had a match, flag multiple matches
+                                if matching_seg:
+                                    seg_ids = sorted([x['seg_id'] for x in [matching_seg, seg]])
+                                    note = ','.join([str(x) for x in seg_ids])
+                                    had_warning('Range address matches multiple street segments', note=note)
+                                    
+                                # Check if the high address is greater than the street max
+                                if check_to < address_high:
+                                    # Otherwise, make the match and keep looking (in case
+                                    # it matches to multiple segments)
+                                    had_warning('High address out of street range')
+                                
+                                matching_seg = seg
+
+                    if matching_seg is None:
+                        # had_error('Out of street range')
+                        # should_add_parcel = False
+                        had_warning('Out of street range')
+                        should_add_parcel = True
+
+
+            '''
+            END STREET MATCH
+            '''
+
+            # Make parcel object
+            if should_add_parcel:
+                parcel = dict(address)
+                #Remove fields not in parcel tables:
+                parcel.pop('zip_code', None)
+                parcel.pop('zip_4', None)
+                #Add fields:
+                parcel.update({
+                    'parcel_id':        parcel_id,
+                    'source_object_id': object_id,
+                    'source_address':   source_address,
+                    'geom':             geometry,
+                })
+                parcels.append(parcel)
+                parcel_map[object_id] = parcel
+
+        # except ValueError as e:
+        #   # print('Parcel {}: {}'.format(parcel_id, e))
+        #   reason = str(e)
+        #   had_error(reason)
+
+        except Exception as e:
+            print('{}: Unhandled error'.format(source_parcel))
+            print(parcel_id)
+            print(traceback.format_exc())
+            raise e
+
+    print('Checking for duplicates...')
+
+    # Check for duplicate parcel IDs. Use source parcels for most results.
+    for source_parcel in source_parcels:
+        # Set object ID here so logging function works
+        object_id = source_parcel['objectid']
+        parcel_id = source_parcel['mapreg']
+        count = parcel_id_counts.get(parcel_id, 0)
+        if count > 1:
+            note = 'Parcel ID count: {}'.format(count)
+            had_warning('Duplicate parcel ID', note=note)
+
+    # Check for duplicate addresses. Use parcels since source parcels don't have
+    # an address.
+    for parcel in parcels:
+        # Set object ID here so logging function works
+        object_id = parcel['source_object_id']
+        street_address = parcel['street_address']
+        count = address_counts.get(street_address, 0)
+        if count > 1:
+            note = 'Address count: {}'.format(count)
+            had_warning('Duplicate address', note=note)
+
+    # Remember how many parcels we went through before we delete them all
+    parcel_count = len(parcels)
+
+    if WRITE_OUT:
+        print('Writing parcels...')
+        parcel_table.write(parcels, chunk_size=50000)
+
+        print('Writing parcel errors...')
+        errors = []
+        source_non_geom_fields = [x for x in source_fields if x != source_geom_field]
+
+        for level in ['error', 'warning']:
+            issue_map = error_map if level == 'error' else warning_map
+            for object_id, issues in issue_map.items():
+                for issue in issues:
+                    reason = issue['reason']
+                    note = issue['note']
+                    source_parcel = source_parcel_map[object_id]
+                    
+                    # Make error row
+                    error = {x: source_parcel[x] if source_parcel[x] is not None \
+                        else '' for x in source_non_geom_fields if x not in error_exempt_fields}
+
+                    # Make this work for integer fields
+                    if error['house'] == '':
+                        error['house'] = None
+                    if error['stcod'] == '':
+                        error['stcod'] = None
+
+                    error.update({
+                        'level':    level,
+                        'reason':   reason,
+                        'notes':    note,
+                    })
+
+                    errors.append(error)
+
+        parcel_error_table.write(errors, chunk_size=150000)
+        del errors
+
+        print('Writing parcel error polygons...')
+        error_polygons = []
+
+        for level in ['error', 'warning']:
+            issue_map = error_map if level == 'error' else warning_map
+
+            for object_id, issues in issue_map.items():
+                # If this object had a geometry error, skip
+                if object_id in bad_geom_parcels:
+                    continue
+
+                # Roll up reasons, notes
+                reasons = [x['reason'] for x in issues]
+                reasons_joined = '; '.join(sorted(reasons))
+                notes = [x['note'] for x in issues if x['note'] != '']
+                notes_joined = '; '.join(notes)
+                source_parcel = source_parcel_map[object_id]
+                
+                # Make error row
+                error_polygon = {x: source_parcel[x] if x is not None \
+                    else '' for x in source_fields if x not in error_exempt_fields}
+
+                # Add/clean up fields
+                if error_polygon['house'] == '':
+                    error_polygon['house'] = None
+                if error_polygon['stcod'] == '':
+                    error_polygon['stcod'] = None
+
+                error_polygon.update({
+                    'reasons':      reasons_joined,
+                    'reason_count': len(reasons),
+                    'notes':        notes_joined,
+                    # 'shape':        source_parcel[wkt_field],
+                })
+                error_polygons.append(error_polygon)
+
+    #    parcel_error_polygon_table.write(error_polygons, chunk_size=50000)
+
+        target_dsn = config['DATABASES']['engine']
+        target_user = target_dsn[target_dsn.index("//") + 2:target_dsn.index(":", target_dsn.index("//"))]
+        target_pw = target_dsn[target_dsn.index(":",target_dsn.index(target_user)) + 1:target_dsn.index("@")]
+        target_name = target_dsn[target_dsn.index("/", target_dsn.index("@")) + 1:]
+        target_conn = psycopg2.connect('dbname={db_name} user={db_user} password={db_pw} host=localhost'.format(db_name=target_name, db_user=target_user, db_pw=target_pw))
+        target_table_name = 'public.dor_parcel_error_polygon'
+        error_polygon_rows = etl.fromdicts(error_polygons)
+        error_polygon_rows.topostgis(target_conn, target_table_name)
+
+        del error_polygons
+
+        print('Creating indexes...')
+        parcel_table.create_index('street_address')
+        # TODO: index error tables?
+
+    #source_db.close()
+    db.close()
+
+    print('Finished in {} seconds'.format(datetime.now() - start))
+    print('Processed {} parcels'.format(parcel_count))
+    print('{} errors'.format(len(error_map)))
+    print('{} warnings'.format(len(warning_map)))
diff --git a/ais/engine/scripts/load_ng911_address_points.py b/ais/engine/scripts/load_ng911_address_points.py
index b9e6ea93..7cce791e 100644
--- a/ais/engine/scripts/load_ng911_address_points.py
+++ b/ais/engine/scripts/load_ng911_address_points.py
@@ -8,109 +8,109 @@
 import traceback
 from pprint import pprint
 
+def main():
 
-print('Starting...')
-start = datetime.now()
+	print('Starting...')
+	start = datetime.now()
 
-"""SET UP"""
+	"""SET UP"""
 
-config = app.config
-source_def = config['BASE_DATA_SOURCES']['ng911_address_points']
-source_db = datum.connect(config['DATABASES'][source_def['db']])
-source_table = source_db[source_def['table']]
+	config = app.config
+	source_def = config['BASE_DATA_SOURCES']['ng911_address_points']
+	source_db = datum.connect(config['DATABASES'][source_def['db']])
+	source_table = source_db[source_def['table']]
 
-source_geom_field = source_table.geom_field
-field_map = source_def['field_map']
-source_where = source_def['where']
+	source_geom_field = source_table.geom_field
+	field_map = source_def['field_map']
+	source_where = source_def['where']
 
-db = datum.connect(config['DATABASES']['engine'])
-ng911_table = db['ng911_address_point']
+	db = datum.connect(config['DATABASES']['engine'])
+	ng911_table = db['ng911_address_point']
 
-Parser = config['PARSER']
-parser = Parser()
+	Parser = config['PARSER']
+	parser = Parser()
 
-"""MAIN"""
+	"""MAIN"""
 
-# Get field names
-source_fields = list(field_map.values())
-source_guid_field = field_map['guid']
-source_address_field = field_map['source_address']
-source_placement_type_field = field_map['placement_type']
+	# Get field names
+	source_fields = list(field_map.values())
+	source_guid_field = field_map['guid']
+	source_address_field = field_map['source_address']
+	source_placement_type_field = field_map['placement_type']
 
-print('Dropping index...')
-ng911_table.drop_index('street_address')
+	print('Dropping index...')
+	ng911_table.drop_index('street_address')
 
-print('Deleting existing NG911 address points...')
-ng911_table.delete()
+	print('Deleting existing NG911 address points...')
+	ng911_table.delete()
 
-print('Reading NG911 address points from source...')
-source_address_points = source_table.read(fields=source_fields, where=source_where)
-address_points = []
+	print('Reading NG911 address points from source...')
+	source_address_points = source_table.read(fields=source_fields, where=source_where)
+	address_points = []
 
-for i, source_address_point in enumerate(source_address_points):
-	try:
-		if i % 100000 == 0:
-			print(i)
-
-		# Get attrs
-		guid = source_address_point[source_guid_field]
-		placement_type = source_address_point[source_placement_type_field]
-		location = source_address_point[source_address_field]
-		geometry = source_address_point[source_geom_field]
-
-		# Handle address
-		source_address = location.strip()
-		
-		if source_address in (None, ''):
-			raise ValueError('No address')
-
-		# Parse
+	for i, source_address_point in enumerate(source_address_points):
 		try:
-			parsed = parser.parse(source_address)
-			comps = parsed['components']
-		except:
-			raise ValueError('Could not parse')
-		address = Address(parsed)
-		street_address = comps['output_address']
-
-		address_point = {
-			'guid': guid,
-			'source_address': source_address,
-			'placement_type': placement_type,
-			'geom': geometry,
-			'address_low': comps['address']['low_num'],
-			'address_low_suffix': comps['address']['addr_suffix'] or '',
-			'address_high': comps['address']['high_num_full'],
-			'street_predir': comps['street']['predir'] or '',
-			'street_name': comps['street']['name'],
-			'street_suffix': comps['street']['suffix'] or '',
-			'street_postdir': comps['street']['postdir'] or '',
-			'unit_num': comps['address_unit']['unit_num'] or '',
-			'unit_type': comps['address_unit']['unit_type'] or '',
-			'street_address': street_address,
-		}
-		address_points.append(address_point)
-
-	except ValueError as e:
-		# FEEDBACK
-		print("value error...")
-		pass
-
-	except Exception as e:
-		print('Unhandled exception on {}'.format(source_address))
-		print(traceback.format_exc())
-		# sys.exit()
-
-print('Writing {} NG911 address points...'.format(len(address_points)))
-ng911_table.write(address_points)
-
-print('Creating index...')
-ng911_table.create_index('street_address')
-
-'''
-FINISH
-'''
-
-db.close()
-print('Finished in {} seconds'.format(datetime.now() - start))
-
+			if i % 100000 == 0:
+				print(i)
+
+			# Get attrs
+			guid = source_address_point[source_guid_field]
+			placement_type = source_address_point[source_placement_type_field]
+			location = source_address_point[source_address_field]
+			geometry = source_address_point[source_geom_field]
+
+			# Handle address
+			source_address = location.strip()
+
+			if source_address in (None, ''):
+				raise ValueError('No address')
+
+			# Parse
+			try:
+				parsed = parser.parse(source_address)
+				comps = parsed['components']
+			except:
+				raise ValueError('Could not parse')
+			address = Address(parsed)
+			street_address = comps['output_address']
+
+			address_point = {
+				'guid': guid,
+				'source_address': source_address,
+				'placement_type': placement_type,
+				'geom': geometry,
+				'address_low': comps['address']['low_num'],
+				'address_low_suffix': comps['address']['addr_suffix'] or '',
+				'address_high': comps['address']['high_num_full'],
+				'street_predir': comps['street']['predir'] or '',
+				'street_name': comps['street']['name'],
+				'street_suffix': comps['street']['suffix'] or '',
+				'street_postdir': comps['street']['postdir'] or '',
+				'unit_num': comps['address_unit']['unit_num'] or '',
+				'unit_type': comps['address_unit']['unit_type'] or '',
+				'street_address': street_address,
+			}
+			address_points.append(address_point)
+
+		except ValueError as e:
+			# FEEDBACK
+			print("value error...")
+			pass
+
+		except Exception as e:
+			print('Unhandled exception on {}'.format(source_address))
+			print(traceback.format_exc())
+			# sys.exit()
+
+	print('Writing {} NG911 address points...'.format(len(address_points)))
+	ng911_table.write(address_points)
+
+	print('Creating index...')
+	ng911_table.create_index('street_address')
+
+	'''
+	FINISH
+	'''
+
+	db.close()
+	print('Finished in {} seconds'.format(datetime.now() - start))
\ No newline at end of file
diff --git a/ais/engine/scripts/load_opa_properties.py b/ais/engine/scripts/load_opa_properties.py
index 1c621219..9e496fe2 100644
--- a/ais/engine/scripts/load_opa_properties.py
+++ b/ais/engine/scripts/load_opa_properties.py
@@ -8,117 +8,118 @@
 import traceback
 from pprint import pprint
 
-
-print('Starting...')
-start = datetime.now()
-
-"""SET UP"""
-
-config = app.config
-source_def = config['BASE_DATA_SOURCES']['properties']
-source_db = datum.connect(config['DATABASES'][source_def['db']])
-ais_source_db = datum.connect(config['DATABASES']['gis'])
-source_table = source_db[source_def['table']]
-field_map = source_def['field_map']
-
-owner_source_def = config['BASE_DATA_SOURCES']['opa_owners']
-owner_table_name = owner_source_def['table']
-
-db = datum.connect(config['DATABASES']['engine'])
-prop_table = db['opa_property']
-
-Parser = config['PARSER']
-parser = Parser()
-
-"""MAIN"""
-
-# Get field names
-source_fields = list(field_map.values())
-source_tencode_field = field_map['tencode']
-source_account_num_field = field_map['account_num']
-source_address_field = field_map['source_address']
-
-print('Dropping index...')
-prop_table.drop_index('street_address')
-
-print('Deleting existing properties...')
-prop_table.delete()
-
-print('Reading owners from source...')
-owner_stmt = """
-select account_num, owners from {}
-""".format(owner_table_name)
-owner_rows = ais_source_db.execute(owner_stmt)
-owner_map = {x[0]: x[1] for x in owner_rows}
-
-print('Reading properties from source...')
-source_props = source_table.read(fields=source_fields)
-props = []
-
-for i, source_prop in enumerate(source_props):
-	try:
-		if i % 100000 == 0:
-			print(i)
-
-		# Get attrs
-		tencode = source_prop[source_tencode_field]
-		account_num = source_prop[source_account_num_field]
-		location = source_prop[source_address_field]
-
-		# Handle address
-		source_address = location.strip()
-
-		# Parse
-		try:
-			parsed = parser.parse(source_address)
-			comps = parsed['components']
-		except:
-			raise ValueError('Could not parse')
-		address = Address(parsed)
-		street_address = comps['output_address']
-
-		# Owners
-		try:
-			owners = owner_map[account_num]
-		except KeyError:
-			owners = ''
-
-		prop = {
-			'account_num': account_num,
-			'source_address': source_address,
-			'tencode': tencode,
-			'owners': owners,
-			'address_low': comps['address']['low_num'],
-			'address_low_suffix': comps['address']['addr_suffix'] or '',
-			'address_high': comps['address']['high_num_full'],
-			'street_predir': comps['street']['predir'] or '',
-			'street_name': comps['street']['name'],
-			'street_suffix': comps['street']['suffix'] or '',
-			'street_postdir': comps['street']['postdir'] or '',
-			'unit_num': comps['address_unit']['unit_num'] or '',
-			'unit_type': comps['address_unit']['unit_type'] or '',
-			'street_address': street_address,
-		}
-		props.append(prop)
-
-	except ValueError as e:
-		# FEEDBACK
-		pass
-
-	except Exception as e:
-		print('Unhandled exception on {}'.format(source_address))
-		print(traceback.format_exc())
-		# sys.exit()
-
-print('Writing properties...')
-prop_table.write(props)
-
-print('Creating index...')
-prop_table.create_index('street_address')
-
-'''
-FINISH
-'''
-
-db.close()
-print('Finished in {} seconds'.format(datetime.now() - start))
+def main():
+
+    print('Starting...')
+    start = datetime.now()
+
+    """SET UP"""
+
+    config = app.config
+    source_def = config['BASE_DATA_SOURCES']['properties']
+    source_db = datum.connect(config['DATABASES'][source_def['db']])
+    ais_source_db = datum.connect(config['DATABASES']['gis'])
+    source_table = source_db[source_def['table']]
+    field_map = source_def['field_map']
+
+    owner_source_def = config['BASE_DATA_SOURCES']['opa_owners']
+    owner_table_name = owner_source_def['table']
+
+    db = datum.connect(config['DATABASES']['engine'])
+    prop_table = db['opa_property']
+
+    Parser = config['PARSER']
+    parser = Parser()
+
+    """MAIN"""
+
+    # Get field names
+    source_fields = list(field_map.values())
+    source_tencode_field = field_map['tencode']
+    source_account_num_field = field_map['account_num']
+    source_address_field = field_map['source_address']
+
+    print('Dropping index...')
+    prop_table.drop_index('street_address')
+
+    print('Deleting existing properties...')
+    prop_table.delete()
+
+    print('Reading owners from source...')
+    owner_stmt = """
+    select account_num, owners from {}
+    """.format(owner_table_name)
+    owner_rows = ais_source_db.execute(owner_stmt)
+    owner_map = {x[0]: x[1] for x in owner_rows}
+
+    print('Reading properties from source...')
+    source_props = source_table.read(fields=source_fields)
+    props = []
+
+    for i, source_prop in enumerate(source_props):
+        try:
+            if i % 100000 == 0:
+                print(i)
+
+            # Get attrs
+            tencode = source_prop[source_tencode_field]
+            account_num = source_prop[source_account_num_field]
+            location = source_prop[source_address_field]
+
+            # Handle address
+            source_address = location.strip()
+
+            # Parse
+            try:
+                parsed = parser.parse(source_address)
+                comps = parsed['components']
+            except:
+                raise ValueError('Could not parse')
+            address = Address(parsed)
+            street_address = comps['output_address']
+
+            # Owners
+            try:
+                owners = owner_map[account_num]
+            except KeyError:
+                owners = ''
+
+            prop = {
+                'account_num': account_num,
+                'source_address': source_address,
+                'tencode': tencode,
+                'owners': owners,
+                'address_low': comps['address']['low_num'],
+                'address_low_suffix': comps['address']['addr_suffix'] or '',
+                'address_high': comps['address']['high_num_full'],
+                'street_predir': comps['street']['predir'] or '',
+                'street_name': comps['street']['name'],
+                'street_suffix': comps['street']['suffix'] or '',
+                'street_postdir': comps['street']['postdir'] or '',
+                'unit_num': comps['address_unit']['unit_num'] or '',
+                'unit_type': comps['address_unit']['unit_type'] or '',
+                'street_address': street_address,
+            }
+            props.append(prop)
+
+        except ValueError as e:
+            # FEEDBACK
+            pass
+
+        except Exception as e:
+            print('Unhandled exception on {}'.format(source_address))
+            print(traceback.format_exc())
+            raise e
+
+    print('Writing properties...')
+    prop_table.write(props)
+
+    print('Creating index...')
+    prop_table.create_index('street_address')
+
+    '''
+    FINISH
+    '''
+
+    db.close()
+    print('Finished in {} seconds'.format(datetime.now() - start))
diff --git a/ais/engine/scripts/load_pwd_parcels.py b/ais/engine/scripts/load_pwd_parcels.py
index 91f046a6..a76af31c 100644
--- a/ais/engine/scripts/load_pwd_parcels.py
+++ b/ais/engine/scripts/load_pwd_parcels.py
@@ -9,142 +9,144 @@
 from pprint import pprint
 import traceback
 
-
-start = datetime.now()
-print('Starting...')
-
-
-"""SET UP"""
-
-config = app.config
-db = datum.connect(config['DATABASES']['engine'])
-parcel_table = db['pwd_parcel']
-parcel_geom_field = parcel_table.geom_field
-
-
-source_def = config['BASE_DATA_SOURCES']['parcels']['pwd']
-source_db_name = source_def['db']
-source_db_url = config['DATABASES'][source_db_name]
-source_db = datum.connect(source_db_url)
-source_field_map = source_def['field_map']
-source_table_name = source_def['table']
-source_table = source_db[source_table_name]
-source_geom_field = source_table.geom_field
-
-# Read in OPA account nums and addresses
-opa_source_def = config['BASE_DATA_SOURCES']['opa_owners']
-opa_source_db_name = opa_source_def['db']
-opa_source_db_url = config['DATABASES'][opa_source_db_name]
-opa_source_db = datum.connect(opa_source_db_url)
-opa_source_table = opa_source_def['table']
-opa_field_map = opa_source_def['field_map']
-opa_rows = source_db[opa_source_table].read()
-opa_map = {x[opa_field_map['account_num']]: x[opa_field_map['street_address']] \
-	for x in opa_rows if x['account_num']}
-
-# Make a list of non-unique addresses in PWD parcels. If a parcel has one of 
-# these addresses, use OPA address instead. Case: 421 S 10TH ST appears three
-# times in parcels but should have unit nums according to OPA.
-print('Loading non-unique parcel addresses...')
-ambig_stmt = '''
-	select address
-	from {}
-	group by address
-	having address is not null and count(*) > 1
-'''.format(source_table_name)
-source_db._c.execute(ambig_stmt)
-ambig_rows = source_db._c.fetchall()
-ambig_addresses = set([x[0] for x in ambig_rows])
-
-
-"""MAIN"""
-
-# # Set up logging
-# LOG_COLS = [
-# 	'parcel_id',
-# 	'source_address',
-# 	'error',
-# ]
-# parent_dir = os.path.abspath(os.path.join(__file__, os.pardir))
-# log = open(parent_dir + '/log/load_pwd_parcels.log', 'w', newline='')
-# log_writer = csv.writer(log)
-# log_writer.writerow(LOG_COLS)
-
-print('Dropping indexes...')
-parcel_table.drop_index('street_address')
-
-print('Deleting existing parcels...')
-parcel_table.delete()
-
-# Get field names
-source_parcel_id_field = source_field_map['parcel_id']
-source_address_field = source_field_map['source_address']
-source_brt_id_field = source_field_map['source_brt_id']
-
-# Read parcels
-print('Reading parcels from source...')
-source_fields = list(source_field_map.values())
-source_parcels = source_table.read(fields=source_fields)
-parcels = []
-
-# Loop over source parcels
-for i, source_parcel in enumerate(source_parcels):
-	try:
-		if i % 50000 == 0:
-			print(i)
-
-		# Get attrs
-		parcel_id = source_parcel[source_parcel_id_field]
-		geometry = source_parcel[source_geom_field]
-		source_brt_id = source_parcel[source_brt_id_field]
-		source_address = source_parcel[source_address_field]
-		
-		if source_address in (None, ''):
-			raise ValueError('No address')
-
-		# If there are multiple parcels with the same address, get OPA
-		# address
-		if source_address in ambig_addresses and source_brt_id in opa_map:
-			source_address = opa_map[source_brt_id]
-
-		try:
-			address = Address(source_address)
-		except:
-			# raise ValueError('Could not parse')
-			raise ValueError('Could not parse: {}'.format(source_address))
-
-		parcel = dict(address)
-		# Remove fields not in parcel tables:
-		parcel.pop('zip_code', None)
-		parcel.pop('zip_4', None)
-		# Add fields:
-		parcel.update({
-			parcel_geom_field:	geometry,
-			'parcel_id':		parcel_id,
-		})
-		parcels.append(parcel)
-
-		# FEEDBACK
-		# if source_address != parcel.street_address:
-		# 	print('{} => {}'.format(source_address, parcel.street_address))
-
-	except ValueError as e:
-		print('Parcel {}: {}'.format(parcel_id, e))
-		# log_writer.writerow([parcel_id, source_address, e])
-
-	except Exception as e:
-		print('{}: Unhandled error'.format(source_parcel))
-		print(traceback.format_exc())
-
-print('Writing parcels...')
-parcel_table.write(parcels, chunk_size=50000)
-# db.save()
-
-print('Creating indexes...')
-parcel_table.create_index('street_address')
-
-#source_db.close()
-db.close()
-# log.close()
-print('Finished in {} seconds'.format(datetime.now() - start))
-print('Wrote {} parcels'.format(len(parcels)))
+def main():
+    start = datetime.now()
+    print('Starting...')
+
+
+    """SET UP"""
+
+    config = app.config
+    db = datum.connect(config['DATABASES']['engine'])
+    parcel_table = db['pwd_parcel']
+    parcel_geom_field = parcel_table.geom_field
+
+
+    source_def = config['BASE_DATA_SOURCES']['parcels']['pwd']
+    source_db_name = source_def['db']
+    source_db_url = config['DATABASES'][source_db_name]
+    source_db = datum.connect(source_db_url)
+    source_field_map = source_def['field_map']
+    source_table_name = source_def['table']
+    source_table = source_db[source_table_name]
+    source_geom_field = source_table.geom_field
+
+    # Read in OPA account nums and addresses
+    opa_source_def = config['BASE_DATA_SOURCES']['opa_owners']
+    opa_source_db_name = opa_source_def['db']
+    opa_source_db_url = config['DATABASES'][opa_source_db_name]
+    opa_source_db = datum.connect(opa_source_db_url)
+    opa_source_table = opa_source_def['table']
+    opa_field_map = opa_source_def['field_map']
+    opa_rows = source_db[opa_source_table].read()
+    opa_map = {x[opa_field_map['account_num']]: x[opa_field_map['street_address']] \
+        for x in opa_rows if x['account_num']}
+
+    # Make a list of non-unique addresses in PWD parcels. If a parcel has one of 
+    # these addresses, use OPA address instead. Case: 421 S 10TH ST appears three
+    # times in parcels but should have unit nums according to OPA.
+    print('Loading non-unique parcel addresses...')
+    ambig_stmt = '''
+        select address
+        from {}
+        group by address
+        having address is not null and count(*) > 1
+    '''.format(source_table_name)
+    source_db._c.execute(ambig_stmt)
+    ambig_rows = source_db._c.fetchall()
+    ambig_addresses = set([x[0] for x in ambig_rows])
+
+
+    """MAIN"""
+
+    # # Set up logging
+    # LOG_COLS = [
+    # 	'parcel_id',
+    # 	'source_address',
+    # 	'error',
+    # ]
+    # parent_dir = os.path.abspath(os.path.join(__file__, os.pardir))
+    # log = open(parent_dir + '/log/load_pwd_parcels.log', 'w', newline='')
+    # log_writer = csv.writer(log)
+    # log_writer.writerow(LOG_COLS)
+
+    print('Dropping indexes...')
+    parcel_table.drop_index('street_address')
+
+    print('Deleting existing parcels...')
+    parcel_table.delete()
+
+    # Get field names
+    source_parcel_id_field = source_field_map['parcel_id']
+    source_address_field = source_field_map['source_address']
+    source_brt_id_field = source_field_map['source_brt_id']
+
+    # Read parcels
+    print('Reading parcels from source...')
+    source_fields = list(source_field_map.values())
+    source_parcels = source_table.read(fields=source_fields)
+    parcels = []
+
+    # Loop over source parcels
+    for i, source_parcel in enumerate(source_parcels):
+        try:
+            if i % 50000 == 0:
+                print(i)
+
+            # Get attrs
+            parcel_id = source_parcel[source_parcel_id_field]
+            geometry = source_parcel[source_geom_field]
+            source_brt_id = source_parcel[source_brt_id_field]
+            source_address = source_parcel[source_address_field]
+            
+            if source_address in (None, ''):
+                raise ValueError('No address')
+
+            # If there are multiple parcels with the same address, get OPA
+            # address
+            if source_address in ambig_addresses and source_brt_id in opa_map:
+                source_address = opa_map[source_brt_id]
+
+            try:
+                address = Address(source_address)
+            except:
+                # raise ValueError('Could not parse')
+                raise ValueError('Could not parse: {}'.format(source_address))
+
+            parcel = dict(address)
+            # Remove fields not in parcel tables:
+            parcel.pop('zip_code', None)
+            parcel.pop('zip_4', None)
+            # Add fields:
+            parcel.update({
+                parcel_geom_field:	geometry,
+                'parcel_id':		parcel_id,
+            })
+            parcels.append(parcel)
+
+            # FEEDBACK
+            # if source_address != parcel.street_address:
+            # 	print('{} => {}'.format(source_address, parcel.street_address))
+
+        except ValueError as e:
+            #print('Parcel {}: {}'.format(parcel_id, e))
+            # log_writer.writerow([parcel_id, source_address, e])
+                    pass
+
+        except Exception as e:
+            print('{}: Unhandled error'.format(source_parcel))
+            print(traceback.format_exc())
+            raise e
+
+    print('Writing parcels...')
+    parcel_table.write(parcels, chunk_size=50000)
+    # db.save()
+
+    print('Creating indexes...')
+    parcel_table.create_index('street_address')
+
+    #source_db.close()
+    db.close()
+    # log.close()
+    print('Finished in {} seconds'.format(datetime.now() - start))
+    print('Wrote {} parcels'.format(len(parcels)))
diff --git a/ais/engine/scripts/load_service_areas.py b/ais/engine/scripts/load_service_areas.py
index 38eabe8c..8d21bbfa 100644
--- a/ais/engine/scripts/load_service_areas.py
+++ b/ais/engine/scripts/load_service_areas.py
@@ -3,30 +3,8 @@
 from shapely.wkt import loads
 import datum
 import petl as etl
-import cx_Oracle
 import geopetl
 from ais import app
-# DEV
-import traceback
-from pprint import pprint
-
-
-start = datetime.now()
-print('Starting...')
-
-"""SET UP"""
-
-config = app.config
-db = datum.connect(config['DATABASES']['engine'])
-poly_table = db['service_area_polygon']
-line_single_table = db['service_area_line_single']
-line_dual_table = db['service_area_line_dual']
-point_table = db['service_area_point']
-layer_table = db['service_area_layer']
-layers = config['SERVICE_AREAS']['layers']
-geom_field = 'shape'
-# sde_srid = 2272
-WRITE_OUT = True
 
 """
 TRANSFORMS
@@ -40,218 +18,245 @@
 """
 
 def convert_to_integer(row, value_field):
-	value = row[value_field]
-	int_value = int(value)
-	row[value_field] = int_value
-	return row
+    value = row[value_field]
+    int_value = int(value)
+    row[value_field] = int_value
+    return row
 
 def remove_whitespace(row, value_field):
-	value = row[value_field]
-	no_whitespace = value.replace(' ', '')
-	row[value_field] = no_whitespace
-	return row
-
-print('\n** SERVICE AREA LAYERS **')
-
-if WRITE_OUT:
-	print('Deleting existing service area layers...')
-	layer_table.delete()
-
-	print('Writing service area layers...')
-	keys = ['layer_id', 'name', 'description']
-	layer_rows = [{key: layer[key] for key in keys} for layer in layers]
-	db['service_area_layer'].write(layer_rows)
-
-print('\n** SERVICE AREAS **')
-
-if WRITE_OUT:
-	print('Dropping indexes...')
-	line_single_table.drop_index('seg_id')
-	line_dual_table.drop_index('seg_id')
-
-	print('Deleting existing service area polys...')
-	poly_table.delete()
-	print('Deleting existing service area single-value lines...')
-	line_single_table.delete()
-	print('Deleting existing service area dual-value lines...')
-	line_dual_table.delete()
-	print('Deleting existing service area points...')
-	point_table.delete()
-
-polys = []
-line_singles = []
-line_duals = []
-points = []
-
-print('Reading service areas...')
-# wkt_field = geom_field + '_wkt'
-
-for layer in layers:
-	layer_id = layer['layer_id']
-	print('  - {}'.format(layer_id))
-	sources = layer['sources']
-
-	# Check for conflicting source types
-	if 'line_single' in sources and 'line_dual' in sources:
-		raise Exception('Too many line sources for {}'.format(layer_id))
-
-	for source_type, source in sources.items():
-		# Connect to DB
-		source_db_name = source['db']
-		try:
-			source_db = datum.connect(config['DATABASES'][source_db_name])
-		except KeyError:
-			print('Database {} not found'.format(layer_id))
-			continue
-
-		source_table_name = source['table']
-		source_table = source_db[source_table_name]
-		# import pdb; pdb.set_trace()
-		source_geom_field = source_table.geom_field
-		# If no object ID field is specified, default to `objectid`.
-		object_id_field = source.get('object_id_field', 'objectid')
-		
-		# If there are transforms, reference their functions
-		transforms = source.get('transforms', [])
-		transform_map = {}
-		for transform in transforms:
-			f = getattr(sys.modules[__name__], transform)
-			transform_map[transform] = f
-		
-		# POLYGON
-		if source_type == 'polygon':
-			value_field = source['value_field']
-			source_fields = [value_field, object_id_field]
-			source_rows = source_table.read(fields=source_fields, \
-				geom_field=geom_field)
-
-			for i, source_row in enumerate(source_rows):
-				# Transform if necessary
-				for f in transform_map.values():
-					source_row = f(source_row, value_field)
-				
-				value = source_row[value_field]
-
-				# Remove excess whitespace from strings. This isn't a transform
-				# because we want to do it to all strings.
-				if isinstance(value, str):
-					value = value.strip()
-
-				poly = {
-					'layer_id': 			layer_id,
-					'source_object_id': 	source_row[object_id_field],
-					'value': 				value or '',
-					'geom': 				source_row[source_geom_field],
-				}
-				polys.append(poly)
-
-		# LINE SINGLE
-		if source_type == 'line_single':
-			value_field = source['value_field']
-			seg_id_field = source['seg_id_field']
-			source_fields = [value_field, object_id_field, seg_id_field]
-			source_rows = source_table.read(fields=source_fields)
-
-			for i, source_row in enumerate(source_rows):
-				# Transform if necessary
-				for f in transform_map.values():
-					source_row = f(source_row, value_field)
-
-				value = source_row[value_field]
-
-				# Remove excess whitespace from strings. This isn't a transform
-				# because we want to do it to all strings.
-				if isinstance(value, str):
-					value = value.strip()
-
-				line_single = {
-					'layer_id': 			layer_id,
-					'source_object_id': 	source_row[object_id_field],
-					'seg_id':				source_row[seg_id_field],
-					'value': 				value or '',
-				}
-				line_singles.append(line_single)
-
-		# LINE DUAL
-		if source_type == 'line_dual':
-			left_value_field = source['left_value_field']
-			right_value_field = source['right_value_field']
-			seg_id_field = source['seg_id_field']
-			source_fields = [
-				left_value_field,
-				right_value_field,
-				object_id_field,
-				seg_id_field
-			]
-			source_rows = source_table.read(fields=source_fields)
-
-			for i, source_row in enumerate(source_rows):
-				# Transform if necessary
-				for f in transform_map.values():
-					source_row = f(source_row, value_field)
-
-				left_value = source_row[left_value_field]
-				right_value = source_row[right_value_field]
-
-				line_dual = {
-					'layer_id': 			layer_id,
-					'source_object_id': 	source_row[object_id_field],
-					'seg_id':				source_row[seg_id_field],
-					'left_value': 			left_value or '',
-					'right_value': 			right_value or '',
-				}
-				line_duals.append(line_dual)
-
-
-		# POINT
-		if source_type == 'point':
-			value_field = source['value_field']
-			seg_id_field = source.get('seg_id_field', '')
-			method = source.get('method', '')
-			source_fields = [value_field, object_id_field] if method != 'seg_id' else [value_field]
-			if seg_id_field:
-				source_fields.append(seg_id_field)
-			source_rows = source_table.read(fields=source_fields, \
-											geom_field=geom_field) if method != 'seg_id' else source_table.read(fields=source_fields)
-
-			for i, source_row in enumerate(source_rows):
-				# Transform if necessary
-				for f in transform_map.values():
-					source_row = f(source_row, value_field)
-
-				value = source_row[value_field]
-
-				# Remove excess whitespace from strings. This isn't a transform
-				# because we want to do it to all strings.
-				if isinstance(value, str):
-					value = value.strip()
-
-				point = {
-					'layer_id': layer_id,
-					'source_object_id': source_row[object_id_field] if method != 'seg_id' else None,
-					'seg_id': source_row[seg_id_field] if seg_id_field else None,
-					'value': value or '',
-					'geom': source_row[source_geom_field] if method != 'seg_id' else loads('POINT(0 0)').wkt,
-				}
-				points.append(point)
-
-if WRITE_OUT:
-	print('Writing service area polygons...')
-	poly_table.write(polys)
-
-	print('Writing service area single-value lines...')
-	line_single_table.write(line_singles)
-
-	print('Writing service area line dual-value lines...')
-	line_dual_table.write(line_duals)
-
-	print('Writing service area points...')
-	point_table.write(points)
-
-	print('Creating indexes...')
-	line_single_table.create_index('seg_id')
-	line_dual_table.create_index('seg_id')
-
-#source_db.close()
-db.close()
-print('Finished in {} seconds'.format(datetime.now() - start))
+    value = row[value_field]
+    no_whitespace = value.replace(' ', '')
+    row[value_field] = no_whitespace
+    return row
+
+def main():
+    start = datetime.now()
+    print('Starting...')
+
+    """SET UP"""
+
+    config = app.config
+    db = datum.connect(config['DATABASES']['engine'])
+    poly_table = db['service_area_polygon']
+    line_single_table = db['service_area_line_single']
+    line_dual_table = db['service_area_line_dual']
+    point_table = db['service_area_point']
+    layer_table = db['service_area_layer']
+    layers = config['SERVICE_AREAS']['layers']
+    geom_field = 'shape'
+    # sde_srid = 2272
+    WRITE_OUT = True
+
+    print('\n** SERVICE AREA LAYERS **')
+
+    if WRITE_OUT:
+        print('Deleting existing service area layers...')
+        layer_table.delete()
+
+        print('Writing service area layers...')
+        keys = ['layer_id', 'name', 'description']
+        layer_rows = [{key: layer[key] for key in keys} for layer in layers]
+        db['service_area_layer'].write(layer_rows)
+
+    print('\n** SERVICE AREAS **')
+
+    if WRITE_OUT:
+        print('Dropping indexes...')
+        line_single_table.drop_index('seg_id')
+        line_dual_table.drop_index('seg_id')
+
+        print('Deleting existing service area polys...')
+        poly_table.delete()
+        print('Deleting existing service area single-value lines...')
+        line_single_table.delete()
+        print('Deleting existing service area dual-value lines...')
+        line_dual_table.delete()
+        print('Deleting existing service area points...')
+        point_table.delete()
+
+    polys = []
+    line_singles = []
+    line_duals = []
+    points = []
+    empty_geometry_fail = False
+
+    print('Reading service areas...')
+
+    for layer in layers:
+        layer_id = layer['layer_id']
+        print('  - {}'.format(layer_id))
+        sources = layer['sources']
+
+        # Check for conflicting source types
+        if 'line_single' in sources and 'line_dual' in sources:
+            raise Exception('Too many line sources for {}'.format(layer_id))
+
+        for source_type, source in sources.items():
+            # Connect to DB
+            source_db_name = source['db']
+            try:
+                source_db = datum.connect(config['DATABASES'][source_db_name])
+            except KeyError:
+                print('Database {} not found'.format(layer_id))
+                continue
+
+            source_table_name = source['table']
+            source_table = source_db[source_table_name]
+            source_geom_field = source_table.geom_field
+            # If no object ID field is specified, default to `objectid`.
+            object_id_field = source.get('object_id_field', 'objectid')
+            
+            # If there are transforms, reference their functions
+            transforms = source.get('transforms', [])
+            transform_map = {}
+            for transform in transforms:
+                f = getattr(sys.modules[__name__], transform)
+                transform_map[transform] = f
+            
+            # POLYGON
+            if source_type == 'polygon':
+                value_field = source['value_field']
+                source_fields = [value_field, object_id_field]
+                source_rows = source_table.read(fields=source_fields, 
+                                                geom_field=geom_field)
+
+                for i, source_row in enumerate(source_rows):
+                    # Transform if necessary
+                    for f in transform_map.values():
+                        source_row = f(source_row, value_field)
+                    
+                    value = source_row[value_field]
+
+                    # Remove excess whitespace from strings. This isn't a transform
+                    # because we want to do it to all strings.
+                    if isinstance(value, str):
+                        value = value.strip()
+
+                    # Alert on null geometries 
+ 
+                    if not source_row[source_geom_field]:
+                        print(f'Empty geometry found in {source_db_name}.{source_table_name}!')
+                        print(f'Source row: {source_row}')
+                        empty_geometry_fail = True
+                    
+                    poly = {
+                        'layer_id': 			layer_id,
+                        'source_object_id': 	source_row[object_id_field],
+                        'value': 				value or '',
+                        'geom': 				source_row[source_geom_field],
+                    }
+                    polys.append(poly)
+
+
+            # LINE SINGLE
+            if source_type == 'line_single':
+                value_field = source['value_field']
+                seg_id_field = source['seg_id_field']
+                source_fields = [value_field, object_id_field, seg_id_field]
+                source_rows = source_table.read(fields=source_fields)
+
+                for i, source_row in enumerate(source_rows):
+                    # Transform if necessary
+                    for f in transform_map.values():
+                        source_row = f(source_row, value_field)
+
+                    value = source_row[value_field]
+
+                    # Remove excess whitespace from strings. This isn't a transform
+                    # because we want to do it to all strings.
+                    if isinstance(value, str):
+                        value = value.strip()
+
+                    line_single = {
+                        'layer_id': 			layer_id,
+                        'source_object_id': 	source_row[object_id_field],
+                        'seg_id':				source_row[seg_id_field],
+                        'value': 				value or '',
+                    }
+                    line_singles.append(line_single)
+
+            # LINE DUAL
+            if source_type == 'line_dual':
+                left_value_field = source['left_value_field']
+                right_value_field = source['right_value_field']
+                seg_id_field = source['seg_id_field']
+                source_fields = [
+                    left_value_field,
+                    right_value_field,
+                    object_id_field,
+                    seg_id_field
+                ]
+                source_rows = source_table.read(fields=source_fields)
+
+                for i, source_row in enumerate(source_rows):
+                    # Transform if necessary
+                    for f in transform_map.values():
+                        source_row = f(source_row, value_field)
+
+                    left_value = source_row[left_value_field]
+                    right_value = source_row[right_value_field]
+
+                    line_dual = {
+                        'layer_id': 			layer_id,
+                        'source_object_id': 	source_row[object_id_field],
+                        'seg_id':				source_row[seg_id_field],
+                        'left_value': 			left_value or '',
+                        'right_value': 			right_value or '',
+                    }
+                    line_duals.append(line_dual)
+
+
+            # POINT
+            if source_type == 'point':
+                value_field = source['value_field']
+                seg_id_field = source.get('seg_id_field', '')
+                method = source.get('method', '')
+                source_fields = [value_field, object_id_field] if method != 'seg_id' else [value_field]
+                if seg_id_field:
+                    source_fields.append(seg_id_field)
+                source_rows = source_table.read(fields=source_fields, \
+                                                geom_field=geom_field) if method != 'seg_id' else source_table.read(fields=source_fields)
+
+                for i, source_row in enumerate(source_rows):
+                    # Transform if necessary
+                    for f in transform_map.values():
+                        source_row = f(source_row, value_field)
+
+                    value = source_row[value_field]
+
+                    # Remove excess whitespace from strings. This isn't a transform
+                    # because we want to do it to all strings.
+                    if isinstance(value, str):
+                        value = value.strip()
+
+                    point = {
+                        'layer_id': layer_id,
+                        'source_object_id': source_row[object_id_field] if method != 'seg_id' else None,
+                        'seg_id': source_row[seg_id_field] if seg_id_field else None,
+                        'value': value or '',
+                        'geom': source_row[source_geom_field] if method != 'seg_id' else loads('POINT(0 0)').wkt,
+                    }
+                    points.append(point)
+    if empty_geometry_fail:
+        raise AssertionError('Encountered earlier empty geometry, that shouldnt happen!! Full row will be logged in earlier logs for this engine build script. Failing out..')
+
+
+    if WRITE_OUT:
+        print('Writing service area polygons...')
+        poly_table.write(polys)
+
+        print('Writing service area single-value lines...')
+        line_single_table.write(line_singles)
+
+        print('Writing service area line dual-value lines...')
+        line_dual_table.write(line_duals)
+
+        print('Writing service area points...')
+        point_table.write(points)
+
+        print('Creating indexes...')
+        line_single_table.create_index('seg_id')
+        line_dual_table.create_index('seg_id')
+
+    db.close()
+    print('Finished in {} seconds'.format(datetime.now() - start))
diff --git a/ais/engine/scripts/load_street_aliases.py b/ais/engine/scripts/load_street_aliases.py
index 0a777912..83617869 100644
--- a/ais/engine/scripts/load_street_aliases.py
+++ b/ais/engine/scripts/load_street_aliases.py
@@ -4,87 +4,85 @@
 from passyunk.data import DIRS_STD, SUFFIXES_STD
 import datum
 from ais import app
-# DEV
-from pprint import pprint
-
-print('Starting...')
-start = datetime.now()
-
-"""SET UP"""
-
-config = app.config
-source_def = config['BASE_DATA_SOURCES']['street_aliases']
-source_db = datum.connect(config['DATABASES'][source_def['db']])
-source_table = source_db[source_def['table']]
-field_map = source_def['field_map']
-db = datum.connect(config['DATABASES']['engine'])
-alias_table = db['street_alias']
-
-
-"""MAIN"""
-
-print('Dropping indexes...')
-alias_table.drop_index('seg_id')
-
-print('Delete existing aliases...')
-alias_table.delete()
-
-print('Reading aliases from source...')
-source_rows = source_table.read()
-aliases = []
-
-# Loop over aliases
-for i, alias_row in enumerate(source_rows):
-	try:
-		# Get attrs
-		predir = alias_row[field_map['street_predir']]
-		name = alias_row[field_map['street_name']]
-		suffix = alias_row[field_map['street_suffix']]
-		postdir = alias_row[field_map['street_postdir']]
-
-		street_fields = ['street_' + x for x in ['predir', 'name', 'suffix', 'postdir']]
-		source_comps = [alias_row[field_map[x]] for x in street_fields]
-		source_comps = [x if x else '' for x in source_comps]
-		source_street_full = ' '.join([x.strip() for x in source_comps if x])
-
-		# Make sure attrs are standardizable
-		invalid_predir = (predir and not predir in DIRS_STD)
-		invalid_suffix = (suffix and suffix not in SUFFIXES_STD)
-		invalid_postdir = (postdir and not postdir in DIRS_STD)
-		if any([invalid_predir, invalid_suffix, invalid_postdir]):
-			raise ValueError('Invalid alias: {}'.format(source_street_full))
-
-		# Standardize
-		predir = DIRS_STD[predir] if predir else None
-		suffix = SUFFIXES_STD[suffix] if suffix else None
-		postdir = DIRS_STD[postdir] if postdir else None
-
-		# Get values
-		aliases.append({
-			'seg_id': alias_row[field_map['seg_id']],
-			'street_predir': predir or '',
-			'street_name': name,
-			'street_suffix': suffix or '',
-			'street_postdir': postdir or '',
-			'street_full': source_street_full or '',
-		})
-
-	except ValueError:
-		# TODO: FEEDBACK
-		pass
-
-	except:
-		print(alias_row)
-		print(traceback.format_exc())
-		sys.exit()
-
-print('Writing aliases...')
-alias_table.write(aliases)
-
-print('Creating indexes...')
-alias_table.create_index('seg_id')
-
-db.save()
-#source_db.close()
-db.close()
-print('Finished in {} seconds'.format(datetime.now() - start))
\ No newline at end of file
+
+def main():
+    print('Starting...')
+    start = datetime.now()
+
+    """SET UP"""
+
+    config = app.config
+    source_def = config['BASE_DATA_SOURCES']['street_aliases']
+    source_db = datum.connect(config['DATABASES'][source_def['db']])
+    source_table = source_db[source_def['table']]
+    field_map = source_def['field_map']
+    db = datum.connect(config['DATABASES']['engine'])
+    alias_table = db['street_alias']
+
+    """MAIN"""
+
+    print('Dropping indexes...')
+    alias_table.drop_index('seg_id')
+
+    print('Delete existing aliases...')
+    alias_table.delete()
+
+    print('Reading aliases from source...')
+    source_rows = source_table.read()
+    aliases = []
+
+    # Loop over aliases
+    for i, alias_row in enumerate(source_rows):
+        try:
+            # Get attrs
+            predir = alias_row[field_map['street_predir']]
+            name = alias_row[field_map['street_name']]
+            suffix = alias_row[field_map['street_suffix']]
+            postdir = alias_row[field_map['street_postdir']]
+
+            street_fields = ['street_' + x for x in ['predir', 'name', 'suffix', 'postdir']]
+            source_comps = [alias_row[field_map[x]] for x in street_fields]
+            source_comps = [x if x else '' for x in source_comps]
+            source_street_full = ' '.join([x.strip() for x in source_comps if x])
+
+            # Make sure attrs are standardizable
+            invalid_predir = (predir and not predir in DIRS_STD)
+            invalid_suffix = (suffix and suffix not in SUFFIXES_STD)
+            invalid_postdir = (postdir and not postdir in DIRS_STD)
+            if any([invalid_predir, invalid_suffix, invalid_postdir]):
+                raise ValueError('Invalid alias: {}'.format(source_street_full))
+
+            # Standardize
+            predir = DIRS_STD[predir] if predir else None
+            suffix = SUFFIXES_STD[suffix] if suffix else None
+            postdir = DIRS_STD[postdir] if postdir else None
+
+            # Get values
+            aliases.append({
+                'seg_id': alias_row[field_map['seg_id']],
+                'street_predir': predir or '',
+                'street_name': name,
+                'street_suffix': suffix or '',
+                'street_postdir': postdir or '',
+                'street_full': source_street_full or '',
+            })
+
+        except ValueError:
+            # TODO: FEEDBACK
+            pass
+
+        except Exception as e:
+            print('Unhandled exception!')
+            print(alias_row)
+            print(traceback.format_exc())
+            raise e
+
+    print('Writing aliases...')
+    alias_table.write(aliases)
+
+    print('Creating indexes...')
+    alias_table.create_index('seg_id')
+
+    db.save()
+    db.close()
+    print('Finished in {} seconds'.format(datetime.now() - start))
diff --git a/ais/engine/scripts/load_streets.py b/ais/engine/scripts/load_streets.py
index c39d94e9..23e11f3f 100644
--- a/ais/engine/scripts/load_streets.py
+++ b/ais/engine/scripts/load_streets.py
@@ -7,122 +7,115 @@
 from datum import Database
 from ais.models import StreetSegment
 
-
-print('Starting...')
-start = datetime.now()
-
-"""SET UP"""
-
-config = app.config
-
-Parser = config['PARSER']
-db = Database(config['DATABASES']['engine'])
-engine_srid = config['ENGINE_SRID']
-
-# Get table params
-source_def = config['BASE_DATA_SOURCES']['streets']
-source_db_name = source_def['db']
-source_db_url = config['DATABASES'][source_db_name]
-field_map = source_def['field_map']
-source_table_name = source_def['table']
-street_full_fields = ['street_' + x for x in ['predir', 'name', 'suffix', 'postdir']]
-source_street_full_fields = [field_map[x] for x in street_full_fields]
-
-# Get table references
-source_db = Database(source_db_url)
-source_table = source_db[source_table_name]
-source_geom_field = source_table.geom_field
-street_table_name = StreetSegment.__table__.name
-street_table = db[street_table_name]
-
-
-"""MAIN"""
-
-parser = Parser()
-
-print('Deleting existing streets...')
-street_table.delete(cascade=True)
-
-print('Reading streets from source...')
-source_fields = list(field_map.values())
-source_rows = source_table.read(to_srid=engine_srid)
-
-streets = []
-error_count = 0
-
-# Loop over streets
-for i, source_row in enumerate(source_rows):
-	try:
-		if i % 10000 == 0:
-			print(i)
-
-		# Parse street name
-		source_street_full_comps = [str(source_row[x]).strip() for x in \
-			source_street_full_fields]
-		# source_street_full_comps = [x for x in source_street_full_comps if x != '']
-		source_street_full_comps = [x for x in source_street_full_comps if x not in ('', None, 'None')]
-		source_street_full = ' '.join(source_street_full_comps)
-		seg_id = source_row[field_map['seg_id']]
-		try:
-			parsed = parser.parse(source_street_full)
-			if parsed['type'] != 'street':
-				raise ValueError('Invalid street')
-
-			# comps = parsed['components']    			<== phladdress
-			comps = parsed['components']['street']    # <== passyunk
-		except Exception as e:
-			raise ValueError('Could not parse')
-
-		# Check for unaddressable streets
-		left_to = source_row[field_map['left_to']]
-		right_to = source_row[field_map['right_to']]
-		if left_to == 0 and right_to == 0:
-			raise ValueError('Not a range')
-
-		street_suffix = comps['suffix']
-		if street_suffix == 'RAMP':
-			raise ValueError('Ramp')
-
-		street_comps = {
-			'street_predir': comps['predir'] or '',
-			'street_name': comps['name'] or '',
-			'street_suffix': comps['suffix'] or '',
-			'street_postdir': comps['postdir'] or '',
-			'street_full': comps['full'],
-		}
-
-		# Stringify numeric fields that should be strings
-		# source_row['zip_left'] = str(source_row['zip_left'])
-		# source_row['zip_right'] = str(source_row['zip_right'])
-
-		# Get values
-		street = {key: source_row[value] for key, value in field_map.items()}
-		street.update(street_comps)
-		street['geom'] = source_row[source_geom_field]
-		streets.append(street)
-
-	except ValueError as e:
-		# FEEDBACK
-		print('{}: {} ({})'.format(e, source_street_full, seg_id))
-		error_count += 1
-
-	except Exception as e:
-		print('Unhandled error on row: {}'.format(i))
-		# pprint(street)
-		print(traceback.format_exc())
-		sys.exit()
-
-'''
-WRITE
-'''
-
-street_table.write(streets, chunk_size=50000)
-
-'''
-FINISH
-'''
-
-print('{} errors'.format(error_count))
-#source_db.close()
-db.close()
-print('Finished in {} seconds'.format(datetime.now() - start))
\ No newline at end of file
+def main():
+    print('Starting...')
+    start = datetime.now()
+
+    """SET UP"""
+
+    config = app.config
+
+    Parser = config['PARSER']
+
+    db = Database(config['DATABASES']['engine'])
+    engine_srid = config['ENGINE_SRID']
+
+    # Get table params
+    source_def = config['BASE_DATA_SOURCES']['streets']
+    source_db_name = source_def['db']
+    source_db_url = config['DATABASES'][source_db_name]
+    field_map = source_def['field_map']
+    source_table_name = source_def['table']
+    street_full_fields = ['street_' + x for x in ['predir', 'name', 'suffix', 'postdir']]
+    source_street_full_fields = [field_map[x] for x in street_full_fields]
+
+    # Get table references
+    source_db = Database(source_db_url)
+    source_table = source_db[source_table_name]
+    source_geom_field = source_table.geom_field
+    street_table_name = StreetSegment.__table__.name
+    street_table = db[street_table_name]
+
+
+    """MAIN"""
+
+    parser = Parser()
+
+    print('Deleting existing streets...')
+    street_table.delete(cascade=True)
+
+    print(f'Reading streets table {source_table} from source...')
+    source_fields = list(field_map.values())
+    source_rows = source_table.read(to_srid=engine_srid)
+    print('Rows retrieved.')
+
+    streets = []
+    error_count = 0
+
+    # Loop over streets
+    for i, source_row in enumerate(source_rows):
+        try:
+            if i % 10000 == 0:
+                print(i)
+
+            # Parse street name
+            source_street_full_comps = [str(source_row[x]).strip() for x in \
+                source_street_full_fields]
+            # source_street_full_comps = [x for x in source_street_full_comps if x != '']
+            source_street_full_comps = [x for x in source_street_full_comps if x not in ('', None, 'None')]
+            source_street_full = ' '.join(source_street_full_comps)
+            seg_id = source_row[field_map['seg_id']]
+            try:
+                parsed = parser.parse(source_street_full)
+                if parsed['type'] != 'street':
+                    raise ValueError('Invalid street')
+                comps = parsed['components']['street']    # <== passyunk
+            except Exception as e:
+                print(f'Could not parse {source_street_full}')
+                pass
+
+            # Check for unaddressable streets
+            left_to = source_row[field_map['left_to']]
+            right_to = source_row[field_map['right_to']]
+            if left_to == 0 and right_to == 0:
+                raise ValueError('Not a range')
+
+            street_suffix = comps['suffix']
+            if street_suffix == 'RAMP':
+                pass
+
+            street_comps = {
+                'street_predir': comps['predir'] or '',
+                'street_name': comps['name'] or '',
+                'street_suffix': comps['suffix'] or '',
+                'street_postdir': comps['postdir'] or '',
+                'street_full': comps['full'],
+            }
+
+            # Get values
+            street = {key: source_row[value] for key, value in field_map.items()}
+            street.update(street_comps)
+            street['geom'] = source_row[source_geom_field]
+            streets.append(street)
+
+        except ValueError as e:
+            error_count += 1
+
+        except Exception as e:
+            print('Unhandled error on row: {}'.format(i))
+            print(traceback.format_exc())
+            raise e
+
+    '''
+    WRITE
+    '''
+
+    street_table.write(streets, chunk_size=50000)
+
+    '''
+    FINISH
+    '''
+
+    print('{} errors'.format(error_count))
+    db.close()
+    print('Finished in {} seconds'.format(datetime.now() - start))
diff --git a/ais/engine/scripts/load_zip_ranges.py b/ais/engine/scripts/load_zip_ranges.py
index 323f230b..0cfdf19d 100644
--- a/ais/engine/scripts/load_zip_ranges.py
+++ b/ais/engine/scripts/load_zip_ranges.py
@@ -10,294 +10,296 @@
 import traceback
 from pprint import pprint
 
-start = datetime.now()
-
-"""SET UP"""
-
-config = app.config
-db = datum.connect(config['DATABASES']['engine'])
-source_db = datum.connect(config['DATABASES']['gis'])
-# source_table = source_db['usps_zip4s']
-source_table = source_db['vw_usps_zip4s_ais']
-field_map = {
-	'usps_id':			'updatekey',
-	'address_low':		'addrlow',
-	'address_high':		'addrhigh',
-	'address_oeb':		'addroeb',
-	'street_predir':	'streetpre',
-	'street_name':		'streetname',
-	'street_suffix':	'streetsuff',
-	'street_postdir':	'streetpost',
-	'unit_type':		'addrsecondaryabbr',
-	'unit_low':			'addrsecondarylow',
-	'unit_high':		'addrsecondaryhigh',
-	'unit_oeb':			'addrsecondaryoeb',
-	'zip_code':			'zipcode',
-	'zip_4_low':		'zip4low',
-	'zip_4_high':		'zip4high',
-}
-numeric_fields = ['address_low', 'address_high']
-zip_range_table = db['zip_range']
-address_zip_table = db['address_zip']
-WRITE_OUT = True
-
-source_fields = [field_map[x] for x in field_map]
-char_fields = [x for x in field_map if not x in numeric_fields]
-
-"""MAIN"""
-
-if WRITE_OUT:
-	print('Dropping indexes...')
-	zip_range_table.drop_index('street_address')
-
-	print('Deleting existing zip ranges...')
-	zip_range_table.delete()
-
-print('Reading zip ranges from source...')
-# TODO: currently filtering out alphanumeric addrlows
-source_rows = source_table.read(fields=source_fields)
-
-zip_ranges = []
-
-for i, source_row in enumerate(source_rows):
-	if i % 25000 == 0:
-		print(i)
-
-	zip_range = {x: source_row[field_map[x]] for x in field_map}
-
-	# Default char fields to empty string
-	for x in char_fields:
-		if zip_range[x] is None:
-			zip_range[x] = ''
-	
-	# Handle differing ZIP4 low/high
-	zip_4_low = source_row[field_map['zip_4_low']]
-	zip_4_high = source_row[field_map['zip_4_high']]
-	if zip_4_low != zip_4_high:
-		zip_4 = ''
-	else:
-		zip_4 = zip_4_low
-
-	# Set ZIP4
-	zip_range.pop('zip_4_low')
-	zip_range.pop('zip_4_high')
-	zip_range['zip_4'] = zip_4
-
-	zip_ranges.append(zip_range)
-
-if WRITE_OUT:
-	print('Writing zip ranges to AIS...')
-	zip_range_table.write(zip_ranges)
-
-	print('Creating indexes...')
-	zip_range_table.create_index('usps_id')
-
-
-print('\n** RELATE TO ADDRESSES**')
-print('Reading addresses...')
-addresses = db['address'].read(fields=['street_address'])
-addresses = [Address(x['street_address']) for x in addresses]
-
-if WRITE_OUT:
-	print('Dropping indexes...')
-	address_zip_table.drop_index('street_address')
-	address_zip_table.drop_index('usps_id')
-	print('Dropping address-zips...')
-	address_zip_table.delete()
-
-# index zip ranges by street_full
-street_full_fields = [
-	'street_predir',
-	'street_name',
-	'street_suffix',
-	'street_postdir',
-]
-
-# For checking alpha unit ranges
-alpha_list = list(map(chr, range(65, 91)))
-alpha_map = {alpha_list[i]: i + 1 for i in range(0, 26)}  # A => 1, Z => 26
-
-GENERIC_UNITS = set(['#', 'APT', 'UNIT', 'STE'])
-
-zip_map_no_units = {}		# street_full => [non-unit ranges]
-zip_map_units = {}			# street_full => [unit ranges]
-address_zips = []
-
-print('Indexing zip ranges by street...')
-for zip_range in zip_ranges:
-	street_full = ' '.join([zip_range[x] for x in street_full_fields \
-		if zip_range[x] != ''])
-	if zip_range['unit_type'] != '':
-		street_zip_ranges = zip_map_units.setdefault(street_full, [])
-	else:
-		street_zip_ranges = zip_map_no_units.setdefault(street_full, [])
-	street_zip_ranges.append(zip_range)
-
-# DEV
-exact_count = 0
-unit_num_count = 0
-unit_alpha_count = 0
-
-# Loop over addresses
-for address in addresses:
-	try:
-		address_low = address.address_low
-		address_high = address.address_high or address_low
-		address_parity = address.parity
-		street_address = address.street_address
-		street_full = address.street_full
-		unit_type = address.unit_type
-		unit_num = address.unit_num
-
-		matching_zip_range = None
-		match_type = None
-
-		# UNIT
-		# TODO: handle unit types like REAR that don't have a unit num
-		if unit_type and unit_num:
-			try:
-				street_zip_ranges = zip_map_units[street_full]
-				
-				# Determine unit character type
-				# ex. numeric, alpha, alphanum
-				if unit_num.isdigit():
-					unit_char_type = 'num'
-				# Only accepting single alpha units for now. Multiple will take
-				# more handling logic.
-				elif unit_num.isalpha() and len(unit_num) == 1:
-					unit_char_type = 'alpha'
-				else:
-					raise ValueError('Unit format not recognized')
-
-				for zip_range in street_zip_ranges:
-					zip_unit_type = zip_range['unit_type']
-					zip_unit_low = zip_range['unit_low']
-					zip_unit_high = zip_range['unit_high']
-					
-					# Check if address matches
-					if not (zip_range['address_low'] <= address_low and \
-						address_high <= zip_range['address_high']):
-						continue
-
-					# Check if parity matches
-					zip_address_parity = zip_range['address_oeb']
-					if zip_address_parity not in ['B', address_parity]:
-						continue
-
-					# Check if unit type matches
-					if unit_type != zip_unit_type and \
-						not (unit_type in GENERIC_UNITS and \
-						zip_unit_type in GENERIC_UNITS):
-						continue
-
-					# Get char type of unit range
-					if zip_unit_low.isdigit() and zip_unit_high.isdigit():
-						zip_unit_char_type = 'num'
-					elif zip_unit_low.isalpha() and zip_unit_high.isalpha():
-						zip_unit_char_type = 'alpha'
-					else:
-						# Unhandled unit char type
-						continue
-
-					# If the types don't match, continue
-					if zip_unit_char_type != unit_char_type:
-						continue
-
-					# Case 1: numeric unit
-					if unit_char_type == 'num':
-						if zip_unit_low <= unit_num <= zip_unit_high:
-							matching_zip_range = zip_range
-							match_type = 'unit_numeric'
-							unit_num_count += 1
-							break
-
-					# Case 2: alpha unit range
-					elif unit_char_type == 'alpha' and \
-						len(zip_unit_low) == 1 and len(zip_unit_high) == 1:
-						try:
-							unit_alpha_i = alpha_map[unit_num]
-							zip_alpha_i_low = alpha_map[zip_unit_low]
-							zip_alpha_i_high = alpha_map[zip_unit_high]
-						except KeyError:
-							print('Unhandled KeyError')
-
-						if zip_alpha_i_low <= unit_alpha_i <= \
-							zip_alpha_i_high:
-							# print('we got an alpha match')
-							# print(street_address)
-							# print(zip_range)
-							# sys.exit()
-							matching_zip_range = zip_range
-							match_type = 'unit_alpha'
-							unit_alpha_count += 1
-							break
-
-			except ValueError:
-				# This should only happen when we had an unrecognized unit
-				# format. Ignore and try to match to base zip range.
-				pass
-
-			except KeyError:
-				pass
-
-		# NON-UNIT
-		# Use this if statement and not an else, because we still want this to
-		# run if the unit search didn't turn anything up.
-		if matching_zip_range is None:
-			try:
-				street_zip_ranges = zip_map_no_units[street_full]
-			except KeyError:
-				raise ValueError('Not a USPS street')
-
-			for zip_range in street_zip_ranges:
-				# Check if parity matches
-				zip_address_parity = zip_range['address_oeb']
-				if zip_address_parity not in ['B', address_parity]:
-					continue
-
-				if zip_range['address_low'] <= address_low and \
-					address_high <= zip_range['address_high']:
-					matching_zip_range = zip_range
-
-					# If there was a unit that we ignored, flag it
-					if unit_type:
-						match_type = 'ignore_unit'
-					else:
-						match_type = 'exact'
-
-					exact_count += 1
-					break
-
-		if matching_zip_range:
-			address_zips.append({
-				'street_address':	street_address,
-				'usps_id':			matching_zip_range['usps_id'],
-				'match_type':		match_type,
-			})
-
-		else:
-			raise ValueError('Could not match to a ZIP range')
-
-	except ValueError as e:
-		# FEEDBACK
-		# print('{}: {}'.format(street_address, e))
-		pass
-
-print(len(address_zips))
-print('num: ' + str(unit_num_count))
-print('alpha: ' + str(unit_alpha_count))
-print('exact: ' + str(exact_count))
-
-if WRITE_OUT:
-	print('Writing address-zips...')
-	address_zip_table.write(address_zips, chunk_size=150000)
-	print('Creating index...')
-	address_zip_table.create_index('street_address')
-	address_zip_table.create_index('usps_id')
-
-################################################################################
-
-source_db.close()
-db.close()
-
-print('Finished in {}'.format(datetime.now() - start))
+def main():
+    start = datetime.now()
+
+    """SET UP"""
+
+    config = app.config
+    db = datum.connect(config['DATABASES']['engine'])
+    source_db = datum.connect(config['DATABASES']['gis'])
+    # source_table = source_db['usps_zip4s']
+    source_table = source_db['vw_usps_zip4s_ais']
+    field_map = {
+        'usps_id':			'updatekey',
+        'address_low':		'addrlow',
+        'address_high':		'addrhigh',
+        'address_oeb':		'addroeb',
+        'street_predir':	'streetpre',
+        'street_name':		'streetname',
+        'street_suffix':	'streetsuff',
+        'street_postdir':	'streetpost',
+        'unit_type':		'addrsecondaryabbr',
+        'unit_low':			'addrsecondarylow',
+        'unit_high':		'addrsecondaryhigh',
+        'unit_oeb':			'addrsecondaryoeb',
+        'zip_code':			'zipcode',
+        'zip_4_low':		'zip4low',
+        'zip_4_high':		'zip4high',
+    }
+    numeric_fields = ['address_low', 'address_high']
+    zip_range_table = db['zip_range']
+    address_zip_table = db['address_zip']
+    WRITE_OUT = True
+
+    source_fields = [field_map[x] for x in field_map]
+    char_fields = [x for x in field_map if not x in numeric_fields]
+
+    """MAIN"""
+
+    if WRITE_OUT:
+        print('Dropping indexes...')
+        zip_range_table.drop_index('street_address')
+
+        print('Deleting existing zip ranges...')
+        zip_range_table.delete()
+
+    print('Reading zip ranges from source...')
+    # TODO: currently filtering out alphanumeric addrlows
+    source_rows = source_table.read(fields=source_fields)
+
+    zip_ranges = []
+
+    for i, source_row in enumerate(source_rows):
+        if i % 25000 == 0:
+            print(i)
+
+        zip_range = {x: source_row[field_map[x]] for x in field_map}
+
+        # Default char fields to empty string
+        for x in char_fields:
+            if zip_range[x] is None:
+                zip_range[x] = ''
+        
+        # Handle differing ZIP4 low/high
+        zip_4_low = source_row[field_map['zip_4_low']]
+        zip_4_high = source_row[field_map['zip_4_high']]
+        if zip_4_low != zip_4_high:
+            zip_4 = ''
+        else:
+            zip_4 = zip_4_low
+
+        # Set ZIP4
+        zip_range.pop('zip_4_low')
+        zip_range.pop('zip_4_high')
+        zip_range['zip_4'] = zip_4
+
+        zip_ranges.append(zip_range)
+
+    if WRITE_OUT:
+        print('Writing zip ranges to AIS...')
+        zip_range_table.write(zip_ranges)
+
+        print('Creating indexes...')
+        zip_range_table.create_index('usps_id')
+
+
+    print('\n** RELATE TO ADDRESSES**')
+    print('Reading addresses...')
+    addresses = db['address'].read(fields=['street_address'])
+    addresses = [Address(x['street_address']) for x in addresses]
+
+    if WRITE_OUT:
+        print('Dropping indexes...')
+        address_zip_table.drop_index('street_address')
+        address_zip_table.drop_index('usps_id')
+        print('Dropping address-zips...')
+        address_zip_table.delete()
+
+    # index zip ranges by street_full
+    street_full_fields = [
+        'street_predir',
+        'street_name',
+        'street_suffix',
+        'street_postdir',
+    ]
+
+    # For checking alpha unit ranges
+    alpha_list = list(map(chr, range(65, 91)))
+    alpha_map = {alpha_list[i]: i + 1 for i in range(0, 26)}  # A => 1, Z => 26
+
+    GENERIC_UNITS = set(['#', 'APT', 'UNIT', 'STE'])
+
+    zip_map_no_units = {}		# street_full => [non-unit ranges]
+    zip_map_units = {}			# street_full => [unit ranges]
+    address_zips = []
+
+    print('Indexing zip ranges by street...')
+    for zip_range in zip_ranges:
+        street_full = ' '.join([zip_range[x] for x in street_full_fields \
+            if zip_range[x] != ''])
+        if zip_range['unit_type'] != '':
+            street_zip_ranges = zip_map_units.setdefault(street_full, [])
+        else:
+            street_zip_ranges = zip_map_no_units.setdefault(street_full, [])
+        street_zip_ranges.append(zip_range)
+
+    # DEV
+    exact_count = 0
+    unit_num_count = 0
+    unit_alpha_count = 0
+
+    # Loop over addresses
+    for address in addresses:
+        try:
+            address_low = address.address_low
+            address_high = address.address_high or address_low
+            address_parity = address.parity
+            street_address = address.street_address
+            street_full = address.street_full
+            unit_type = address.unit_type
+            unit_num = address.unit_num
+
+            matching_zip_range = None
+            match_type = None
+
+            # UNIT
+            # TODO: handle unit types like REAR that don't have a unit num
+            if unit_type and unit_num:
+                try:
+                    street_zip_ranges = zip_map_units[street_full]
+                    
+                    # Determine unit character type
+                    # ex. numeric, alpha, alphanum
+                    if unit_num.isdigit():
+                        unit_char_type = 'num'
+                    # Only accepting single alpha units for now. Multiple will take
+                    # more handling logic.
+                    elif unit_num.isalpha() and len(unit_num) == 1:
+                        unit_char_type = 'alpha'
+                    else:
+                        raise ValueError('Unit format not recognized')
+
+                    for zip_range in street_zip_ranges:
+                        zip_unit_type = zip_range['unit_type']
+                        zip_unit_low = zip_range['unit_low']
+                        zip_unit_high = zip_range['unit_high']
+                        
+                        # Check if address matches
+                        if not (zip_range['address_low'] <= address_low and \
+                            address_high <= zip_range['address_high']):
+                            continue
+
+                        # Check if parity matches
+                        zip_address_parity = zip_range['address_oeb']
+                        if zip_address_parity not in ['B', address_parity]:
+                            continue
+
+                        # Check if unit type matches
+                        if unit_type != zip_unit_type and \
+                            not (unit_type in GENERIC_UNITS and \
+                            zip_unit_type in GENERIC_UNITS):
+                            continue
+
+                        # Get char type of unit range
+                        if zip_unit_low.isdigit() and zip_unit_high.isdigit():
+                            zip_unit_char_type = 'num'
+                        elif zip_unit_low.isalpha() and zip_unit_high.isalpha():
+                            zip_unit_char_type = 'alpha'
+                        else:
+                            # Unhandled unit char type
+                            continue
+
+                        # If the types don't match, continue
+                        if zip_unit_char_type != unit_char_type:
+                            continue
+
+                        # Case 1: numeric unit
+                        if unit_char_type == 'num':
+                            if zip_unit_low <= unit_num <= zip_unit_high:
+                                matching_zip_range = zip_range
+                                match_type = 'unit_numeric'
+                                unit_num_count += 1
+                                break
+
+                        # Case 2: alpha unit range
+                        elif unit_char_type == 'alpha' and \
+                            len(zip_unit_low) == 1 and len(zip_unit_high) == 1:
+                            try:
+                                unit_alpha_i = alpha_map[unit_num]
+                                zip_alpha_i_low = alpha_map[zip_unit_low]
+                                zip_alpha_i_high = alpha_map[zip_unit_high]
+                            except KeyError:
+                                #print('Unhandled KeyError')
+                                                            pass
+
+                            if zip_alpha_i_low <= unit_alpha_i <= \
+                                zip_alpha_i_high:
+                                # print('we got an alpha match')
+                                # print(street_address)
+                                # print(zip_range)
+                                # sys.exit()
+                                matching_zip_range = zip_range
+                                match_type = 'unit_alpha'
+                                unit_alpha_count += 1
+                                break
+
+                except ValueError:
+                    # This should only happen when we had an unrecognized unit
+                    # format. Ignore and try to match to base zip range.
+                    pass
+
+                except KeyError:
+                    pass
+
+            # NON-UNIT
+            # Use this if statement and not an else, because we still want this to
+            # run if the unit search didn't turn anything up.
+            if matching_zip_range is None:
+                try:
+                    street_zip_ranges = zip_map_no_units[street_full]
+                except KeyError:
+                    raise ValueError('Not a USPS street')
+
+                for zip_range in street_zip_ranges:
+                    # Check if parity matches
+                    zip_address_parity = zip_range['address_oeb']
+                    if zip_address_parity not in ['B', address_parity]:
+                        continue
+
+                    if zip_range['address_low'] <= address_low and \
+                        address_high <= zip_range['address_high']:
+                        matching_zip_range = zip_range
+
+                        # If there was a unit that we ignored, flag it
+                        if unit_type:
+                            match_type = 'ignore_unit'
+                        else:
+                            match_type = 'exact'
+
+                        exact_count += 1
+                        break
+
+            if matching_zip_range:
+                address_zips.append({
+                    'street_address':	street_address,
+                    'usps_id':			matching_zip_range['usps_id'],
+                    'match_type':		match_type,
+                })
+
+            else:
+                raise ValueError('Could not match to a ZIP range')
+
+        except ValueError as e:
+            # FEEDBACK
+            # print('{}: {}'.format(street_address, e))
+            pass
+
+    print(len(address_zips))
+    print('num: ' + str(unit_num_count))
+    print('alpha: ' + str(unit_alpha_count))
+    print('exact: ' + str(exact_count))
+
+    if WRITE_OUT:
+        print('Writing address-zips...')
+        address_zip_table.write(address_zips, chunk_size=150000)
+        print('Creating index...')
+        address_zip_table.create_index('street_address')
+        address_zip_table.create_index('usps_id')
+
+    ################################################################################
+
+    source_db.close()
+    db.close()
+
+    print('Finished in {}'.format(datetime.now() - start))
diff --git a/ais/engine/scripts/make_address_summary.py b/ais/engine/scripts/make_address_summary.py
index fd7ada6e..fd8a16cd 100644
--- a/ais/engine/scripts/make_address_summary.py
+++ b/ais/engine/scripts/make_address_summary.py
@@ -10,439 +10,443 @@
 import traceback
 from pprint import pprint
 
-print('Starting...')
-start = datetime.now()
-
-# TODO: This should probably make a DB query for each address, rather than chunking
-# into street names. Getting hard to manage.
-
-"""SET UP"""
-
-config = app.config
-db = datum.connect(config['DATABASES']['engine'])
-tag_fields = config['ADDRESS_SUMMARY']['tag_fields']
-non_summary_tags = config['ADDRESS_SUMMARY']['non_summary_tags']
-geocode_table = db['geocode']
-address_table = db['address']
-max_values = config['ADDRESS_SUMMARY']['max_values']
-geocode_types = config['ADDRESS_SUMMARY']['geocode_types']
-geocode_priority_map = config['ADDRESS_SUMMARY']['geocode_priority']
-#geocode_types_on_curb = config['ADDRESS_SUMMARY']['geocode_types_on_curb']
-geocode_types_in_street = config['ADDRESS_SUMMARY']['geocode_types_in_street']
-
-tag_table = db['address_tag']
-link_table = db['address_link']
-address_summary_table = db['address_summary']
+def main():
+
+    print('Starting...')
+    start = datetime.now()
+
+    # TODO: This should probably make a DB query for each address, rather than chunking
+    # into street names. Getting hard to manage.
+
+    """SET UP"""
+
+    config = app.config
+    db = datum.connect(config['DATABASES']['engine'])
+    tag_fields = config['ADDRESS_SUMMARY']['tag_fields']
+    non_summary_tags = config['ADDRESS_SUMMARY']['non_summary_tags']
+    geocode_table = db['geocode']
+    address_table = db['address']
+    max_values = config['ADDRESS_SUMMARY']['max_values']
+    geocode_types = config['ADDRESS_SUMMARY']['geocode_types']
+    geocode_priority_map = config['ADDRESS_SUMMARY']['geocode_priority']
+    #geocode_types_on_curb = config['ADDRESS_SUMMARY']['geocode_types_on_curb']
+    geocode_types_in_street = config['ADDRESS_SUMMARY']['geocode_types_in_street']
+
+    tag_table = db['address_tag']
+    link_table = db['address_link']
+    address_summary_table = db['address_summary']
+
+    # DEV
+    WRITE_OUT = True
+
+    def wkt_to_xy(wkt):
+        xy = wkt.replace('POINT(', '')
+        xy = xy.replace(')', '')
+        split = xy.split(' ')
+        return float(split[0]), float(split[1])
+
+
+    print('Reading address links...')
+    link_map = {}
+    link_rows = link_table.read()
+    for link_row in link_rows:
+        address_1 = link_row['address_1']
+        address_2 = link_row['address_2']
+        relationship = link_row['relationship']
+        if not address_1 in link_map:
+            link_map[address_1] = []
+        link_map[address_1].append(link_row)
+
+
+    def get_tag_by_key(tag_rows, search_key):
+        for tag_row in tag_rows:
+            if tag_row['key'] == search_key:
+                return tag_row['value']
+        return None
+
+
+    # Get street names for chunking addresses
+    print('Reading street names...')
+    street_name_stmt = '''
+        select distinct street_name from address order by street_name
+    '''
+    street_names = [x['street_name'] for x in db.execute(street_name_stmt)]
 
-# DEV
-WRITE_OUT = True
-
-def wkt_to_xy(wkt):
-    xy = wkt.replace('POINT(', '')
-    xy = xy.replace(')', '')
-    split = xy.split(' ')
-    return float(split[0]), float(split[1])
-
-
-print('Reading address links...')
-link_map = {}
-link_rows = link_table.read()
-for link_row in link_rows:
-    address_1 = link_row['address_1']
-    address_2 = link_row['address_2']
-    relationship = link_row['relationship']
-    if not address_1 in link_map:
-        link_map[address_1] = []
-    link_map[address_1].append(link_row)
-
-
-def get_tag_by_key(tag_rows, search_key):
-    for tag_row in tag_rows:
-        if tag_row['key'] == search_key:
-            return tag_row['value']
-    return None
-
-
-# Get street names for chunking addresses
-print('Reading street names...')
-street_name_stmt = '''
-	select distinct street_name from address order by street_name
-'''
-street_names = [x['street_name'] for x in db.execute(street_name_stmt)]
-
-if WRITE_OUT:
-    print('Dropping indexes...')
-    address_summary_table.drop_index('street_address')
-    trgm_idx_stmt = '''
-	DROP INDEX IF EXISTS address_summary_opa_owners_trigram_idx;
-	'''
-    db.execute(trgm_idx_stmt)
-    db.save()
+    if WRITE_OUT:
+        print('Dropping indexes...')
+        address_summary_table.drop_index('street_address')
+        trgm_idx_stmt = '''
+        DROP INDEX IF EXISTS address_summary_opa_owners_trigram_idx;
+        '''
+        db.execute(trgm_idx_stmt)
+        db.save()
 
-    print('Deleting existing summary rows...')
-    address_summary_table.delete()
-
-    print('Creating temporary street name index...')
-    address_table.create_index('street_name')
-
-print('Reading XYs...')
-geocode_rows = geocode_table.read( \
-    fields=['street_address', 'geocode_type'], \
-    geom_field='geom' \
-    )
-geocode_map = {}  # street_address => [geocode rows]
-for geocode_row in geocode_rows:
-    street_address = geocode_row['street_address']
-    geocode_type_str = (list(geocode_priority_map.keys())[list(geocode_priority_map.values()).index(geocode_row['geocode_type'])])
-    geocode_row['geocode_type'] = geocode_type_str
-    if not street_address in geocode_map:
-        geocode_map[street_address] = []
-    geocode_map[street_address].append(geocode_row)
-
-print('Indexing addresses...')
-address_rows_all = address_table.read()
-street_map = {}  # street_name => [address rows]
-for address_row in address_rows_all:
-    street_name = address_row['street_name']
-    if not street_name in street_map:
-        street_map[street_name] = []
-    street_map[street_name].append(address_row)
-
-address_map = {x['street_address']: x for x in address_rows_all}
-
-print('Reading ungeocoded OPA addresses...')
-ungeocoded_opa_addresses_stmt = '''
-    select street_address from opa_property
-    except
-    select street_address from geocode
-'''
-ungeocoded_opa_address_dicts = db.execute(ungeocoded_opa_addresses_stmt)
-ungeocoded_opa_addresses = [row['street_address'] for row in ungeocoded_opa_address_dicts]
-
-print('Reading unit children...')
-unit_child_stmt = '''
-	select address_1, address_2
-	from address_link
-	where relationship = 'has generic unit'
-'''
-unit_child_rows = db.execute(unit_child_stmt)
-unit_child_map = {}  # unit parent => [unit children]
-unit_children_set = set()  # use this to lookup children quickly
-
-for unit_child_row in unit_child_rows:
-    child_address = unit_child_row['address_1']
-    parent_address = unit_child_row['address_2']
-    unit_child_map.setdefault(parent_address, [])
-    unit_child_map[parent_address].append(child_address)
-    unit_children_set.add(child_address)
-
-# Make a map of generic ("pound") unit addresses and corresponding tags for
-# all child unit addreses (APT, UNIT). This is to consolidate redundant addrs
-# like 1 CHESTNUT ST UNIT 1, 1 CHESTNUT # 1. Default to pounds.
-generic_unit_tags = {}  # pound address => {tag key: [tag vals]}
-
-summary_rows = []
-geocode_errors = 0
-
-"""MAIN"""
-
-cur_first_character = None
-
-print('Reading addresses...')
-for i, street_name in enumerate(street_names):
-    first_character = street_name[0]
-    if first_character != cur_first_character:
-        print(street_name)
-        cur_first_character = first_character
-
-    address_rows = street_map[street_name]
-
-    # Get address tags
-    tag_map = {}  # street_address => tag_key => [tag values]
-    tag_keys = [x['tag_key'] for x in tag_fields]
-    tag_where = "key in ({})".format(', '.join(["'{}'".format(x) for x in tag_keys]))
-    tag_stmt = '''
-		select street_address, key, value from address_tag
-		where street_address in (
-			select street_address from address where street_name = '{}'
-		)
-	'''.format(street_name)
-    tag_rows = db.execute(tag_stmt)
-
-    # Make tag map
-    for tag_row in tag_rows:
-        street_address = tag_row['street_address']
-        if not street_address in tag_map:
-            tag_map[street_address] = []
-        # tag_map_obj = {tag_row['key']: tag_row['value']}
-        tag_map[street_address].append(tag_row)
-
-    for i, address_row in enumerate(address_rows):
-        street_address = address_row['street_address']
-
-        # Skip unit children
-        if street_address in unit_children_set:
-            continue
-
-        summary_row = deepcopy(address_row)
-        tag_rows = tag_map.get(street_address)
-
-        # If this address has unit children, append those tags
-        if street_address in unit_child_map:
-            unit_children = unit_child_map[street_address]
-            for unit_child in unit_children:
-                if unit_child in tag_map and tag_rows is not None:
-                    tag_rows += tag_map[unit_child]
+        print('Deleting existing summary rows...')
+        address_summary_table.delete()
 
-        '''
-        GET TAG FIELDS
-        '''
+        print('Creating temporary street name index...')
+        address_table.create_index('street_name')
 
-        for tag_field in tag_fields:
-            field_name = tag_field['name']
-            if field_name in non_summary_tags:
+    print('Reading XYs...')
+    geocode_rows = geocode_table.read( \
+        fields=['street_address', 'geocode_type'], \
+        geom_field='geom' \
+        )
+    geocode_map = {}  # street_address => [geocode rows]
+    for geocode_row in geocode_rows:
+        street_address = geocode_row['street_address']
+        geocode_type_str = (list(geocode_priority_map.keys())[list(geocode_priority_map.values()).index(geocode_row['geocode_type'])])
+        geocode_row['geocode_type'] = geocode_type_str
+        if not street_address in geocode_map:
+            geocode_map[street_address] = []
+        geocode_map[street_address].append(geocode_row)
+
+    print('Indexing addresses...')
+    address_rows_all = address_table.read()
+    street_map = {}  # street_name => [address rows]
+    for address_row in address_rows_all:
+        street_name = address_row['street_name']
+        if not street_name in street_map:
+            street_map[street_name] = []
+        street_map[street_name].append(address_row)
+
+    address_map = {x['street_address']: x for x in address_rows_all}
+
+    print('Reading ungeocoded OPA addresses...')
+    ungeocoded_opa_addresses_stmt = '''
+        select street_address from opa_property
+        except
+        select street_address from geocode
+    '''
+    ungeocoded_opa_address_dicts = db.execute(ungeocoded_opa_addresses_stmt)
+    ungeocoded_opa_addresses = [row['street_address'] for row in ungeocoded_opa_address_dicts]
+
+    print('Reading unit children...')
+    unit_child_stmt = '''
+        select address_1, address_2
+        from address_link
+        where relationship = 'has generic unit'
+    '''
+    unit_child_rows = db.execute(unit_child_stmt)
+    unit_child_map = {}  # unit parent => [unit children]
+    unit_children_set = set()  # use this to lookup children quickly
+
+    for unit_child_row in unit_child_rows:
+        child_address = unit_child_row['address_1']
+        parent_address = unit_child_row['address_2']
+        unit_child_map.setdefault(parent_address, [])
+        unit_child_map[parent_address].append(child_address)
+        unit_children_set.add(child_address)
+
+    # Make a map of generic ("pound") unit addresses and corresponding tags for
+    # all child unit addreses (APT, UNIT). This is to consolidate redundant addrs
+    # like 1 CHESTNUT ST UNIT 1, 1 CHESTNUT # 1. Default to pounds.
+    generic_unit_tags = {}  # pound address => {tag key: [tag vals]}
+
+    summary_rows = []
+    geocode_errors = 0
+
+    """MAIN"""
+
+    cur_first_character = None
+
+    print('Reading addresses...')
+    for i, street_name in enumerate(street_names):
+        first_character = street_name[0]
+        if first_character != cur_first_character:
+            #print(street_name)
+            cur_first_character = first_character
+
+        address_rows = street_map[street_name]
+
+        # Get address tags
+        tag_map = {}  # street_address => tag_key => [tag values]
+        tag_keys = [x['tag_key'] for x in tag_fields]
+        tag_where = "key in ({})".format(', '.join(["'{}'".format(x) for x in tag_keys]))
+        tag_stmt = '''
+            select street_address, key, value from address_tag
+            where street_address in (
+                select street_address from address where street_name = '{}'
+            )
+        '''.format(street_name)
+        tag_rows = db.execute(tag_stmt)
+
+        # Make tag map
+        for tag_row in tag_rows:
+            street_address = tag_row['street_address']
+            if not street_address in tag_map:
+                tag_map[street_address] = []
+            # tag_map_obj = {tag_row['key']: tag_row['value']}
+            tag_map[street_address].append(tag_row)
+
+        for i, address_row in enumerate(address_rows):
+            street_address = address_row['street_address']
+
+            # Skip unit children
+            if street_address in unit_children_set:
                 continue
-            tag_key = tag_field['tag_key']
-            field_type = tag_field['type']
-            values = []
-
-            # If the address has tags at all
-            if tag_rows:
-                # Loop trying to find
-                for tag_row in tag_rows:
-                    if tag_row['key'] == tag_key:
-                        # Make uppercase
-                        value = tag_row['value'].upper()
-                        values.append(value)
-
-            values = list(set(filter(None, values)))
-            if len(values) > 0:
-                # Hack to supersede generic unit parser tags over base address parser tags
-                # TODO: handle in Passyunk / standardize '#' unit_types to generic type (APT or UNIT) / fix source addresses
-                if 'usps' in tag_key:
-                    value_address_map = {}
-                    generic_usps_value = ''
+
+            summary_row = deepcopy(address_row)
+            tag_rows = tag_map.get(street_address)
+
+            # If this address has unit children, append those tags
+            if street_address in unit_child_map:
+                unit_children = unit_child_map[street_address]
+                for unit_child in unit_children:
+                    if unit_child in tag_map and tag_rows is not None:
+                        tag_rows += tag_map[unit_child]
+
+            '''
+            GET TAG FIELDS
+            '''
+
+            for tag_field in tag_fields:
+                field_name = tag_field['name']
+                if field_name in non_summary_tags:
+                    continue
+                tag_key = tag_field['tag_key']
+                field_type = tag_field['type']
+                values = []
+
+                # If the address has tags at all
+                if tag_rows:
+                    # Loop trying to find
                     for tag_row in tag_rows:
                         if tag_row['key'] == tag_key:
-                            tag_address = tag_row['street_address']
-                            if tag_address not in value_address_map:
-                                value_address_map[tag_address] = [] #make list just in case there's more than one generic unit address with a unique value
+                            # Make uppercase
                             value = tag_row['value'].upper()
-                            value_address_map[tag_address].append(value)
-                    for address in value_address_map:
-                        if '#' not in address:
-                            value = value_address_map[address][0] # arbitrarily choose first value
-                        else:
-                            generic_usps_value = value_address_map[address][0] # arbitrarily choose first value
-                    value = value if value else generic_usps_value
-                else:
-                    values = sorted(values)
-                    # value = '|'.join(values[:max_values])
-                    value = '|'.join(values)
-                    value = value[:2000]
-            else:
-                if field_type == 'number':
-                    value = None
+                            values.append(value)
+
+                values = list(set(filter(None, values)))
+                if len(values) > 0:
+                    # Hack to supersede generic unit parser tags over base address parser tags
+                    # TODO: handle in Passyunk / standardize '#' unit_types to generic type (APT or UNIT) / fix source addresses
+                    if 'usps' in tag_key:
+                        value_address_map = {}
+                        generic_usps_value = ''
+                        for tag_row in tag_rows:
+                            if tag_row['key'] == tag_key:
+                                tag_address = tag_row['street_address']
+                                if tag_address not in value_address_map:
+                                    value_address_map[tag_address] = [] #make list just in case there's more than one generic unit address with a unique value
+                                value = tag_row['value'].upper()
+                                value_address_map[tag_address].append(value)
+                        for address in value_address_map:
+                            if '#' not in address:
+                                value = value_address_map[address][0] # arbitrarily choose first value
+                            else:
+                                generic_usps_value = value_address_map[address][0] # arbitrarily choose first value
+                        value = value if value else generic_usps_value
+                    else:
+                        values = sorted(values)
+                        # value = '|'.join(values[:max_values])
+                        value = '|'.join(values)
+                        value = value[:2000]
                 else:
-                    value = ''
-
-            summary_row[field_name] = value
-        # print('{} => {}'.format(field_name, value))
-
-        # Geocode
-        geocode_rows = geocode_map.get(street_address, [])
-        if len(geocode_rows) == 0: geocode_errors += 1
-
-        xy_map = {x['geocode_type']: x['geom'] for x in geocode_rows}
-        geocode_vals = None
-        # Geocode parcel xys
-        for geocode_type in geocode_types:
-            if geocode_type in xy_map:
-                xy_wkt = xy_map[geocode_type]
-                x, y = wkt_to_xy(xy_wkt)
-
+                    if field_type == 'number':
+                        value = None
+                    else:
+                        value = ''
+
+                summary_row[field_name] = value
+            # print('{} => {}'.format(field_name, value))
+
+            # Geocode
+            geocode_rows = geocode_map.get(street_address, [])
+            if len(geocode_rows) == 0: geocode_errors += 1
+
+            xy_map = {x['geocode_type']: x['geom'] for x in geocode_rows}
+            geocode_vals = None
+            # Geocode parcel xys
+            for geocode_type in geocode_types:
+                if geocode_type in xy_map:
+                    xy_wkt = xy_map[geocode_type]
+                    x, y = wkt_to_xy(xy_wkt)
+
+                    geocode_vals = {
+                        'geocode_type': geocode_type,
+                        'geocode_x': x,
+                        'geocode_y': y,
+                        'geocode_street_x': None,
+                        'geocode_street_y': None,
+                    }
+                    break
+            # Geocode parcel xys in street (same geocode_type as parcel xy)
+            for geocode_type in geocode_types_in_street:
+                if geocode_type in xy_map:
+                    xy_wkt = xy_map[geocode_type]
+                    x, y = wkt_to_xy(xy_wkt)
+                    #TODO: Resolve this quickfix
+                    try:
+                        geocode_vals['geocode_street_x'] = x
+                        geocode_vals['geocode_street_y'] = y
+                    except:
+                        #print("Could not get geocode_street values for: ", street_address)
+                        pass
+
+                    break
+
+            # Only write out addresses with an XY
+            if geocode_vals:
+                summary_row.update(geocode_vals)
+                summary_rows.append(summary_row)
+            elif street_address in ungeocoded_opa_addresses:
                 geocode_vals = {
-                    'geocode_type': geocode_type,
-                    'geocode_x': x,
-                    'geocode_y': y,
-                    'geocode_street_x': None,
-                    'geocode_street_y': None,
-                }
-                break
-        # Geocode parcel xys in street (same geocode_type as parcel xy)
-        for geocode_type in geocode_types_in_street:
-            if geocode_type in xy_map:
-                xy_wkt = xy_map[geocode_type]
-                x, y = wkt_to_xy(xy_wkt)
-                #TODO: Resolve this quickfix
-                try:
-                    geocode_vals['geocode_street_x'] = x
-                    geocode_vals['geocode_street_y'] = y
-                except:
-                    print("Could not get geocode_street values for: ", street_address)
-
-                break
-
-        # Only write out addresses with an XY
-        if geocode_vals:
-            summary_row.update(geocode_vals)
-            summary_rows.append(summary_row)
-        elif street_address in ungeocoded_opa_addresses:
-            geocode_vals = {
-                    'geocode_type': 'unable_to_geocode',
-                    'geocode_x': None,
-                    'geocode_y': None,
-                    'geocode_street_x': None,
-                    'geocode_street_y': None,
-                }
-            summary_row.update(geocode_vals)
-            summary_rows.append(summary_row)
-
-
-"""WRITE OUT"""
-
-if WRITE_OUT:
-    print('Writing summary rows...')
-    address_summary_table.write(summary_rows, chunk_size=100000)
-    del summary_rows
-
-    print('Creating indexes...')
-    address_summary_table.create_index('street_address')
-
-    index_stmt = '''
-		CREATE EXTENSION IF NOT EXISTS pg_trgm;
-        CREATE INDEX address_summary_opa_owners_trigram_idx ON address_summary USING GIN (opa_owners gin_trgm_ops);
-	'''
-    db.execute(index_stmt)
-    db.save()
+                        'geocode_type': 'unable_to_geocode',
+                        'geocode_x': None,
+                        'geocode_y': None,
+                        'geocode_street_x': None,
+                        'geocode_street_y': None,
+                    }
+                summary_row.update(geocode_vals)
+                summary_rows.append(summary_row)
 
-    print('Deleting temporary street name index...')
-    address_summary_table.drop_index('street_name')
 
-    print('Populating seg IDs...')
-    seg_stmt = '''
-		update address_summary asm
-		set seg_id = ast.seg_id, seg_side = ast.seg_side
-		from address_street ast
-		where ast.street_address = asm.street_address
-    '''
-    db.execute(seg_stmt)
-    db.save()
+    """WRITE OUT"""
 
-    print('Populating street codes...')
-    stcode_stmt = '''
-	    update address_summary asm
-	    set street_code = sts.street_code
-		from street_segment sts
-		where sts.seg_id = asm.seg_id
-    '''
-    db.execute(stcode_stmt)
-    db.save()
+    if WRITE_OUT:
+        print('Writing summary rows...')
+        address_summary_table.write(summary_rows, chunk_size=100000)
+        del summary_rows
 
-    # Update street codes for ranged addresses with overlapping street segs
-    rstcode_stmt = '''
-        with scnulls as (
-        select street_address, address_low, address_low_suffix, address_low_frac, street_predir, street_name, street_suffix, street_postdir
-        from address_summary asm 
-        where street_code is null and address_high is not null
-        )
-        update address_summary asm
-        set street_code = final.street_code
-        from
-        (
-        select asm.street_address, asmj.street_code 
-        from scnulls asm
-        inner join address_summary asmj on asmj.street_code is not null and asmj.address_low = asm.address_low and asmj.address_low_suffix = asm.address_low_suffix and asmj.address_low_frac = asm.address_low_frac
-        and asm.street_predir = asmj.street_predir and asm.street_name = asmj.street_name and asmj.street_suffix = asm.street_suffix and asmj.street_postdir = asm.street_postdir
-        group by asm.street_address, asmj.street_code
-        )final
-        where final.street_address = asm.street_address    
-    '''
-    db.execute(rstcode_stmt)
-    db.save()
-
-    print('Populating OPA accounts...')
-    prop_stmt = '''
-    	update address_summary asm
-    	set opa_account_num = op.account_num,
-    		opa_owners = op.owners,
-    		opa_address = op.street_address
-    	from address_property ap, opa_property op
-    	where asm.street_address = ap.street_address and
-    		ap.opa_account_num = op.account_num
-    '''
-    db.execute(prop_stmt)
-    db.save()
+        print('Creating indexes...')
+        address_summary_table.create_index('street_address')
 
-    print('Populating li_parcel_id')
-    li_pin_stmt = '''
-        with bpis as 
-        (
-            select street_address, max(value) as bin_parcel_id from
+        index_stmt = '''
+            CREATE EXTENSION IF NOT EXISTS pg_trgm;
+            CREATE INDEX address_summary_opa_owners_trigram_idx ON address_summary USING GIN (opa_owners gin_trgm_ops);
+        '''
+        db.execute(index_stmt)
+        db.save()
+
+        print('Deleting temporary street name index...')
+        address_summary_table.drop_index('street_name')
+
+        print('Populating seg IDs...')
+        seg_stmt = '''
+            update address_summary asm
+            set seg_id = ast.seg_id, seg_side = ast.seg_side
+            from address_street ast
+            where ast.street_address = asm.street_address
+        '''
+        db.execute(seg_stmt)
+        db.save()
+
+        print('Populating street codes...')
+        stcode_stmt = '''
+            update address_summary asm
+            set street_code = sts.street_code
+            from street_segment sts
+            where sts.seg_id = asm.seg_id
+        '''
+        db.execute(stcode_stmt)
+        db.save()
+
+        # Update street codes for ranged addresses with overlapping street segs
+        rstcode_stmt = '''
+            with scnulls as (
+            select street_address, address_low, address_low_suffix, address_low_frac, street_predir, street_name, street_suffix, street_postdir
+            from address_summary asm 
+            where street_code is null and address_high is not null
+            )
+            update address_summary asm
+            set street_code = final.street_code
+            from
             (
-                select street_address, value from address_tag where key = 'bin_parcel_id'
-            ) foo
-            group by street_address
+            select asm.street_address, asmj.street_code 
+            from scnulls asm
+            inner join address_summary asmj on asmj.street_code is not null and asmj.address_low = asm.address_low and asmj.address_low_suffix = asm.address_low_suffix and asmj.address_low_frac = asm.address_low_frac
+            and asm.street_predir = asmj.street_predir and asm.street_name = asmj.street_name and asmj.street_suffix = asm.street_suffix and asmj.street_postdir = asm.street_postdir
+            group by asm.street_address, asmj.street_code
+            )final
+            where final.street_address = asm.street_address    
+        '''
+        db.execute(rstcode_stmt)
+        db.save()
+        
+        print('Populating OPA accounts...')
+        prop_stmt = '''
+            update address_summary asm
+            set opa_account_num = op.account_num,
+                opa_owners = op.owners,
+                opa_address = op.street_address
+            from address_property ap, opa_property op
+            where asm.street_address = ap.street_address and
+                ap.opa_account_num = op.account_num
+        '''
+        db.execute(prop_stmt)
+        db.save()
 
-        )
-        ,
-        pwd_parcel_ids as (
-                select asum.street_address, split_part(asum.pwd_parcel_id, '|', 1) as pwd_parcel_id
+        print('Populating li_parcel_id')
+        li_pin_stmt = '''
+            with bpis as 
+            (
+                select street_address, max(value) as bin_parcel_id from
+                (
+                    select street_address, value from address_tag where key = 'bin_parcel_id'
+                ) foo
+                group by street_address
+
+            )
+            ,
+            pwd_parcel_ids as (
+                    select asum.street_address, split_part(asum.pwd_parcel_id, '|', 1) as pwd_parcel_id
+                    from address_summary asum
+                    inner join pwd_parcel pp on split_part(asum.pwd_parcel_id, '|', 1) = pp.parcel_id::text
+            )
+            ,
+            opa_account_nums as (
+                    select op.street_address, op.account_num as opa_account_num
+                    from opa_property op
+                    inner join (select street_address from address_summary) asum on asum.street_address = op.street_address
+            )
+            ,
+            choices as (
+                select distinct asum.street_address, ppis.pwd_parcel_id, ops.opa_account_num, bpis.bin_parcel_id
                 from address_summary asum
-                inner join pwd_parcel pp on split_part(asum.pwd_parcel_id, '|', 1) = pp.parcel_id::text
-        )
-        ,
-        opa_account_nums as (
-                select op.street_address, op.account_num as opa_account_num
-                from opa_property op
-                inner join (select street_address from address_summary) asum on asum.street_address = op.street_address
-        )
-		,
-        choices as (
-            select distinct asum.street_address, ppis.pwd_parcel_id, ops.opa_account_num, bpis.bin_parcel_id
-            from address_summary asum
-            left join bpis on bpis.street_address = asum.street_address
-			left join pwd_parcel_ids ppis on ppis.street_address = asum.street_address
-			left join opa_account_nums ops on ops.street_address = asum.street_address
-        )
-		,
-		address_sources as
-		(
-			select street_address, string_agg(source_name,'|') as sources
-			from (select distinct street_address, source_name from source_address where source_name not in ('AIS', 'voters', 'info_commercial', 'info_residents', 'li_eclipse_location_ids', 'li_address_keys') 
-                        order by street_address, source_name) foo
-			group by street_address
-		)
-		,
-        final as (
-        select ch.street_address, 
-                case 
-                when ch.pwd_parcel_id != '' then ch.pwd_parcel_id
-                when  ch.bin_parcel_id != '' then ch.bin_parcel_id
-                when ch.opa_account_num != '' then '-' || ch.opa_account_num
-                else ''
-            end as li_parcel_id,
-			s.sources
-            from choices ch
-			left join address_sources s on s.street_address = ch.street_address
-        )
-		update address_summary asm
-        set li_parcel_id = final.li_parcel_id
-        from final
-        where final.street_address = asm.street_address and sources is not null
+                left join bpis on bpis.street_address = asum.street_address
+                left join pwd_parcel_ids ppis on ppis.street_address = asum.street_address
+                left join opa_account_nums ops on ops.street_address = asum.street_address
+            )
+            ,
+            address_sources as
+            (
+                select street_address, string_agg(source_name,'|') as sources
+                from (select distinct street_address, source_name from source_address where source_name not in ('AIS', 'voters', 'info_commercial', 'info_residents', 'li_eclipse_location_ids', 'li_address_keys') 
+                            order by street_address, source_name) foo
+                group by street_address
+            )
+            ,
+            final as (
+            select ch.street_address, 
+                    case 
+                    when ch.pwd_parcel_id != '' then ch.pwd_parcel_id
+                    when  ch.bin_parcel_id != '' then ch.bin_parcel_id
+                    when ch.opa_account_num != '' then '-' || ch.opa_account_num
+                    else ''
+                end as li_parcel_id,
+                s.sources
+                from choices ch
+                left join address_sources s on s.street_address = ch.street_address
+            )
+            update address_summary asm
+            set li_parcel_id = final.li_parcel_id
+            from final
+            where final.street_address = asm.street_address and sources is not null
+        '''
+        db.execute(li_pin_stmt)
+        db.save()
+
+    # Insert ungeocoded opa addresses into geocode table with null geoms:
+    print("Inserting ungeocoded opa addresses into geocode table with null geom...")
+    stmt = '''
+        insert into geocode (street_address, geocode_type) values ('{street_address}', 99)
     '''
-    db.execute(li_pin_stmt)
+    for street_address in ungeocoded_opa_addresses:
+        db.execute(stmt.format(street_address=street_address))
     db.save()
+    db.close()
+
+    print('{} geocode errors'.format(geocode_errors))
+    print('Finished in {} seconds'.format(datetime.now() - start))
 
-# Insert ungeocoded opa addresses into geocode table with null geoms:
-print("Inserting ungeocoded opa addresses into geocode table with null geom...")
-stmt = '''
-    insert into geocode (street_address, geocode_type) values ('{street_address}', 99)
-'''
-for street_address in ungeocoded_opa_addresses:
-    db.execute(stmt.format(street_address=street_address))
-db.save()
-db.close()
-
-print('{} geocode errors'.format(geocode_errors))
-print('Finished in {} seconds'.format(datetime.now() - start))
diff --git a/ais/engine/scripts/make_linked_tags.py b/ais/engine/scripts/make_linked_tags.py
index 6261104c..73debf50 100644
--- a/ais/engine/scripts/make_linked_tags.py
+++ b/ais/engine/scripts/make_linked_tags.py
@@ -1,365 +1,367 @@
 from datetime import datetime
 import datum
-
 from ais import app
+import petl as etl
+import os
+import psutil
+
+def copy(db, out_file: str, query: str): 
+    '''Run a postgres COPY command with the database and query specified to out_file'''
+    with open(out_file, 'w') as out_file: 
+        db._c.copy_expert(query, out_file)
+
+def cleanup(filename: str): 
+    '''Print the virtual memory consumption and remove the downloaded file'''
+    print(psutil.virtual_memory())
+    os.remove(filename)
+    print(f'Removed {filename}')
+
+def create_map(rows, map_name: str, column_name: str, map_dict: dict) -> dict: 
+    '''
+    Read a csv and create a dictionary mapping where the keys are the unique 
+    values in column "column_name" and the values are a list of matching table rows.
+    To append to an existing mapping, pass it to map_dict.
+    '''
+    i = -1
+    for i, row in enumerate(rows):
+        value = row[column_name]
+        if not value in map_dict:
+            map_dict[value] = []
+        map_dict[value].append(row)
+    if i > -1: # Used to prevent an error if rows == []
+        print(f"length {map_name}: {len(map_dict)}. Total values looped through: {i + 1}")
+    return map_dict
+
+def main():
+    WRITE_OUT = True
+
+    start = datetime.now()
+    print('Starting at ', start)
+
+    config = app.config
+    Parser = config['PARSER']
+    parser = Parser()
+    db = datum.connect(config['DATABASES']['engine'])
+    address_tag_table = db['address_tag']
+    tag_fields = config['ADDRESS_SUMMARY']['tag_fields']
+    geocode_where = "WHERE geocode_type in (1,2)"
+
+    print('Deleting linked tags...')
+    del_stmt = '''
+        Delete from address_tag where linked_address != ''
+    '''
+    db.execute(del_stmt)
+    db.save()
+
+    print('Reading address links...')
+    address_link_file = 'address_link.csv'
+    query = f"copy (select * from address_link ORDER BY ADDRESS_1, ADDRESS_2) TO STDOUT WITH CSV HEADER;"
+    copy(db, address_link_file, query)
+    link_map = create_map(
+        rows=etl.fromcsv(address_link_file).dicts(), 
+        map_name='link_map', column_name='address_1', map_dict={})
+    cleanup(address_link_file)
+
+    print('Reading address tags')
+    address_tag_file = 'address_tag.csv'
+    query = f"copy (select * from address_tag) TO STDOUT WITH CSV HEADER;"
+    copy(db, address_tag_file, query)
+    tag_map = create_map(
+        rows=etl.fromcsv(address_tag_file).dicts(), 
+        map_name='tag_map', column_name='street_address', map_dict={})
+    cleanup(address_tag_file)
+
+    print('Reading geocode rows...')
+    geocode_file = 'geocode.csv'
+    query = f"copy (select * from geocode {geocode_where}) TO STDOUT WITH CSV HEADER;"
+    copy(db, geocode_file, query)
+    
+    geocode_map = {}
+    rows = etl.fromcsv(geocode_file).convert('geocode_type', int).dicts()
+    for row in rows:
+        street_address = row['street_address']
+        if not street_address in geocode_map:
+            geocode_map[street_address] = {'pwd': '', 'dor': ''}
+        if row['geocode_type'] == 1:
+            geocode_map[street_address]['pwd'] = row['geom']
+        else:
+            geocode_map[street_address]['dor'] = row['geom']
+    print(f"length geocode_map: {len(geocode_map)}")
+    cleanup(geocode_file)
+    
+    #define traversal order
+    traversal_order = ['has generic unit', 'matches unit', 'has base', 'overlaps', 'in range']
+    
+    print('Reading addresses...')
+    address_file = 'address.csv'
+    query = f"copy (select * from address) TO STDOUT WITH CSV HEADER;"
+    copy(db, address_file, query)
+    address_rows = etl.fromcsv(address_file).dicts()
+
+    print('Making linked tags...')
+    linked_tags_map = []
+    new_linked_tags = []
+    i = 1
+    done = False
+
+    rejected_link_map = {}
+
+    while not done:
+
+        print("Linked tags iteration: ", i)
+
+        # add new tags to tag map
+        tag_map = create_map(
+            rows=new_linked_tags, map_name='tag_map', column_name='street_address', 
+            map_dict=tag_map)
+
+        new_linked_tags = []
+        # loop through addresses
+        for address_row in address_rows:
+            street_address = address_row['street_address']
+            # get address tags associated with street address
+            mapped_tags = tag_map.get(street_address)
+            links = link_map.get(street_address)
+            # sort links by traversal order
+            sorted_links = []
+            if links:
+                for rel in traversal_order:
+                    for link in links:
+                        if link['relationship'] == rel:
+                            sorted_links.append(link)
+                            # links.remove(link)
+            # loop through tag fields in config
+            for tag_field in tag_fields:
+                found = False
+                # Skip tag_fields where 'traverse_links' value is false
+                if tag_field['traverse_links'] != 'true':
+                    continue
+                tag_key = tag_field['tag_key']
+                # Look for tag in tag_map for street_address
+                tag_value = None
+                # Check if already have value for this tag
+                if mapped_tags:
+                    mapped_tags_for_key = [mapped_tag for mapped_tag in mapped_tags if mapped_tag.get('key', '') == tag_key]
+                    first_mapped_tag_for_key = sorted(mapped_tags_for_key, key=lambda s: s['value'])[0] if mapped_tags_for_key else None
+                    if first_mapped_tag_for_key:
+                        tag_value = first_mapped_tag_for_key['value']
+                        found = True
+                # Otherwise, look for tag in address links
+                if not links or found:
+                    # Do something if tag can't be found by traversing links so API doesn't look for it
+                    continue
+                # loop through links
+                for slink in sorted_links:
+                    if found == True:
+                        break
+                    link_address = slink.get('address_2')
+                    # Don't allow tags from links with different non-null pwd or dor geocoded geoms:
+                    if all(a in geocode_map for a in (link_address, street_address)):
+                        # TODO: different constraints based on tag type (i.e. dor/pwd ids)
+                        # if either parcel geocodes have different geoms don't inherit:
+                        if (geocode_map[link_address]['pwd'] is not None and geocode_map[link_address]['pwd'] != geocode_map[street_address]['pwd']) and \
+                        (geocode_map[link_address]['dor'] is not None and geocode_map[link_address]['dor'] != geocode_map[street_address]['dor']):
+                            if street_address not in rejected_link_map:
+                                rejected_link_map[street_address] = []
+                            rejected_link_map[street_address].append(link_address)
+                            continue
 
-WRITE_OUT = True
-
-start = datetime.now()
-print('Starting at ', start)
-
-config = app.config
-Parser = config['PARSER']
-parser = Parser()
-db = datum.connect(config['DATABASES']['engine'])
-address_table = db['address']
-address_tag_table = db['address_tag']
-source_address_table = db['source_address']
-address_link_table = db['address_link']
-tag_fields = config['ADDRESS_SUMMARY']['tag_fields']
-geocode_table = db['geocode']
-geocode_where = "geocode_type in (1,2)"
-
-
-print('Deleting linked tags...')
-del_stmt = '''
-    Delete from address_tag where linked_address != ''
-'''
-db.execute(del_stmt)
-db.save()
-
-print('Reading address links...')
-link_map = {}
-link_rows = address_link_table.read(sort=['ADDRESS_1', 'ADDRESS_2'])
-for link_row in link_rows:
-    address_1 = link_row['address_1']
-    if not address_1 in link_map:
-        link_map[address_1] = []
-    link_map[address_1].append(link_row)
-
-#define traversal order
-traversal_order = ['has generic unit', 'matches unit', 'has base', 'overlaps', 'in range']
-
-print('Reading address tags...')
-tag_map = {}
-tag_rows = address_tag_table.read()
-for tag_row in tag_rows:
-    street_address = tag_row['street_address']
-    if not street_address in tag_map:
-        tag_map[street_address] = []
-    tag_map[street_address].append(tag_row)
-
-err_map = {}
-
-print('Reading geocode rows...')
-geocode_map = {}
-geocode_rows = geocode_table.read(where=geocode_where)
-print('Mapping geocode rows...')
-for geocode_row in geocode_rows:
-    street_address = geocode_row['street_address']
-    geocode_type = geocode_row['geocode_type']
-    if not street_address in geocode_map:
-        geocode_map[street_address] = {'pwd': '', 'dor': ''}
-    if geocode_row['geocode_type'] == 1:
-        geocode_map[street_address]['pwd'] = geocode_row['geom']
-    else:
-        geocode_map[street_address]['dor'] = geocode_row['geom']
-
-print('Reading addresses...')
-address_rows = address_table.read()
-print('Making linked tags...')
-linked_tags_map = []
-new_linked_tags = []
-i = 1
-done = False
-
-rejected_link_map = {}
-
-while not done:
-
-    print("Linked tags iteration: ", i)
-
-    # add new tags to tag map
-    for new_tag_row in new_linked_tags:
-        street_address = new_tag_row['street_address']
-        if not street_address in tag_map:
-            tag_map[street_address] = []
-        tag_map[street_address].append(new_tag_row)
-
+                    # get tags for current link
+                    link_tags = tag_map.get(link_address)
+                    if link_tags:
+                        link_tags_for_key = [link_tag for link_tag in link_tags if link_tag.get('key', '') == tag_key]
+                        first_link_tag_for_key = sorted(link_tags_for_key, key=lambda s: s['value'])[0] if link_tags_for_key else None
+                        if first_link_tag_for_key:
+                            tag_value = first_link_tag_for_key['value']
+                            link_path = slink['relationship']
+                            linked_path = first_link_tag_for_key['linked_path'] if first_link_tag_for_key['linked_path'] else link_address
+                            linked_address = first_link_tag_for_key['linked_address'] if first_link_tag_for_key['linked_address'] else link_address
+                            linked_path = street_address + ' ' + link_path + ' ' + linked_path
+                            add_tag_dict = {'street_address': street_address, 'key': tag_key, 'value': tag_value,
+                                            'linked_address': linked_address, 'linked_path': linked_path}
+                            new_linked_tags.append(add_tag_dict)
+                            found = True
+
+        if len(new_linked_tags) > 0:
+            linked_tags_map = linked_tags_map + new_linked_tags
+        else:
+            done = 'done'
+        i += 1
+
+    """WRITE OUT"""
+
+    if WRITE_OUT:
+        print('Writing ', len(linked_tags_map), ' linked tags to address_tag table...')
+        address_tag_table.write(linked_tags_map, chunk_size=150000)
+        print('Rejected links: ')
+        for key, value in rejected_link_map.items():
+            value=list(set(value))
+            print('{key}: {value}'.format(key=key, value=value))
+
+    # Finally, loop through addresses one last time checking for tags with keys not in tag table, and for each tag lookup
+    # tag linked_addresses in address_link table address_2 for street_address having unit type & num matching the current
+    # loop address.
+    print("Searching for linked tags via path: has base in range unit child")
     new_linked_tags = []
-    # loop through addresses
+
+    print("Reading addresses...")
+    address_file = 'address.csv'
+    query = f"copy (select * from address where unit_num != '' order by street_address) TO STDOUT WITH CSV HEADER;"
+    copy(db, address_file, query)
+    address_rows = etl.fromcsv(address_file).convert('address_low', int).dicts()
+
+    print('Reading address tags...')
+    tag_map = {}
+    tag_sel_stmt = '''
+        select a.*, t.key, t.value, t.linked_address, t.linked_path
+        from (
+          select street_address
+          from address
+          where unit_num != '')
+          a
+        left join address_tag t on t.street_address = a.street_address
+        order by a.street_address, t.key, t.value
+    '''
+    tag_rows = db.execute(tag_sel_stmt)
+    tag_map = create_map(
+        rows=tag_rows, map_name='tag_map', column_name='street_address', map_dict={})
+
+    print('Reading address links...')
+    link_sel_stmt = '''
+        select al.*
+        from (
+            SELECT *
+            from address
+            where address_high is not Null) a
+        inner join address_link al on al.address_2 = a.street_address
+        where relationship = 'has base'
+        order by address_1
+    '''
+    link_rows = db.execute(link_sel_stmt) # Change to copy
+    link_map = create_map(
+        rows=link_rows, map_name='link_map', column_name='address_2', map_dict={})
+
+    i=0
+    rejected_link_map = {}
+    print('Looping through {} addresses...'.format(len(address_rows))) # Remove this
     for address_row in address_rows:
+        i+=1
+        unit_num = address_row['unit_num']
         street_address = address_row['street_address']
+        low_num = address_row['address_low']
+        unit_type = address_row['unit_type']
+        street_predir = address_row['street_predir']
+        street_name = address_row['street_name']
+        street_suffix = address_row['street_suffix']
+        street_postdir = address_row['street_postdir']
+        low_num = low_num if low_num else ''
+        unit_type = unit_type if unit_type else ''
+        unit_num = unit_num if unit_num else ''
+        street_name = street_name if street_name else ''
+        street_predir = street_predir if street_predir else ''
+        street_suffix = street_suffix if street_suffix else ''
+        street_postdir = street_postdir if street_postdir else ''
+
         # get address tags associated with street address
         mapped_tags = tag_map.get(street_address)
-        links = link_map.get(street_address)
-        # sort links by traversal order
-        sorted_links = []
-        if links:
-            for rel in traversal_order:
-                for link in links:
-                    if link['relationship'] == rel:
-                        sorted_links.append(link)
-                        # links.remove(link)
-        # loop through tag fields in config
+        tag_fields = [tag_field for tag_field in tag_fields if tag_field['traverse_links'] == 'true']
         for tag_field in tag_fields:
             found = False
-            # Skip tag_fields where 'traverse_links' value is false
-            if tag_field['traverse_links'] != 'true':
-                continue
             tag_key = tag_field['tag_key']
             # Look for tag in tag_map for street_address
             tag_value = None
-            # Check if already have value for this tag
             if mapped_tags:
-                mapped_tags_for_key = [mapped_tag for mapped_tag in mapped_tags if mapped_tag.get('key', '') == tag_key]
-                first_mapped_tag_for_key = sorted(mapped_tags_for_key, key=lambda s: s['value'])[0] if mapped_tags_for_key else None
-                if first_mapped_tag_for_key:
-                    tag_value = first_mapped_tag_for_key['value']
-                    found = True
-            # Otherwise, look for tag in address links
-            if not links or found:
-                # Do something if tag can't be found by traversing links so API doesn't look for it
+                for mapped_tag in mapped_tags:
+                    mapped_key = mapped_tag.get('key')
+                # if street address has this tag already, continue to next tag_field
+                    if mapped_key == tag_key:
+                        found = True
+                        break
+            else:
+                continue
+            if found: # already have tag, go to next
+                continue
+            # Get set of linked addresses from address tags
+            linked_addresses = set([(tag['linked_address'], tag['linked_path']) for tag in mapped_tags])
+            if not linked_addresses:
                 continue
-            # loop through links
-            for slink in sorted_links:
-                if found == True:
+            # Loop through linked address links looking for relationship = 'has_base'
+            for linked_address, linked_path in linked_addresses:
+                if linked_address == None:
+                    continue
+                if found:
                     break
-                link_address = slink.get('address_2')
-                # Don't allow tags from links with different non-null pwd or dor geocoded geoms:
-                if all(a in geocode_map for a in (link_address, street_address)):
-                    # TODO: different constraints based on tag type (i.e. dor/pwd ids)
-                    # if either parcel geocodes have different geoms don't inherit:
-                    if (geocode_map[link_address]['pwd'] is not None and geocode_map[link_address]['pwd'] != geocode_map[street_address]['pwd']) and \
-                    (geocode_map[link_address]['dor'] is not None and geocode_map[link_address]['dor'] != geocode_map[street_address]['dor']):
-                        if street_address not in rejected_link_map:
-                            rejected_link_map[street_address] = []
-                        rejected_link_map[street_address].append(link_address)
-                        continue
-
-                # get tags for current link
-                link_tags = tag_map.get(link_address)
-                if link_tags:
-                    link_tags_for_key = [link_tag for link_tag in link_tags if link_tag.get('key', '') == tag_key]
-                    first_link_tag_for_key = sorted(link_tags_for_key, key=lambda s: s['value'])[0] if link_tags_for_key else None
-                    if first_link_tag_for_key:
-                        tag_value = first_link_tag_for_key['value']
-                        link_path = slink['relationship']
-                        linked_path = first_link_tag_for_key['linked_path'] if first_link_tag_for_key['linked_path'] else link_address
-                        linked_address = first_link_tag_for_key['linked_address'] if first_link_tag_for_key['linked_address'] else link_address
-                        linked_path = street_address + ' ' + link_path + ' ' + linked_path
-                        add_tag_dict = {'street_address': street_address, 'key': tag_key, 'value': tag_value,
-                                        'linked_address': linked_address, 'linked_path': linked_path}
-                        new_linked_tags.append(add_tag_dict)
-                        found = True
-
-    if len(new_linked_tags) > 0:
-        linked_tags_map = linked_tags_map + new_linked_tags
-    else:
-        done = 'done'
-    i += 1
-
-"""WRITE OUT"""
+                links = link_map.get(linked_address)
+                if not links:
+                    continue
+                for link in links:
+                    if found:
+                        break
+                    if link.get('relationship') == 'has base':
+                        l_street_address = link['address_1']
+                        parsed = parser.parse(l_street_address)
+                        if all(a in geocode_map for a in (l_street_address, linked_address)):
+                            # if both parcel geocodes have different geoms don't use:
+                            if (geocode_map[l_street_address]['pwd'] is not None and geocode_map[l_street_address]['pwd'] !=
+                                geocode_map[linked_address]['pwd']) and \
+                                    (geocode_map[l_street_address]['dor'] is not None and geocode_map[l_street_address]['dor'] !=
+                                        geocode_map[linked_address]['dor']):
+                                if linked_address not in rejected_link_map:
+                                    rejected_link_map[linked_address] = []
+                                rejected_link_map[linked_address].append(l_street_address)
+                                continue
+
+                        l_low_num = parsed['components']['address']['low_num']
+                        l_high_num = parsed['components']['address']['high_num_full']
+                        l_street_full = parsed['components']['street']['full']
+                        l_unit_type = parsed['components']['address_unit']['unit_type']
+                        l_unit_num = parsed['components']['address_unit']['unit_num']
+                        l_street_name = parsed['components']['street']['name']
+                        l_street_predir = parsed['components']['street']['predir']
+                        l_street_suffix = parsed['components']['street']['suffix']
+                        l_street_postdir = parsed['components']['street']['postdir']
+                        l_low_num = l_low_num if l_low_num else ''
+                        l_street_full = l_street_full if l_street_full else ''
+                        l_unit_type = l_unit_type if l_unit_type else ''
+                        l_unit_num = l_unit_num if l_unit_num else ''
+                        l_street_name = l_street_name if l_street_name else ''
+                        l_street_predir = l_street_predir if l_street_predir else ''
+                        l_street_suffix = l_street_suffix if l_street_suffix else ''
+                        l_street_postdir = l_street_postdir if l_street_postdir else ''
+
+                        # Condition for -- has base in range unit child -- search:
+                        if l_low_num == low_num and l_street_predir == street_predir and l_street_name == street_name \
+                            and l_street_suffix == street_suffix and l_street_postdir == street_postdir \
+                                and l_unit_type == unit_type and l_unit_num == unit_num:
+                            # Search tag map for this address and see if it has the missing key
+                            link_tags = tag_map.get(link['address_1'])
+                            for link_tag in link_tags:
+                                mapped_key = link_tag['key']
+                                # if street address has this tag already, continue to next tag_field
+                                if mapped_key == tag_key:
+                                    tag_value = link_tag['value']
+                                    #linked_path = link_tag['linked_path'] if link_tag['linked_path'] else linked_address
+                                    linked_address = link['address_1']
+                                    linked_path = linked_path + ' unit child ' + linked_address
+                                    add_tag_dict = {'street_address': street_address, 'key': tag_key,
+                                                    'value': tag_value,
+                                                    'linked_address': linked_address, 'linked_path': linked_path}
+                                    new_linked_tags.append(add_tag_dict)
+                                    found = True
+                                    break
+
+    """WRITE OUT"""
+
+    if WRITE_OUT and len(new_linked_tags) > 0:
+        print('Writing ', len(new_linked_tags), ' linked tags to address_tag table...')
+        address_tag_table.write(new_linked_tags, chunk_size=150000)
 
-if WRITE_OUT:
-    print('Writing ', len(linked_tags_map), ' linked tags to address_tag table...')
-    address_tag_table.write(linked_tags_map, chunk_size=150000)
     print('Rejected links: ')
     for key, value in rejected_link_map.items():
         value=list(set(value))
         print('{key}: {value}'.format(key=key, value=value))
 
-# Finally, loop through addresses one last time checking for tags with keys not in tag table, and for each tag lookup
-# tag linked_addresses in address_link table address_2 for street_address having unit type & num matching the current
-# loop address.
-print("Searching for linked tags via path: has base in range unit child")
-new_linked_tags = []
-debugz = False
-
-print("Reading addresses...")
-#del address_rows
-where = "unit_num != ''"
-sort = "street_address"
-address_rows = address_table.read(where=where, sort=sort)
-
-print('Reading address tags...')
-tag_map = {}
-#tag_rows = address_tag_table.read()
-tag_sel_stmt = '''
-    select a.*, t.key, t.value, t.linked_address, t.linked_path
-    from (
-      select street_address
-      from address
-      where unit_num != '')
-      a
-    left join address_tag t on t.street_address = a.street_address
-    order by a.street_address, t.key, t.value
-'''
-tag_rows = db.execute(tag_sel_stmt)
-for tag_row in tag_rows:
-    street_address = tag_row['street_address']
-    if not street_address in tag_map:
-        tag_map[street_address] = []
-    tag_map[street_address].append(tag_row)
-
-print('Reading address links...')
-link_map = {}
-link_sel_stmt = '''
-    select al.*
-    from (
-        SELECT *
-        from address
-        where address_high is not Null) a
-    inner join address_link al on al.address_2 = a.street_address
-    where relationship = 'has base'
-    order by address_1
-'''
-# where = "relationship = 'has base'"
-# sort = "address_2"
-# link_rows = address_link_table.read(where=where, sort=sort)
-link_rows = db.execute(link_sel_stmt)
-for link_row in link_rows:
-    address_2 = link_row['address_2']
-    if not address_2 in link_map:
-        link_map[address_2] = []
-    link_map[address_2].append(link_row)
-
-i=0
-rejected_link_map = {}
-print('Looping through {} addresses...'.format(len(address_rows)))
-for address_row in address_rows:
-    i+=1
-    if i % 10000 == 0:
-        print(i)
-    unit_num = address_row['unit_num']
-    street_address = address_row['street_address']
-    low_num = address_row['address_low']
-    unit_type = address_row['unit_type']
-    street_predir = address_row['street_predir']
-    street_name = address_row['street_name']
-    street_suffix = address_row['street_suffix']
-    street_postdir = address_row['street_postdir']
-    low_num = low_num if low_num else ''
-    unit_type = unit_type if unit_type else ''
-    unit_num = unit_num if unit_num else ''
-    street_name = street_name if street_name else ''
-    street_predir = street_predir if street_predir else ''
-    street_suffix = street_suffix if street_suffix else ''
-    street_postdir = street_postdir if street_postdir else ''
-
-    # get address tags associated with street address
-    mapped_tags = tag_map.get(street_address)
-    tag_fields = [tag_field for tag_field in tag_fields if tag_field['traverse_links'] == 'true']
-    for tag_field in tag_fields:
-        found = False
-        # Skip tag_fields where 'traverse_links' value is false
-        # if tag_field['traverse_links'] != 'true':
-        #     continue
-        tag_key = tag_field['tag_key']
-        # Look for tag in tag_map for street_address
-        tag_value = None
-        if mapped_tags:
-            for mapped_tag in mapped_tags:
-                mapped_key = mapped_tag.get('key')
-            # if street address has this tag already, continue to next tag_field
-                if mapped_key == tag_key:
-                    found = True
-                    break
-        else:
-            continue
-        if found: # already have tag, go to next
-            continue
-        # Get set of linked addresses from address tags
-        linked_addresses = set([(tag['linked_address'], tag['linked_path']) for tag in mapped_tags])
-        if not linked_addresses:
-            continue
-        # Loop through linked address links looking for relationship = 'has_base'
-        for linked_address, linked_path in linked_addresses:
-            if linked_address == None:
-                continue
-            if found:
-                break
-            links = link_map.get(linked_address)
-            if not links:
-                continue
-            for link in links:
-                if found:
-                    break
-                if link.get('relationship') == 'has base':
-                    l_street_address = link['address_1']
-                    parsed = parser.parse(l_street_address)
-                    if all(a in geocode_map for a in (l_street_address, linked_address)):
-                        # if both parcel geocodes have different geoms don't use:
-                        if (geocode_map[l_street_address]['pwd'] is not None and geocode_map[l_street_address]['pwd'] !=
-                            geocode_map[linked_address]['pwd']) and \
-                                (geocode_map[l_street_address]['dor'] is not None and geocode_map[l_street_address]['dor'] !=
-                                    geocode_map[linked_address]['dor']):
-                            if linked_address not in rejected_link_map:
-                                rejected_link_map[linked_address] = []
-                            rejected_link_map[linked_address].append(l_street_address)
-                            continue
-
-                    l_low_num = parsed['components']['address']['low_num']
-                    l_high_num = parsed['components']['address']['high_num_full']
-                    l_street_full = parsed['components']['street']['full']
-                    l_unit_type = parsed['components']['address_unit']['unit_type']
-                    l_unit_num = parsed['components']['address_unit']['unit_num']
-                    l_street_name = parsed['components']['street']['name']
-                    l_street_predir = parsed['components']['street']['predir']
-                    l_street_suffix = parsed['components']['street']['suffix']
-                    l_street_postdir = parsed['components']['street']['postdir']
-                    l_low_num = l_low_num if l_low_num else ''
-                    l_street_full = l_street_full if l_street_full else ''
-                    l_unit_type = l_unit_type if l_unit_type else ''
-                    l_unit_num = l_unit_num if l_unit_num else ''
-                    l_street_name = l_street_name if l_street_name else ''
-                    l_street_predir = l_street_predir if l_street_predir else ''
-                    l_street_suffix = l_street_suffix if l_street_suffix else ''
-                    l_street_postdir = l_street_postdir if l_street_postdir else ''
-
-                    # Condition for -- has base in range unit child -- search:
-                    if l_low_num == low_num and l_street_predir == street_predir and l_street_name == street_name \
-                        and l_street_suffix == street_suffix and l_street_postdir == street_postdir \
-                            and l_unit_type == unit_type and l_unit_num == unit_num:
-                        # Search tag map for this address and see if it has the missing key
-                        link_tags = tag_map.get(link['address_1'])
-                        for link_tag in link_tags:
-                            mapped_key = link_tag['key']
-                            # if street address has this tag already, continue to next tag_field
-                            if mapped_key == tag_key:
-                                tag_value = link_tag['value']
-                                #linked_path = link_tag['linked_path'] if link_tag['linked_path'] else linked_address
-                                linked_address = link['address_1']
-                                linked_path = linked_path + ' unit child ' + linked_address
-                                add_tag_dict = {'street_address': street_address, 'key': tag_key,
-                                                'value': tag_value,
-                                                'linked_address': linked_address, 'linked_path': linked_path}
-                                #print(street_address)
-                                #print("ADDING NEW TAG: ", add_tag_dict)
-                                new_linked_tags.append(add_tag_dict)
-                                #print("new linked tags: ", len(new_linked_tags))
-                                found = True
-                                break
-
-"""WRITE OUT"""
-
-if WRITE_OUT and len(new_linked_tags) > 0:
-    print('Writing ', len(new_linked_tags), ' linked tags to address_tag table...')
-    address_tag_table.write(new_linked_tags, chunk_size=150000)
-
-print('Rejected links: ')
-for key, value in rejected_link_map.items():
-    value = list(set(value))
-    print('{key}: {value}'.format(key=key, value=value))
-# print(rejected_link_map)
-
-print("Cleaning up...")
-del link_rows
-del tag_rows
-del address_rows
-del link_map
-del tag_map
-# del linked_tags_map
-
-transpired = datetime.now() - start
-print("Finished in ", transpired, " minutes.")
+    cleanup(address_file)
+    
+    transpired = datetime.now() - start
+    print("Finished in ", transpired, " minutes.")
diff --git a/ais/engine/scripts/make_service_area_summary.py b/ais/engine/scripts/make_service_area_summary.py
index 6fe44415..08bb7e3b 100644
--- a/ais/engine/scripts/make_service_area_summary.py
+++ b/ais/engine/scripts/make_service_area_summary.py
@@ -11,384 +11,385 @@
 import traceback
 from pprint import pprint
 
-
-print('Starting...')
-start = datetime.now()
-
-"""
-TODO
-- This might perform better if we do one big spatial join at the beginning
-  between address summary and service area polygons.
-"""
-
-"""SET UP"""
-config = app.config
-db = datum.connect(config['DATABASES']['engine'])
-sa_layer_defs = config['SERVICE_AREAS']['layers']
-sa_layer_ids = [x['layer_id'] for x in sa_layer_defs]
-poly_table = db['service_area_polygon']
-line_single_table = db['service_area_line_single']
-line_dual_table = db['service_area_line_dual']
-point_table = db['service_area_point']
-#sa_summary_table = db['service_area_summary']
-address_summary_table = db['address_summary']
-address_summary_fields = [
-	'street_address',
-	'geocode_x',
-	'geocode_y',
-	# 'seg_id',
-	# 'seg_side',
-]
-sa_summary_fields = [{'name': 'street_address', 'type': 'text'}]
-sa_summary_fields += [{'name': x, 'type': 'text'} for x in sa_layer_ids]
-sa_summary_row_template = {x: '' for x in sa_layer_ids}
-
-# DEV
-WRITE_OUT = True
-
-# Keep poly rows in memory so we make less trips to the database for overlapping
-# points.
-# xy_map = {}  # x => y => [sa_poly_rows]
-
-"""MAIN"""
-#
-if WRITE_OUT:
-	print('Dropping service area summary table...')
-	db.drop_table('service_area_summary')
-
-	print('Creating service area summary table...')
-	db.create_table('service_area_summary', sa_summary_fields)
-
-sa_summary_table = db['service_area_summary']
-
-# print('Reading single-value service area lines...')
-# line_single_map = {}  # layer_id => seg_id => value
-# line_singles = ais_db.read(line_single_table, ['*'])
-
-# for line_single in line_singles:
-# 	layer_id = line_single['layer_id']
-# 	seg_id = line_single['seg_id']
-# 	value = line_single['value']
-# 	if layer_id not in line_single_map:
-# 		line_single_map[layer_id] = {}
-# 	line_single_map[layer_id][seg_id] = value
-
-# print('Reading dual-value service area lines...')
-# line_dual_map = {}  # layer_id => seg_id => value
-# line_duals = ais_db.read(line_dual_table, ['*'])
-
-# for line_dual in line_duals:
-# 	layer_id = line_dual['layer_id']
-# 	seg_id = line_dual['seg_id']
-# 	left_value = line_dual['left_value']
-# 	right_value = line_dual['right_value']
-# 	if layer_id not in line_dual_map:
-# 		line_dual_map[layer_id] = {}
-
-# 	line_dual_map[layer_id][seg_id] = {}
-# 	line_dual_map[layer_id][seg_id]['left'] = left_value
-# 	line_dual_map[layer_id][seg_id]['right'] = right_value
-
-print('Reading address summary...')
-address_summary_rows = address_summary_table.read(\
-	fields=address_summary_fields, \
-	sort=['geocode_x', 'geocode_y']\
-)
-
-sa_summary_rows = []
-
-# Sort address summary rows by X, Y and use these to compare the last row
-# to the current one. This minimizes trips to the database for poly values.
-last_x = None
-last_y = None
-last_sa_rows = None
-#
-print('Intersecting addresses and service area polygons...')
-for i, address_summary_row in enumerate(address_summary_rows):
-	try:
-		if i % 10000 == 0:
-			print(i)
-
-			# Write in chunks
-			if WRITE_OUT: #and i % 50000 == 0:
-				sa_summary_table.write(sa_summary_rows)
-				sa_summary_rows = []
-
-		# Get attributes
-		street_address = address_summary_row['street_address']
-		# seg_id = address_summary_row['seg_id']
-		# seg_side = address_summary_row['seg_side']
-		x = address_summary_row['geocode_x']
-		y = address_summary_row['geocode_y']
-
-		sa_rows = None
-		# if x in xy_map:
-		# 	y_map = xy_map[x]
-		# 	if y in y_map:
-		# 		sa_rows = y_map[y]
-		if last_x and (last_x == x and last_y == y):
-			sa_rows = last_sa_rows
-
-		if sa_rows is None and None not in (x,y):
-			# Get intersecting service areas
-			where = 'ST_Intersects(geom, ST_SetSrid(ST_Point({}, {}), 2272))'.format(x, y)
-			sa_rows = poly_table.read(fields=['layer_id', 'value'], where=where, return_geom=False)
-
-			# Add to map
-			# x_map = xy_map[x] = {}
-			# x_map[y] = sa_rows
-
-		# Create and insert summary row
-		sa_summary_row = deepcopy(sa_summary_row_template)
-		sa_summary_row['street_address'] = street_address
-		if sa_rows: 
-			update_dict = {}
-			for x in sa_rows: 
-				if update_dict.get(x['layer_id']) == None: 
-					update_dict[x['layer_id']] = []
-				update_dict[x['layer_id']].append(x['value'])
-			for layer_id, _ in update_dict.items(): 
-				update_dict[layer_id].sort()
-				update_dict[layer_id] = '|'.join(update_dict[layer_id])
-			sa_summary_row.update(update_dict)
-
-		sa_summary_rows.append(sa_summary_row)
-
-		last_x = x
-		last_y = y
-		last_sa_rows = sa_rows
-
-	except:
-		print(traceback.format_exc())
-		sys.exit()
-
-# Clear out XY map
-# xy_map = {}
-
-if WRITE_OUT:
-	print('Writing service area summary rows...')
-	sa_summary_table.write(sa_summary_rows)
-	del sa_summary_rows
-
-# # Update where method = yes_or_no:
-# for sa_layer_def in sa_layer_defs:
-# 	layer_id = sa_layer_def['layer_id']
-# 	if 'polygon' in sa_layer_def['sources']:
-# 		method = sa_layer_def['sources']['polygon'].get('method')
-# 		if method == 'yes_or_no':
-# 			stmt = '''
-# 					UPDATE service_area_summary sas
-# 					SET {layer_id} = (
-# 					CASE
-# 					WHEN {layer_id} != '' THEN 'yes'
-# 					ELSE 'no'
-# 					END);
-# 					'''.format(layer_id=layer_id)
-# 			db.execute(stmt)
-# 			# print(ais_db.c.rowcount)
-# 			db.save()
-################################################################################
-# SERVICE AREA LINES
-################################################################################
-
-if WRITE_OUT:
-	print('\n** SERVICE AREA LINES ***\n')
-	print('Creating indexes...')
-	sa_summary_table.create_index('street_address')
-
-	print('Creating temporary indexes...')
-	address_summary_table.create_index('seg_id')
-
-	for sa_layer_def in sa_layer_defs:
-		layer_id = sa_layer_def['layer_id']
-
-		if 'line_single' in sa_layer_def['sources']:
-			print('Updating from {}...'.format(layer_id))
-			stmt = '''
-				UPDATE service_area_summary sas
-				SET {layer_id} = sals.value
-				FROM address_summary ads, service_area_line_single sals
-				WHERE
-					sas.street_address = ads.street_address AND
-					sals.seg_id = ads.seg_id AND
-					sals.layer_id = '{layer_id}' AND
-					sals.value <> ''
-			'''.format(layer_id=layer_id)
-			db.execute(stmt)
-			# print(ais_db.c.rowcount)
-			db.save()
-
-		elif 'line_dual' in sa_layer_def['sources']:
-			print('Updating from {}...'.format(layer_id))
-			stmt = '''
-				UPDATE service_area_summary sas
-				SET {layer_id} = CASE WHEN (ads.seg_side = 'L') THEN sald.left_value ELSE sald.right_value END
-				FROM address_summary ads, service_area_line_dual sald
-				WHERE sas.street_address = ads.street_address AND
-					sald.seg_id = ads.seg_id AND
-					sald.layer_id = '{layer_id}' AND
-					CASE WHEN (ads.seg_side = 'L') THEN sald.left_value ELSE sald.right_value END <> ''
-			'''.format(layer_id=layer_id)
-			db.execute(stmt)
-			# print(ais_db.c.rowcount)
-			db.save()
-
-	print('Dropping temporary index...')
-	address_summary_table.drop_index('seg_id')
-
-#############################################################################
-# SERVICE AREA POINTS
-#############################################################################
-if WRITE_OUT:
-	print('Finding nearest service area point to each address...')
-	for sa_layer_def in sa_layer_defs:
-		layer_id = sa_layer_def['layer_id']
-		if 'point' in sa_layer_def['sources']:
-			method = sa_layer_def['sources']['point'].get('method')
-			if method == 'nearest':
-				print('Updating from {}...'.format(layer_id))
-				stmt = '''
-						with sap_layer as
-						(
-							select sap.*
-							from service_area_point sap
-							where sap.layer_id = '{layer_id}'
-						)
-						update service_area_summary sas
-						set {layer_id} = sapf.value
-						from
-							(
-							select ads.street_address, saplv.value
-							from address_summary ads
-							cross join lateral
-							(
-								select sap_layer.value
-								from sap_layer
-								order by st_setsrid(st_point(ads.geocode_x, ads.geocode_y), 2272) <-> sap_layer.geom limit 1
-							) as saplv
-							) sapf
-						where sas.street_address = sapf.street_address
-					'''.format(layer_id=layer_id)
-				db.execute(stmt)
-				db.save()
-
-			elif method == 'seg_id':
-				print('Updating from {}...'.format(layer_id))
-				stmt = '''
-						UPDATE service_area_summary sas
-						SET {layer_id} = sap.value
-						FROM address_summary ads, service_area_point sap
-						WHERE
-							sas.street_address = ads.street_address AND
-							sap.seg_id = ads.seg_id AND
-							sap.layer_id = '{layer_id}' AND
-							sap.value <> ''
-					'''.format(layer_id=layer_id)
-				db.execute(stmt)
-				db.save()
-
-#################
-# NEAREST POLY	#
-#################
-if WRITE_OUT:
-	print('Finding nearest service area polygon to each address...')
-	for sa_layer_def in sa_layer_defs:
-		layer_id = sa_layer_def['layer_id']
-		if 'polygon' in sa_layer_def['sources']:
-			method = sa_layer_def['sources']['polygon'].get('method')
-			if method != 'nearest_poly':
-				continue
-			print('Updating from {}...'.format(layer_id))
-			stmt = '''
-					with sap_layer as
-					(
-						select sap.*
-						from service_area_polygon sap
-						where sap.layer_id = '{layer_id}'
-					)
-					update service_area_summary sas
-					set {layer_id} = sapf.value
-					from
-						(
-						select ads.street_address, saplv.value
-						from address_summary ads
-						cross join lateral
-						(
-							select sap_layer.value
-							from sap_layer
-							order by st_setsrid(st_point(ads.geocode_x, ads.geocode_y), 2272) <-> sap_layer.geom limit 1
-						) as saplv
-						) sapf
-					where sas.street_address = sapf.street_address
-				'''.format(layer_id=layer_id)
-			db.execute(stmt)
-			db.save()
-################################
-# Update where method = yes_or_no:
-print("Updating service_area_summary values where method = 'yes_or_no'")
-for sa_layer_def in sa_layer_defs:
-	layer_id = sa_layer_def['layer_id']
-	method = sa_layer_def.get('value_method')
-	if method == 'yes_or_no':
-		stmt = '''
-				UPDATE service_area_summary sas
-				SET {layer_id} = (
-				CASE
-				WHEN {layer_id} != '' THEN 'Yes'
-				ELSE 'No'
-				END);
-				'''.format(layer_id=layer_id)
-		db.execute(stmt)
-		db.save()
-#################################
-# TODO Update address summary zip_code with point-in-poly value where USPS seg-based is Null (parameterize field to update, set in config, and execute in for loop)
-print("Updating null address_summary zip_codes from service_areas...")
-stmt = '''
-DROP INDEX public.address_summary_opa_owners_trigram_idx;
-DROP INDEX public.address_summary_sort_idx;
-DROP INDEX public.address_summary_street_address_idx;
-DROP INDEX public.ix_address_summary_dor_parcel_id;
-DROP INDEX public.ix_address_summary_opa_account_num;
-DROP INDEX public.ix_address_summary_pwd_parcel_id;
-DROP INDEX public.ix_address_summary_seg_id;
-
-UPDATE address_summary asum
-SET zip_code = sas.zip_code
-from service_area_summary sas
-where sas.street_address = asum.street_address and (asum.zip_code is Null or asum.zip_code in ('', null));
-
-CREATE INDEX ix_address_summary_seg_id
-    ON public.address_summary USING btree
-    (seg_id);
-    
-CREATE INDEX address_summary_opa_owners_trigram_idx
-    ON public.address_summary USING gin
-    (opa_owners gin_trgm_ops);
-
-CREATE INDEX address_summary_sort_idx
-    ON public.address_summary USING btree
-    (street_name, street_suffix, street_predir, street_postdir, address_low, address_high, unit_num);
-	
-CREATE INDEX address_summary_street_address_idx
-    ON public.address_summary USING btree
-    (street_address);
-
-CREATE INDEX ix_address_summary_dor_parcel_id
-    ON public.address_summary USING btree
-    (dor_parcel_id);
-
-CREATE INDEX ix_address_summary_opa_account_num
-    ON public.address_summary USING btree
-    (opa_account_num);
-
-CREATE INDEX ix_address_summary_pwd_parcel_id
-    ON public.address_summary USING btree
-    (pwd_parcel_id);
-'''
-db.execute(stmt)
-db.save()
-#################################
-# Clean up:
-db.close()
-
-print('Finished in {}'.format(datetime.now() - start))
+def main():
+
+    print('Starting...')
+    start = datetime.now()
+
+    """
+    TODO
+    - This might perform better if we do one big spatial join at the beginning
+      between address summary and service area polygons.
+    """
+
+    """SET UP"""
+    config = app.config
+    db = datum.connect(config['DATABASES']['engine'])
+    sa_layer_defs = config['SERVICE_AREAS']['layers']
+    sa_layer_ids = [x['layer_id'] for x in sa_layer_defs]
+    poly_table = db['service_area_polygon']
+    line_single_table = db['service_area_line_single']
+    line_dual_table = db['service_area_line_dual']
+    point_table = db['service_area_point']
+    #sa_summary_table = db['service_area_summary']
+    address_summary_table = db['address_summary']
+    address_summary_fields = [
+        'street_address',
+        'geocode_x',
+        'geocode_y',
+        # 'seg_id',
+        # 'seg_side',
+    ]
+    sa_summary_fields = [{'name': 'street_address', 'type': 'text'}]
+    sa_summary_fields += [{'name': x, 'type': 'text'} for x in sa_layer_ids]
+    sa_summary_row_template = {x: '' for x in sa_layer_ids}
+
+    # DEV
+    WRITE_OUT = True
+
+    # Keep poly rows in memory so we make less trips to the database for overlapping
+    # points.
+    # xy_map = {}  # x => y => [sa_poly_rows]
+
+    """MAIN"""
+    #
+    if WRITE_OUT:
+        print('Dropping service area summary table...')
+        db.drop_table('service_area_summary')
+
+        print('Creating service area summary table...')
+        db.create_table('service_area_summary', sa_summary_fields)
+
+    sa_summary_table = db['service_area_summary']
+
+    # print('Reading single-value service area lines...')
+    # line_single_map = {}  # layer_id => seg_id => value
+    # line_singles = ais_db.read(line_single_table, ['*'])
+
+    # for line_single in line_singles:
+    # 	layer_id = line_single['layer_id']
+    # 	seg_id = line_single['seg_id']
+    # 	value = line_single['value']
+    # 	if layer_id not in line_single_map:
+    # 		line_single_map[layer_id] = {}
+    # 	line_single_map[layer_id][seg_id] = value
+
+    # print('Reading dual-value service area lines...')
+    # line_dual_map = {}  # layer_id => seg_id => value
+    # line_duals = ais_db.read(line_dual_table, ['*'])
+
+    # for line_dual in line_duals:
+    # 	layer_id = line_dual['layer_id']
+    # 	seg_id = line_dual['seg_id']
+    # 	left_value = line_dual['left_value']
+    # 	right_value = line_dual['right_value']
+    # 	if layer_id not in line_dual_map:
+    # 		line_dual_map[layer_id] = {}
+
+    # 	line_dual_map[layer_id][seg_id] = {}
+    # 	line_dual_map[layer_id][seg_id]['left'] = left_value
+    # 	line_dual_map[layer_id][seg_id]['right'] = right_value
+
+    print('Reading address summary...')
+    address_summary_rows = address_summary_table.read(\
+        fields=address_summary_fields, \
+        sort=['geocode_x', 'geocode_y']\
+    )
+
+    sa_summary_rows = []
+
+    # Sort address summary rows by X, Y and use these to compare the last row
+    # to the current one. This minimizes trips to the database for poly values.
+    last_x = None
+    last_y = None
+    last_sa_rows = None
+    #
+    print('Intersecting addresses and service area polygons...')
+    for i, address_summary_row in enumerate(address_summary_rows):
+        try:
+            if i % 10000 == 0:
+                print(i)
+
+                # Write in chunks
+                if WRITE_OUT: #and i % 50000 == 0:
+                    sa_summary_table.write(sa_summary_rows)
+                    sa_summary_rows = []
+
+            # Get attributes
+            street_address = address_summary_row['street_address']
+            # seg_id = address_summary_row['seg_id']
+            # seg_side = address_summary_row['seg_side']
+            x = address_summary_row['geocode_x']
+            y = address_summary_row['geocode_y']
+
+            sa_rows = None
+            # if x in xy_map:
+            # 	y_map = xy_map[x]
+            # 	if y in y_map:
+            # 		sa_rows = y_map[y]
+            if last_x and (last_x == x and last_y == y):
+                sa_rows = last_sa_rows
+
+            if sa_rows is None and None not in (x,y):
+                # Get intersecting service areas
+                where = 'ST_Intersects(geom, ST_SetSrid(ST_Point({}, {}), 2272))'.format(x, y)
+                sa_rows = poly_table.read(fields=['layer_id', 'value'], where=where, return_geom=False)
+
+                # Add to map
+                # x_map = xy_map[x] = {}
+                # x_map[y] = sa_rows
+
+            # Create and insert summary row
+            sa_summary_row = deepcopy(sa_summary_row_template)
+            sa_summary_row['street_address'] = street_address
+            if sa_rows: 
+                update_dict = {}
+                for x in sa_rows: 
+                    if update_dict.get(x['layer_id']) == None: 
+                        update_dict[x['layer_id']] = []
+                    update_dict[x['layer_id']].append(x['value'])
+                for layer_id, _ in update_dict.items(): 
+                    update_dict[layer_id].sort()
+                    update_dict[layer_id] = '|'.join(update_dict[layer_id])
+                sa_summary_row.update(update_dict)
+
+            sa_summary_rows.append(sa_summary_row)
+
+            last_x = x
+            last_y = y
+            last_sa_rows = sa_rows
+
+        except Exception as e:
+            print(traceback.format_exc())
+            raise e
+
+    # Clear out XY map
+    # xy_map = {}
+
+    if WRITE_OUT:
+        print('Writing service area summary rows...')
+        sa_summary_table.write(sa_summary_rows)
+        del sa_summary_rows
+
+    # # Update where method = yes_or_no:
+    # for sa_layer_def in sa_layer_defs:
+    # 	layer_id = sa_layer_def['layer_id']
+    # 	if 'polygon' in sa_layer_def['sources']:
+    # 		method = sa_layer_def['sources']['polygon'].get('method')
+    # 		if method == 'yes_or_no':
+    # 			stmt = '''
+    # 					UPDATE service_area_summary sas
+    # 					SET {layer_id} = (
+    # 					CASE
+    # 					WHEN {layer_id} != '' THEN 'yes'
+    # 					ELSE 'no'
+    # 					END);
+    # 					'''.format(layer_id=layer_id)
+    # 			db.execute(stmt)
+    # 			# print(ais_db.c.rowcount)
+    # 			db.save()
+    ################################################################################
+    # SERVICE AREA LINES
+    ################################################################################
+
+    if WRITE_OUT:
+        print('\n** SERVICE AREA LINES ***\n')
+        print('Creating indexes...')
+        sa_summary_table.create_index('street_address')
+
+        print('Creating temporary indexes...')
+        address_summary_table.create_index('seg_id')
+
+        for sa_layer_def in sa_layer_defs:
+            layer_id = sa_layer_def['layer_id']
+
+            if 'line_single' in sa_layer_def['sources']:
+                print('Updating from {}...'.format(layer_id))
+                stmt = '''
+                    UPDATE service_area_summary sas
+                    SET {layer_id} = sals.value
+                    FROM address_summary ads, service_area_line_single sals
+                    WHERE
+                        sas.street_address = ads.street_address AND
+                        sals.seg_id = ads.seg_id AND
+                        sals.layer_id = '{layer_id}' AND
+                        sals.value <> ''
+                '''.format(layer_id=layer_id)
+                db.execute(stmt)
+                # print(ais_db.c.rowcount)
+                db.save()
+
+            elif 'line_dual' in sa_layer_def['sources']:
+                print('Updating from {}...'.format(layer_id))
+                stmt = '''
+                    UPDATE service_area_summary sas
+                    SET {layer_id} = CASE WHEN (ads.seg_side = 'L') THEN sald.left_value ELSE sald.right_value END
+                    FROM address_summary ads, service_area_line_dual sald
+                    WHERE sas.street_address = ads.street_address AND
+                        sald.seg_id = ads.seg_id AND
+                        sald.layer_id = '{layer_id}' AND
+                        CASE WHEN (ads.seg_side = 'L') THEN sald.left_value ELSE sald.right_value END <> ''
+                '''.format(layer_id=layer_id)
+                db.execute(stmt)
+                # print(ais_db.c.rowcount)
+                db.save()
+
+        print('Dropping temporary index...')
+        address_summary_table.drop_index('seg_id')
+
+    #############################################################################
+    # SERVICE AREA POINTS
+    #############################################################################
+    if WRITE_OUT:
+        print('Finding nearest service area point to each address...')
+        for sa_layer_def in sa_layer_defs:
+            layer_id = sa_layer_def['layer_id']
+            if 'point' in sa_layer_def['sources']:
+                method = sa_layer_def['sources']['point'].get('method')
+                if method == 'nearest':
+                    print('Updating from {}...'.format(layer_id))
+                    stmt = '''
+                            with sap_layer as
+                            (
+                                select sap.*
+                                from service_area_point sap
+                                where sap.layer_id = '{layer_id}'
+                            )
+                            update service_area_summary sas
+                            set {layer_id} = sapf.value
+                            from
+                                (
+                                select ads.street_address, saplv.value
+                                from address_summary ads
+                                cross join lateral
+                                (
+                                    select sap_layer.value
+                                    from sap_layer
+                                    order by st_setsrid(st_point(ads.geocode_x, ads.geocode_y), 2272) <-> sap_layer.geom limit 1
+                                ) as saplv
+                                ) sapf
+                            where sas.street_address = sapf.street_address
+                        '''.format(layer_id=layer_id)
+                    db.execute(stmt)
+                    db.save()
+
+                elif method == 'seg_id':
+                    print('Updating from {}...'.format(layer_id))
+                    stmt = '''
+                            UPDATE service_area_summary sas
+                            SET {layer_id} = sap.value
+                            FROM address_summary ads, service_area_point sap
+                            WHERE
+                                sas.street_address = ads.street_address AND
+                                sap.seg_id = ads.seg_id AND
+                                sap.layer_id = '{layer_id}' AND
+                                sap.value <> ''
+                        '''.format(layer_id=layer_id)
+                    db.execute(stmt)
+                    db.save()
+
+    #################
+    # NEAREST POLY	#
+    #################
+    if WRITE_OUT:
+        print('Finding nearest service area polygon to each address...')
+        for sa_layer_def in sa_layer_defs:
+            layer_id = sa_layer_def['layer_id']
+            if 'polygon' in sa_layer_def['sources']:
+                method = sa_layer_def['sources']['polygon'].get('method')
+                if method != 'nearest_poly':
+                    continue
+                print('Updating from {}...'.format(layer_id))
+                stmt = '''
+                        with sap_layer as
+                        (
+                            select sap.*
+                            from service_area_polygon sap
+                            where sap.layer_id = '{layer_id}'
+                        )
+                        update service_area_summary sas
+                        set {layer_id} = sapf.value
+                        from
+                            (
+                            select ads.street_address, saplv.value
+                            from address_summary ads
+                            cross join lateral
+                            (
+                                select sap_layer.value
+                                from sap_layer
+                                order by st_setsrid(st_point(ads.geocode_x, ads.geocode_y), 2272) <-> sap_layer.geom limit 1
+                            ) as saplv
+                            ) sapf
+                        where sas.street_address = sapf.street_address
+                    '''.format(layer_id=layer_id)
+                db.execute(stmt)
+                db.save()
+    ################################
+    # Update where method = yes_or_no:
+    print("Updating service_area_summary values where method = 'yes_or_no'")
+    for sa_layer_def in sa_layer_defs:
+        layer_id = sa_layer_def['layer_id']
+        method = sa_layer_def.get('value_method')
+        if method == 'yes_or_no':
+            stmt = '''
+                    UPDATE service_area_summary sas
+                    SET {layer_id} = (
+                    CASE
+                    WHEN {layer_id} != '' THEN 'Yes'
+                    ELSE 'No'
+                    END);
+                    '''.format(layer_id=layer_id)
+            db.execute(stmt)
+            db.save()
+    #################################
+    # TODO Update address summary zip_code with point-in-poly value where USPS seg-based is Null (parameterize field to update, set in config, and execute in for loop)
+    print("Updating null address_summary zip_codes from service_areas...")
+    stmt = '''
+    DROP INDEX public.address_summary_opa_owners_trigram_idx;
+    DROP INDEX public.address_summary_sort_idx;
+    DROP INDEX public.address_summary_street_address_idx;
+    DROP INDEX public.ix_address_summary_dor_parcel_id;
+    DROP INDEX public.ix_address_summary_opa_account_num;
+    DROP INDEX public.ix_address_summary_pwd_parcel_id;
+    DROP INDEX public.ix_address_summary_seg_id;
+
+    UPDATE address_summary asum
+    SET zip_code = sas.zip_code
+    from service_area_summary sas
+    where sas.street_address = asum.street_address and (asum.zip_code is Null or asum.zip_code in ('', null));
+
+    CREATE INDEX ix_address_summary_seg_id
+        ON public.address_summary USING btree
+        (seg_id);
+        
+    CREATE INDEX address_summary_opa_owners_trigram_idx
+        ON public.address_summary USING gin
+        (opa_owners gin_trgm_ops);
+
+    CREATE INDEX address_summary_sort_idx
+        ON public.address_summary USING btree
+        (street_name, street_suffix, street_predir, street_postdir, address_low, address_high, unit_num);
+        
+    CREATE INDEX address_summary_street_address_idx
+        ON public.address_summary USING btree
+        (street_address);
+
+    CREATE INDEX ix_address_summary_dor_parcel_id
+        ON public.address_summary USING btree
+        (dor_parcel_id);
+
+    CREATE INDEX ix_address_summary_opa_account_num
+        ON public.address_summary USING btree
+        (opa_account_num);
+
+    CREATE INDEX ix_address_summary_pwd_parcel_id
+        ON public.address_summary USING btree
+        (pwd_parcel_id);
+    '''
+    db.execute(stmt)
+    db.save()
+    #################################
+    # Clean up:
+    db.close()
+
+    print('Finished in {}'.format(datetime.now() - start))
diff --git a/ais/engine/scripts/make_street_intersections.py b/ais/engine/scripts/make_street_intersections.py
index 2a983a00..34090051 100644
--- a/ais/engine/scripts/make_street_intersections.py
+++ b/ais/engine/scripts/make_street_intersections.py
@@ -11,281 +11,288 @@
 from datum import Database
 from ais.models import StreetIntersection
 
+def main():
+    print('Starting...')
+    start = datetime.now()
 
-print('Starting...')
-start = datetime.now()
+    """SET UP"""
 
-"""SET UP"""
+    config = app.config
+    engine_srid = config['ENGINE_SRID']
+    Parser = config['PARSER']
+    db = Database(config['DATABASES']['engine'])
+    dsn = config['DATABASES']['engine']
+    db_user = dsn[dsn.index("//") + 2:dsn.index(":", dsn.index("//"))]
+    db_pw = dsn[dsn.index(":",dsn.index(db_user)) + 1:dsn.index("@")]
+    db_name = dsn[dsn.index("/", dsn.index("@")) + 1:]
+    pg_db = psycopg2.connect('dbname={db_name} user={db_user} password={db_pw} host=localhost'.format(db_name=db_name, db_user=db_user, db_pw=db_pw))
 
-config = app.config
-engine_srid = config['ENGINE_SRID']
-Parser = config['PARSER']
-db = Database(config['DATABASES']['engine'])
-dsn = config['DATABASES']['engine']
-db_user = dsn[dsn.index("//") + 2:dsn.index(":", dsn.index("//"))]
-db_pw = dsn[dsn.index(":",dsn.index(db_user)) + 1:dsn.index("@")]
-db_name = dsn[dsn.index("/", dsn.index("@")) + 1:]
-pg_db = psycopg2.connect('dbname={db_name} user={db_user} password={db_pw} host=localhost'.format(db_name=db_name, db_user=db_user, db_pw=db_pw))
+    # Get table params
+    source_def = config['BASE_DATA_SOURCES']['streets']
+    source_db_name = source_def['db']
+    source_db_url = config['DATABASES'][source_db_name]
+    field_map = source_def['field_map']
+    centerline_table_name = source_def['table']
+    nodes_table_name = 'GIS_STREETS.Street_Nodes'
+    street_full_fields = ['street_' + x for x in ['predir', 'name', 'suffix', 'postdir']]
+    source_street_full_fields = [field_map[x] for x in street_full_fields]
+    con_dsn = source_db_url[source_db_url.index("//") + 2:]
+    con_user = con_dsn[:con_dsn.index(":")]
+    con_pw = con_dsn[con_dsn.index(":") + 1 : con_dsn.index("@")]
+    con_db = con_dsn[con_dsn.index("@") + 1:]
+    con = cx_Oracle.connect(con_user, con_pw, con_db)
+    # Get table references
+    source_db = Database(source_db_url)
+    centerline_table = source_db[centerline_table_name]
+    node_table = source_db[nodes_table_name]
+    source_geom_field = centerline_table.geom_field
+    intersection_table_name = StreetIntersection.__table__.name
+    intersection_table = db[intersection_table_name]
 
-# Get table params
-source_def = config['BASE_DATA_SOURCES']['streets']
-source_db_name = source_def['db']
-source_db_url = config['DATABASES'][source_db_name]
-field_map = source_def['field_map']
-centerline_table_name = source_def['table']
-nodes_table_name = 'GIS_STREETS.Street_Nodes'
-street_full_fields = ['street_' + x for x in ['predir', 'name', 'suffix', 'postdir']]
-source_street_full_fields = [field_map[x] for x in street_full_fields]
-con_dsn = source_db_url[source_db_url.index("//") + 2:]
-con_user = con_dsn[:con_dsn.index(":")]
-con_pw = con_dsn[con_dsn.index(":") + 1 : con_dsn.index("@")]
-con_db = con_dsn[con_dsn.index("@") + 1:]
-con = cx_Oracle.connect(con_user, con_pw, con_db)
-# Get table references
-source_db = Database(source_db_url)
-centerline_table = source_db[centerline_table_name]
-node_table = source_db[nodes_table_name]
-source_geom_field = centerline_table.geom_field
-intersection_table_name = StreetIntersection.__table__.name
-intersection_table = db[intersection_table_name]
 
+    """MAIN"""
 
-"""MAIN"""
+    parser = Parser()
 
-parser = Parser()
+    print(f'Deleting existing table {str(intersection_table_name)}...')
+    intersection_table.delete(cascade=True)
 
-print('Deleting existing intersections...')
-intersection_table.delete(cascade=True)
+    print('Creating temporary table "street_centerlines"...')
 
-print('Creating temporary tables...')
+    st_cent_stmt = '''
+        DROP table if exists street_centerlines;
+        Create table street_centerlines
+        (
+          street_code numeric(10,0),
+          street_name text,
+          street_full text,
+          street_predir text,
+          street_postdir text,
+          street_suffix text,
+          street_type text,
+          fnode numeric(10,0),
+          tnode numeric(10,0),
+          geom geometry(MultiLineString, 2272)
+    );
+    '''
+    db.execute(st_cent_stmt)
+    db.save()
 
-st_cent_stmt = '''
-    DROP table if exists street_centerlines;
-    Create table street_centerlines
-    (
-      street_code numeric(10,0),
-      street_name text,
-      street_full text,
-      street_predir text,
-      street_postdir text,
-      street_suffix text,
-      street_type text,
-      fnode numeric(10,0),
-      tnode numeric(10,0),
-      geom geometry(MultiLineString, 2272)
-);
-'''
-db.execute(st_cent_stmt)
-db.save()
-
-st_node_stmt = '''
-    DROP table if exists street_nodes;
-    CREATE TABLE public.street_nodes
-    (
-      objectid numeric(10,0),
-      streetcl_ numeric(10,0),
-      node_id numeric(10,0),
-      int_id numeric(10,0),
-      intersecti character varying(255),
-      geom geometry(Point,2272),
-      CONSTRAINT street_nodes_pkey PRIMARY KEY (objectid)
-);
-'''
-db.execute(st_node_stmt)
-db.save()
+    st_node_stmt = '''
+        DROP table if exists street_nodes;
+        CREATE TABLE public.street_nodes
+        (
+          objectid numeric(10,0),
+          streetcl_ numeric(10,0),
+          node_id numeric(10,0),
+          int_id numeric(10,0),
+          intersecti character varying(255),
+          geom geometry(Point,2272),
+          CONSTRAINT street_nodes_pkey PRIMARY KEY (objectid)
+    );
+    '''
+    db.execute(st_node_stmt)
+    db.save()
 
-print('Reading streets from source...')
-source_fields = list(field_map.values())
-# where = "st_type != 'RAMP'"
-centerline_rows = centerline_table.read(to_srid=engine_srid)
-centerlines = []
-nodes = []
-error_count = 0
+    source_fields = list(field_map.values())
+    # where = "st_type != 'RAMP'"
+    print(f'Reading {centerline_table} from source...')
+    centerline_rows = centerline_table.read(to_srid=engine_srid)
+    centerlines = []
+    nodes = []
+    error_count = 0
 
-for i, cl_row in enumerate(centerline_rows):
-    try:
+    for i, cl_row in enumerate(centerline_rows):
         if i % 10000 == 0:
             print(i)
-
-        # Parse street name
-        source_street_full_comps = [str(cl_row[x]).strip() for x in \
-                                    source_street_full_fields]
-        source_street_full_comps = [x for x in source_street_full_comps if x != '']
-        source_street_full = ' '.join(source_street_full_comps)
-        seg_id = cl_row[field_map['seg_id']]
-        parsed = None
         try:
-            parsed = parser.parse(source_street_full)
-            if parsed['type'] != 'street':
-                raise ValueError('Invalid street')
+            # Parse street name
+            source_street_full_comps = [str(cl_row[x]).strip() for x in \
+                                        source_street_full_fields]
+            source_street_full_comps = [x for x in source_street_full_comps if x != '']
+            source_street_full = ' '.join(source_street_full_comps)
+            seg_id = cl_row[field_map['seg_id']]
+            parsed = None
+            try:
+                parsed = parser.parse(source_street_full)
+                if parsed['type'] != 'street':
+                    raise ValueError('Invalid street')
+
+                # comps = parsed['components']    			<== phladdress
+                comps = parsed['components']['street']  # <== passyunk
+            except:
+                pass
+            # Test with this version allowing all nodes in (including ramps, etc.) - if troublesome remove
+            # except Exception as e:
+            #     raise ValueError('Could not parse')
+            if parsed['type'] == 'street':
+                street_comps = {
+                    'street_predir': comps['predir'] or '',
+                    'street_name': comps['name'] or '',
+                    'street_suffix': comps['suffix'] or '',
+                    'street_postdir': comps['postdir'] or '',
+                    'street_full': comps['full'],
+                }
+            else:
+                #print(source_street_full)
+                street_comps = {
+                # 'street_predir': cl_row[field_map['street_predir']] or '',
+                # 'street_name': cl_row[field_map['street_name']] or '',
+                # 'street_suffix': cl_row[field_map['street_suffix']] or '',
+                # 'street_postdir': cl_row[field_map['street_postdir']] or '',
+                'street_full': source_street_full,
+                }
 
-            # comps = parsed['components']    			<== phladdress
-            comps = parsed['components']['street']  # <== passyunk
-        except:
-            pass
-        # Test with this version allowing all nodes in (including ramps, etc.) - if troublesome remove
-        # except Exception as e:
-        #     raise ValueError('Could not parse')
-        if parsed['type'] == 'street':
-            street_comps = {
-                'street_predir': comps['predir'] or '',
-                'street_name': comps['name'] or '',
-                'street_suffix': comps['suffix'] or '',
-                'street_postdir': comps['postdir'] or '',
-                'street_full': comps['full'],
-            }
-        else:
-            print(source_street_full)
-            street_comps = {
-            # 'street_predir': cl_row[field_map['street_predir']] or '',
-            # 'street_name': cl_row[field_map['street_name']] or '',
-            # 'street_suffix': cl_row[field_map['street_suffix']] or '',
-            # 'street_postdir': cl_row[field_map['street_postdir']] or '',
-            'street_full': source_street_full,
-            }
+            centerline = {key: cl_row[value] for key, value in field_map.items()}
+            centerline.update(street_comps)
+            centerline['geom'] = cl_row[source_geom_field]
+            centerline['fnode'] = cl_row['fnode_']
+            centerline['tnode'] = cl_row['tnode_']
+            centerline['street_type'] = cl_row['st_type']
+            centerline.pop('left_to', None)
+            centerline.pop('left_from', None)
+            centerline.pop('right_to', None)
+            centerline.pop('right_from', None)
+            centerline.pop('seg_id', None)
+            centerlines.append(centerline)
 
-        centerline = {key: cl_row[value] for key, value in field_map.items()}
-        centerline.update(street_comps)
-        centerline['geom'] = cl_row[source_geom_field]
-        centerline['fnode'] = cl_row['fnode_']
-        centerline['tnode'] = cl_row['tnode_']
-        centerline['street_type'] = cl_row['st_type']
-        centerline.pop('left_to', None)
-        centerline.pop('left_from', None)
-        centerline.pop('right_to', None)
-        centerline.pop('right_from', None)
-        centerline.pop('seg_id', None)
-        centerlines.append(centerline)
+        except ValueError as e:
+            # FEEDBACK
+            #print('{}: {} ({})'.format(e, source_street_full, seg_id))
+            error_count += 1
 
-    except ValueError as e:
-        # FEEDBACK
-        print('{}: {} ({})'.format(e, source_street_full, seg_id))
-        error_count += 1
+        except Exception as e:
+            print('Unhandled error on row: {}'.format(i))
+            print(traceback.format_exc())
+            raise e
 
-    except Exception as e:
-        print('Unhandled error on row: {}'.format(i))
-        # pprint(street)
-        print(traceback.format_exc())
-        sys.exit()
+    centerline_table = db['street_centerlines']
+    nodes_table = db['street_nodes']
+    print(nodes_table)
 
-centerline_table = db['street_centerlines']
-nodes_table = db['street_nodes']
-print(nodes_table)
+    '''
+    WRITE
+    '''
+    print(f'Copying temporary street_nodes table "{str(nodes_table_name)}"...')
+    #etl.fromoraclesde(con, nodes_table_name, fields=['objectid', 'streetcl_', 'node_id', 'int_id', 'intersecti'])\
+    #    .rename({'shape': 'geom'})\
+    #    .topostgis(pg_db, 'street_nodes')
+    # shape field is already named geom for whatever reason. -Roland, 11-30-22
 
-'''
-WRITE
-'''
-print("Copying temporary street_nodes table...")
-etl.fromoraclesde(con, nodes_table_name, fields=['objectid', 'streetcl_', 'node_id', 'int_id', 'intersecti'])\
-    .rename({'shape': 'geom'})\
-    .topostgis(pg_db, 'street_nodes')
+    nodes_rows = etl.fromoraclesde(con, nodes_table_name, fields=['objectid', 'streetcl_', 'node_id', 'int_id', 'intersecti'])
 
-print("Writing temporary centerline table...")
-centerline_table.write(centerlines, chunk_size=50000)
+    # Rename to match our street_nodes table shape field, which is geom.
+    nodes_rows = nodes_rows.rename({'shape': 'geom'})
 
-intersections = []
-error_count = 0
+    # Write to our local db
+    nodes_rows.topostgis(pg_db, 'street_nodes')
 
-st_int_stmt =\
-'''
-with distinct_st1scns as
-(
-	select distinct *
-	from
-	(
-	with scsn as (
-	select sn.*, sc.street_code
-	from street_nodes sn
-	left join street_centerlines sc on sc.tnode = sn.node_id
-	union
-	select sn.*, sc.street_code
-	from street_nodes sn
-	left join street_centerlines sc on sc.fnode = sn.node_id	)
-	,
-	scsn_distinct as
-	(select distinct on (node_id, street_code) *
-	from scsn
-	)
-	,
-	scsnd_join as
-	(select scsnd.*, scsn.street_code as st_code_2
-	from scsn_distinct scsnd
-	left join scsn on scsn.node_id = scsnd.node_id and scsn.street_code != scsnd.street_code
-	)
-	,
-	scsndj_distinct as
-	(
-	select objectid, node_id, int_id, intersecti, geom, street_code as street_1_code, st_code_2 as street_2_code
-	from scsnd_join
-	where street_code < st_code_2
-	union
-	select objectid, node_id, int_id, intersecti, geom, street_code as street_1_code, st_code_2 as street_2_code
-	from scsnd_join
-	where (street_code is null or st_code_2 is null) and not (street_code is null and st_code_2 is null)
-	order by int_id
-	)
-	,
-	scsndjd_distinct as
-	(
-	select distinct on (int_id, street_1_code, street_2_code) *
-	from scsndj_distinct
-    order by int_id, street_1_code, street_2_code , node_id
-	)
-	select scsn.*, sc.street_predir as street_1_predir, sc.street_name as street_1_name, sc.street_suffix as street_1_suffix, sc.street_postdir as street_1_postdir, sc.street_full as street_1_full, sc.street_type as street_1_type
-	from scsndjd_distinct scsn
-	left join street_centerlines sc on sc.street_code = scsn.street_1_code
-	order by scsn.node_id
-	) st1cns
-	order by node_id
-)
-,
-st12scns as
-(
-select scsn1.*, sc.street_predir as street_2_predir, sc.street_name as street_2_name, sc.street_suffix as street_2_suffix, sc.street_postdir as street_2_postdir, sc.street_full as street_2_full, sc.street_type as street_2_type
-from distinct_st1scns scsn1
-left join street_centerlines sc on sc.street_code = street_2_code
-order by scsn1.node_id
-)
-,
-final AS
-(
-select distinct node_id, int_id, street_1_code, street_1_name, street_1_full, street_1_predir, street_1_postdir, street_1_suffix,
-  street_2_code, street_2_name, street_2_full, street_2_predir, street_2_postdir, street_2_suffix, geom
-from st12scns
-WHERE int_id is not NULL and street_1_type != 'RAMP' and street_2_type != 'RAMP'
-order by node_id
-)
-INSERT INTO street_intersection (node_id, int_id, street_1_code, street_1_name, street_1_full, street_1_predir, street_1_postdir, street_1_suffix, street_2_code,
-street_2_name, street_2_full, street_2_predir, street_2_postdir, street_2_suffix, geom)
-    (SELECT final.node_id, final.int_id, final.street_1_code, final.street_1_name, final.street_1_full, final.street_1_predir, final.street_1_postdir,
-    final.street_1_suffix, final.street_2_code, final.street_2_name, final.street_2_full, final.street_2_predir,
-    final.street_2_postdir, final.street_2_suffix, final.geom from final)
-;
-'''
+    print("Writing temporary centerline table...")
+    centerline_table.write(centerlines, chunk_size=50000)
+
+    intersections = []
+    error_count = 0
+
+    st_int_stmt =\
+    '''
+    with distinct_st1scns as
+    (
+        select distinct *
+        from
+        (
+        with scsn as (
+        select sn.*, sc.street_code
+        from street_nodes sn
+        left join street_centerlines sc on sc.tnode = sn.node_id
+        union
+        select sn.*, sc.street_code
+        from street_nodes sn
+        left join street_centerlines sc on sc.fnode = sn.node_id	)
+        ,
+        scsn_distinct as
+        (select distinct on (node_id, street_code) *
+        from scsn
+        )
+        ,
+        scsnd_join as
+        (select scsnd.*, scsn.street_code as st_code_2
+        from scsn_distinct scsnd
+        left join scsn on scsn.node_id = scsnd.node_id and scsn.street_code != scsnd.street_code
+        )
+        ,
+        scsndj_distinct as
+        (
+        select objectid, node_id, int_id, intersecti, geom, street_code as street_1_code, st_code_2 as street_2_code
+        from scsnd_join
+        where street_code < st_code_2
+        union
+        select objectid, node_id, int_id, intersecti, geom, street_code as street_1_code, st_code_2 as street_2_code
+        from scsnd_join
+        where (street_code is null or st_code_2 is null) and not (street_code is null and st_code_2 is null)
+        order by int_id
+        )
+        ,
+        scsndjd_distinct as
+        (
+        select distinct on (int_id, street_1_code, street_2_code) *
+        from scsndj_distinct
+        order by int_id, street_1_code, street_2_code , node_id
+        )
+        select scsn.*, sc.street_predir as street_1_predir, sc.street_name as street_1_name, sc.street_suffix as street_1_suffix, sc.street_postdir as street_1_postdir, sc.street_full as street_1_full, sc.street_type as street_1_type
+        from scsndjd_distinct scsn
+        left join street_centerlines sc on sc.street_code = scsn.street_1_code
+        order by scsn.node_id
+        ) st1cns
+        order by node_id
+    )
+    ,
+    st12scns as
+    (
+    select scsn1.*, sc.street_predir as street_2_predir, sc.street_name as street_2_name, sc.street_suffix as street_2_suffix, sc.street_postdir as street_2_postdir, sc.street_full as street_2_full, sc.street_type as street_2_type
+    from distinct_st1scns scsn1
+    left join street_centerlines sc on sc.street_code = street_2_code
+    order by scsn1.node_id
+    )
+    ,
+    final AS
+    (
+    select distinct node_id, int_id, street_1_code, street_1_name, street_1_full, street_1_predir, street_1_postdir, street_1_suffix,
+      street_2_code, street_2_name, street_2_full, street_2_predir, street_2_postdir, street_2_suffix, geom
+    from st12scns
+    WHERE int_id is not NULL and street_1_type != 'RAMP' and street_2_type != 'RAMP'
+    order by node_id
+    )
+    INSERT INTO street_intersection (node_id, int_id, street_1_code, street_1_name, street_1_full, street_1_predir, street_1_postdir, street_1_suffix, street_2_code,
+    street_2_name, street_2_full, street_2_predir, street_2_postdir, street_2_suffix, geom)
+        (SELECT final.node_id, final.int_id, final.street_1_code, final.street_1_name, final.street_1_full, final.street_1_predir, final.street_1_postdir,
+        final.street_1_suffix, final.street_2_code, final.street_2_name, final.street_2_full, final.street_2_predir,
+        final.street_2_postdir, final.street_2_suffix, final.geom from final)
+    ;
+    '''
 
-print("Writing street intersection table...")
-db.execute(st_int_stmt)
-db.save()
+    print("Writing street_intersection table...")
+    db.execute(st_int_stmt)
+    db.save()
 
-print("Deleting temporary centerline table...")
-del_st_cent_stmt =\
-'''
-    Drop table if exists street_centerlines;
-'''
-db.execute(del_st_cent_stmt)
-db.save()
+    print("Deleting temporary streets_centerline table...")
+    del_st_cent_stmt =\
+    '''
+        Drop table if exists street_centerlines;
+    '''
+    db.execute(del_st_cent_stmt)
+    db.save()
 
-print("Deleting temporary nodes table...")
-del_st_node_stmt =\
-'''
-    Drop table if exists street_nodes;
-'''
-db.execute(del_st_node_stmt)
-db.save()
-'''
-FINISH
-'''
+    print("Deleting temporary nodes table...")
+    del_st_node_stmt =\
+    '''
+        Drop table if exists street_nodes;
+    '''
+    db.execute(del_st_node_stmt)
+    db.save()
+    '''
+    FINISH
+    '''
 
-#source_db.close()
-db.close()
-print('Finished in {} seconds'.format(datetime.now() - start))
+    #source_db.close()
+    db.close()
+    print('Finished in {} seconds'.format(datetime.now() - start))
diff --git a/ais/engine/tests/test_engine.py b/ais/engine/tests/test_engine.py
deleted file mode 100644
index 91755a65..00000000
--- a/ais/engine/tests/test_engine.py
+++ /dev/null
@@ -1,144 +0,0 @@
-import subprocess
-import datum
-import pytest
-from ais import app
-config = app.config
-db = datum.connect(config['DATABASES']['engine'])
-
-@pytest.fixture
-def startup():
-    """Startup fixture: make database connections and define tables to ignore"""
-    new_db_map = {
-        'ais-api-broad':     'engine_broad',
-        'ais-api-market':    'engine_market',
-    }
-    proc = subprocess.Popen(['bash', '-c', '. ../../../bin/eb_env_utils.sh; get_prod_env'], stdout=subprocess.PIPE)
-    output = proc.stdout.read()
-    old_prod_env = output.rstrip()
-    old_prod_env = old_prod_env.decode('utf-8')
-    old_db = datum.connect(config['DATABASES'][new_db_map[old_prod_env]])
-    new_db = datum.connect(config['DATABASES']['engine'])
-    unused_tables =  ('spatial_ref_sys', 'alembic_version', 'multiple_seg_line', 'service_area_diff', 'address_zip', 'zip_range', 'dor_parcel_address_analysis')
-    changed_tables = ('ng911_address_point',)
-    ignore_tables = unused_tables + changed_tables
-
-    return {'new_db': new_db, 'old_db': old_db, 'unused_tables': unused_tables, 'changed_tables': changed_tables, 'ignore_tables': ignore_tables}
-
-def test_no_duplicates(startup):
-    """ Don't allow duplicate street_addresses in address_summary """
-
-    new_db = startup['new_db']
-    total_stmt = "select count(*) as total_addresses from address_summary"
-    distinct_stmt = "select count(*) as distinct_addresses from (select distinct street_address from address_summary) foo"
-    num_total_row = new_db.execute(total_stmt)
-    num_distinct_row = new_db.execute(distinct_stmt)
-    num_total = num_total_row[0]['total_addresses']
-    num_distinct = num_distinct_row[0]['distinct_addresses']
-    assert num_total == num_distinct
-
-
-def test_compare_num_tables(startup):
-    """Test #1: Check if all tables are included in build"""
-    # assert len(startup['new_db'].tables) == len(startup['old_db'].tables)
-    new_db = startup['new_db']
-    old_db = startup['old_db']
-    table_count_stmt = "select count(*) from information_schema.tables where table_schema = 'public' AND table_type = 'BASE TABLE' and table_name not in {}".format(str(startup['ignore_tables']))
-    print(table_count_stmt)
-    new_table_count = new_db.execute(table_count_stmt)
-    old_table_count = old_db.execute(table_count_stmt)
-    assert new_table_count == old_table_count
-
-#@pytest.mark.skip(reason="temp change of eclipse_location_ids source table with more rows")
-def test_num_rows_bt_db_tables(startup):
-    """"Test #2: Check if all tables within 10% of rows as old version"""
-    new_db = startup['new_db']
-    old_db = startup['old_db']
-    list_tables_stmt = "select table_name from information_schema.tables where table_schema = 'public' AND table_type = 'BASE TABLE'"
-    new_db_tables = new_db.execute(list_tables_stmt)
-    old_db_tables = old_db.execute(list_tables_stmt)
-    # new_db_tables = startup['new_db'].tables
-    for ntable in new_db_tables:
-        table_name = ntable['table_name']
-        if table_name in startup['ignore_tables']:
-            continue
-
-        # ndb_table = startup['new_db'][ntable]
-        # n_rows = ndb_table.count
-        row_count_stmt = "select count(*) as count from {}".format(table_name)
-        n_rows = new_db.execute(row_count_stmt)
-        o_rows = old_db.execute(row_count_stmt)
-        # odb_table = startup['old_db'][ntable]
-        # o_rows = odb_table.count
-        fdif = abs((n_rows[0]['count'] - o_rows[0]['count']) / o_rows[0]['count'])
-
-        assert fdif <= 0.1, (ntable, fdif)
-
-@pytest.mark.skip(reason="added geocode type for ng911")
-def test_geocode_types(startup):
-    """Test #3: Check if all geocode types present (compare new an old builds)"""
-    new_db = startup['new_db']
-    old_db = startup['old_db']
-
-    def get_geo_types(db):
-        stmt = "SELECT DISTINCT geocode_type FROM geocode"
-        geo_types = db.execute(stmt)
-        results = sorted([f['geocode_type'] for f in geo_types])
-
-        return results
-
-    n_geo_types = get_geo_types(new_db)
-    o_geo_types = get_geo_types(old_db)
-
-    assert n_geo_types == o_geo_types
-
-
-def test_matching_indexes(startup):
-    """Test #4: Check if all indexes are present (compare new an old builds)"""
-    stmt = '''
-        SELECT n.nspname as "Schema",
-          c.relname as "Name",
-          CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'i'
-        THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' END as "Type",
-          u.usename as "Owner",
-         c2.relname as "Table"
-        FROM pg_catalog.pg_class c
-             JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid
-             JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid
-             LEFT JOIN pg_catalog.pg_user u ON u.usesysid = c.relowner
-             LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
-        WHERE c.relkind IN ('i','')
-              AND n.nspname NOT IN ('pg_catalog', 'pg_toast')
-              AND pg_catalog.pg_table_is_visible(c.oid)
-              AND c2.relname NOT IN {ignore_tables}
-        ORDER BY 1,2;
-    '''.format(ignore_tables=startup['unused_tables'])
-    new_db_result = startup['new_db'].execute(stmt)
-    old_db_result = startup['old_db'].execute(stmt)
-
-    unmatched_indexes = []
-    for old_row in old_db_result:
-        #assert 1 == 2, (old_row, dir(old_row), old_row.items())
-        found = False
-        if found: continue
-        for new_row in new_db_result:
-            if new_row['Name'] == old_row['Name']:
-                found = True
-                break
-        if not found:
-            unmatched_indexes.append({'name': old_row['Name'], 'table': old_row['Table']})
-    assert len(unmatched_indexes) == 0, (unmatched_indexes)
-    # assert len(new_db_result) == len(old_db_result), (
-    # "new db has {} more indexes.".format(len(new_db_result) - len(old_db_result)))
-
-
-@pytest.fixture(scope="module")
-def teardown():
-    """Teardown fixture: close db connections"""
-    new_db = startup['new_db']
-    old_db = startup['old_db']
-    yield
-    new_db.close()
-    old_db.close()
-    return (new_db, old_db)
-
-
diff --git a/ais/models.py b/ais/models.py
index fbd6e7d8..7191c649 100644
--- a/ais/models.py
+++ b/ais/models.py
@@ -179,6 +179,7 @@ class DorCondominium(db.Model):
     street_full = db.Column(db.Text)
     source_object_id = db.Column(db.Integer)
 
+
 ##############
 # PROPERTIES #
 ##############
@@ -225,7 +226,8 @@ class NG911AddressPoint(db.Model):
     guid = db.Column(db.Text)
     placement_type = db.Column(db.Text)
     geom = db.Column(Geometry(geometry_type='POINT', srid=ENGINE_SRID))
-    
+
+
 #############
 # ADDRESSES #
 #############
@@ -1177,9 +1179,10 @@ def get_address_geoms(self, request=None, i=0):
 
 try:
     class ServiceAreaSummary(db.Model):
-        __table__ = db.Table('service_area_summary',
-                             db.MetaData(bind=db.engine),
-                             autoload=True)
+        with app.app_context():
+            __table__ = db.Table('service_area_summary',
+                                 db.MetaData(bind=db.engine),
+                                 autoload=True)
 except NoSuchTableError:
     ServiceAreaSummary = None
     # if table hasn't been created yet, suppress error
diff --git a/ais/api/tests/__init__.py b/ais/tests/__init__.py
similarity index 100%
rename from ais/api/tests/__init__.py
rename to ais/tests/__init__.py
diff --git a/ais/tests/api/__init__.py b/ais/tests/api/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/ais/api/tests/test_models.py b/ais/tests/api/test_models.py
similarity index 100%
rename from ais/api/tests/test_models.py
rename to ais/tests/api/test_models.py
diff --git a/ais/api/tests/test_paginator.py b/ais/tests/api/test_paginator.py
similarity index 96%
rename from ais/api/tests/test_paginator.py
rename to ais/tests/api/test_paginator.py
index 1df7c2cf..e92fcb4d 100644
--- a/ais/api/tests/test_paginator.py
+++ b/ais/tests/api/test_paginator.py
@@ -7,8 +7,8 @@
 # Test that empty results return a 404
 
 import pytest
-from ..views import validate_page_param
-from ..paginator import Paginator
+from ...api.views import validate_page_param
+from ...api.paginator import Paginator
 
 @pytest.fixture
 def full_paginator():
diff --git a/ais/api/tests/test_views.py b/ais/tests/api/test_views.py
similarity index 98%
rename from ais/api/tests/test_views.py
rename to ais/tests/api/test_views.py
index ecdb6356..8a526d31 100644
--- a/ais/api/tests/test_views.py
+++ b/ais/tests/api/test_views.py
@@ -1,11 +1,12 @@
 import json
 import pytest
-from ais import app, app_db
+from ... import app, app_db
 from operator import eq, gt
 
-@pytest.fixture
+# TODO: Why do these tests all fail with 404 Response Error when run on their own?
+@pytest.fixture(scope='module')
 def client():
-    app.config['TESTING'] = True
+    app.testing = True
     return app.test_client()
 
 def assert_status(response, *expected_status_codes):
@@ -268,10 +269,6 @@ def test_fractional_addresses_are_ok(client):
 
 def test_allows_0_as_address_low_num(client):
     response = client.get('/addresses/0-98 Sharpnack')
-    # data = json.loads(response.get_data().decode())
-    # feature = data['features'][0]
-    # assert_status(response, 200)
-    # assert feature['match_type'] == 'parsed'
     assert_status(response, 404)
 
 def test_allows_0_as_block_low_num(client):
@@ -292,7 +289,7 @@ def test_address_query_can_end_in_comma(client):
     assert_status(response, 200)
 
 def test_opa_query_returns_child_address(client):
-    ignore_addresses = ['1501-53 N 24TH ST', '514-32 N CREIGHTON ST', '901-99 MARKET ST', '630-50 W FISHER AVE', '630R-50 W FISHER AVE', '1501-39 MARKET ST', '8842-54 FRANKFORD AVE', '1131-45 VINE ST', '750-86 N 46TH ST', '1000A-52 FRANKFORD AVE', '4215-19 LUDLOW ST', '3118-98 CHESTNUT ST', '1501S-39 MARKET ST', '3423-35 WEYMOUTH ST', '4421R-51 N PHILIP ST', '5911R-27 BELFIELD AVE',  '4130-50 CITY AVE', '3302R-64 N 3RD ST', '430-32 FAIRMOUNT AVE', '5501-35 E WISTER ST', '4131-63 WHITAKER AVE', '2611-21 W HUNTINGDON ST', '5541 VINE ST', '5539-51 VINE ST']
+    ignore_addresses = ['1501-53 N 24TH ST', '514-32 N CREIGHTON ST', '901-99 MARKET ST', '630-50 W FISHER AVE', '630R-50 W FISHER AVE', '1501-39 MARKET ST', '8842-54 FRANKFORD AVE', '1131-45 VINE ST', '750-86 N 46TH ST', '1000A-52 FRANKFORD AVE', '4215-19 LUDLOW ST', '3118-98 CHESTNUT ST', '1501S-39 MARKET ST', '3423-35 WEYMOUTH ST', '4421R-51 N PHILIP ST', '5911R-27 BELFIELD AVE',  '4130-50 CITY AVE', '3302R-64 N 3RD ST', '430-32 FAIRMOUNT AVE', '5501-35 E WISTER ST', '7326-30 OXFORD AVE', '1214-32 N 26TH ST', '4131-63 WHITAKER AVE']
 
     CHILD_SQL = '''
         SELECT child.street_address, parent.street_address
@@ -307,7 +304,9 @@ def test_opa_query_returns_child_address(client):
           AND parent.street_address not in {}
         LIMIT 1
     '''.format(tuple(ignore_addresses))
-    result = app_db.engine.execute(CHILD_SQL)
+    # Must use the app import like this to get context so we can run SQL commands
+    with app.app_context():
+        result = app_db.engine.execute(CHILD_SQL)
     child_address, parent_address = result.first()
 
     response = client.get('/addresses/{}?opa_only'.format(child_address))
@@ -342,7 +341,9 @@ def test_block_can_exclude_non_opa(client):
             AND (base_address_summary.opa_account_num != address_summary.opa_account_num OR base_address_summary.opa_account_num IS NULL)
           ) AS block_addresses
     '''
-    result = app_db.engine.execute(BLOCK_COUNT_SQL)
+    # Must use the app import like this to get context so we can run SQL commands
+    with app.app_context():
+        result = app_db.engine.execute(BLOCK_COUNT_SQL)
     block_count = result.first()[0]
 
     # Ensure that no join collisions happen
diff --git a/ais/tests/conftest.py b/ais/tests/conftest.py
new file mode 100644
index 00000000..96cdfb59
--- /dev/null
+++ b/ais/tests/conftest.py
@@ -0,0 +1,23 @@
+import pytest
+
+# Add pytest option to allow us to pass test names to skip
+def pytest_addoption(parser):
+    parser.addoption("--skip", action="store", default=None,
+                     help="Specify tests to skip by providing a comma-separated list of test names.")
+
+# Loop through passed --skip value and remove tests from being run
+def pytest_collection_modifyitems(config, items):
+    # Get the list of test names to skip from the command line
+    skip_tests = config.getoption("--skip")
+    if skip_tests:
+        # Split the comma-separated list of test names into a list
+        skip_list = [test.strip() for test in skip_tests.split(",")]
+        # Filter out the tests that are in the skip list
+        deselected = []
+        for item in items:
+            if item.nodeid.split("::")[-1] in skip_list:
+                item.add_marker(pytest.mark.skip(reason="Skipped because --skip option was provided"))
+                deselected.append(item)
+        # Remove the deselected tests from the item list
+        for item in deselected:
+            items.remove(item)
diff --git a/ais/tests/engine/__init__.py b/ais/tests/engine/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/ais/tests/engine/test_engine.py b/ais/tests/engine/test_engine.py
new file mode 100644
index 00000000..e858b929
--- /dev/null
+++ b/ais/tests/engine/test_engine.py
@@ -0,0 +1,190 @@
+import subprocess
+import pytest
+import os
+from ais import app
+import psycopg2
+from psycopg2.extras import RealDictCursor
+# Loads flask vars from ais/instance/config.py
+config = app.config
+
+
+@pytest.fixture
+def startup():
+    """Startup fixture: make database connections and define tables to ignore"""
+    def db_cursor(**creds):
+        conn = psycopg2.connect(**creds)
+        cursor = conn.cursor(cursor_factory=RealDictCursor)
+        return cursor
+
+    # Get our current directory
+    proc = subprocess.Popen(['bash', '-c', 'pwd'], stdout=subprocess.PIPE)
+    output = proc.stdout.read()
+    working_dir = output.rstrip()
+    working_dir = working_dir.decode('utf-8')
+    print("Current working directly is: " + working_dir)
+
+    engine_to_test = os.environ.get('ENGINE_TO_TEST', None)
+    engine_to_compare = os.environ.get('ENGINE_TO_COMPARE', None)
+
+    print(f'running tests against {engine_to_test} and comparing against {engine_to_compare}')
+
+    # Get full creds based on RDS CNAME passed to us
+    if 'blue' in engine_to_compare:
+        engine_to_compare_cur = db_cursor(**config["BLUE_DATABASE"])
+    elif 'green' in engine_to_compare:
+        engine_to_compare_cur = db_cursor(**config["GREEN_DATABASE"])
+
+    if engine_to_test == 'localhost':
+        engine_to_test_cur = db_cursor(**config["LOCAL_BUILD_DATABASE"])
+    else:
+        # Get full creds based on RDS CNAME passed to us
+        if 'blue' in engine_to_test:
+            engine_to_test_cur = db_cursor(**config["BLUE_DATABASE"])
+        elif 'green' in engine_to_test:
+            engine_to_test_cur = db_cursor(**config["GREEN_DATABASE"])
+
+    unused_tables =  ('spatial_ref_sys', 'alembic_version', 'multiple_seg_line', 'service_area_diff', 'address_zip', 'zip_range', 'dor_parcel_address_analysis', 'address_summary_transformed')
+    changed_tables = ('ng911_address_point',)
+    ignore_tables = unused_tables + changed_tables
+
+    return {'engine_to_test_cur': engine_to_test_cur,
+            'engine_to_compare_cur': engine_to_compare_cur,
+            'unused_tables': unused_tables,
+            'changed_tables': changed_tables,
+            'ignore_tables': ignore_tables}
+
+def test_no_duplicates(startup):
+    """ Don't allow duplicate street_addresses in address_summary """
+
+    engine_to_test_cur = startup['engine_to_test_cur']
+    total_stmt = "select count(*) as total_addresses from address_summary"
+    distinct_stmt = "select count(*) as distinct_addresses from (select distinct street_address from address_summary) foo"
+    engine_to_test_cur.execute(total_stmt)
+    num_total_row = engine_to_test_cur.fetchall()
+
+    engine_to_test_cur.execute(distinct_stmt)
+    num_distinct_row = engine_to_test_cur.fetchall()
+
+    num_total = num_total_row[0]['total_addresses']
+    num_distinct = num_distinct_row[0]['distinct_addresses']
+    assert num_total == num_distinct
+
+
+def test_compare_num_tables(startup):
+    """Test #1: Check if all tables are included in build"""
+    # assert len(startup['engine_to_test_cur'].tables) == len(startup['engine_to_compare_cur'].tables)
+    engine_to_test_cur = startup['engine_to_test_cur']
+    engine_to_compare_cur = startup['engine_to_compare_cur']
+    table_count_stmt = "select count(*) from information_schema.tables where table_schema = 'public' AND table_type = 'BASE TABLE' and table_name not in {}".format(str(startup['ignore_tables']))
+    engine_to_test_cur.execute(table_count_stmt)
+    new_table_count = engine_to_test_cur.fetchall()
+
+    engine_to_compare_cur.execute(table_count_stmt)
+    old_table_count = engine_to_compare_cur.fetchall()
+    assert new_table_count == old_table_count
+
+#@pytest.mark.skip(reason="temp change of eclipse_location_ids source table with more rows")
+def test_num_rows_bt_db_tables(startup):
+    """"Test #2: Check if all tables within 10% of rows as old version"""
+    engine_to_test_cur = startup['engine_to_test_cur']
+    engine_to_compare_cur = startup['engine_to_compare_cur']
+    list_tables_stmt = "select table_name from information_schema.tables where table_schema = 'public' AND table_type = 'BASE TABLE'"
+    engine_to_test_cur.execute(list_tables_stmt)
+    engine_to_test_cur_tables = engine_to_test_cur.fetchall()
+
+    engine_to_compare_cur.execute(list_tables_stmt)
+    engine_to_compare_cur_tables = engine_to_compare_cur.fetchall()
+    # engine_to_test_cur_tables = startup['engine_to_test_cur'].tables
+    for ntable in engine_to_test_cur_tables:
+        table_name = ntable['table_name']
+        if table_name in startup['ignore_tables']:
+            continue
+
+        # ndb_table = startup['engine_to_test_cur'][ntable]
+        # n_rows = ndb_table.count
+        row_count_stmt = "select count(*) as count from {}".format(table_name)
+        engine_to_test_cur.execute(row_count_stmt)
+        n_rows = engine_to_test_cur.fetchall()
+
+        engine_to_compare_cur.execute(row_count_stmt)
+        o_rows = engine_to_compare_cur.fetchall()
+        # odb_table = startup['engine_to_compare_cur'][ntable]
+        # o_rows = odb_table.count
+        fdif = abs((n_rows[0]['count'] - o_rows[0]['count']) / o_rows[0]['count'])
+
+        print('Making sure percent of rows that have changed are less than a 10% threshold')
+        assert fdif <= 0.1, (ntable, fdif)
+
+
+#@pytest.mark.skip(reason="added geocode type for ng911")
+def test_geocode_types(startup):
+    """Test #3: Check if all geocode types present (compare new an old builds)"""
+    engine_to_test_cur = startup['engine_to_test_cur']
+    engine_to_compare_cur = startup['engine_to_compare_cur']
+
+    def get_geo_types(db):
+        stmt = "SELECT DISTINCT geocode_type FROM geocode"
+        db.execute(stmt)
+        geo_types = db.fetchall()
+        results = sorted([f['geocode_type'] for f in geo_types])
+
+        return results
+
+    n_geo_types = get_geo_types(engine_to_test_cur)
+    o_geo_types = get_geo_types(engine_to_compare_cur)
+
+    assert n_geo_types == o_geo_types
+
+
+def test_matching_indexes(startup):
+    """Test #4: Check if all indexes are present (compare new an old builds)"""
+    stmt = '''
+        SELECT n.nspname as "Schema",
+          c.relname as "Name",
+          CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'i'
+        THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' END as "Type",
+          u.usename as "Owner",
+         c2.relname as "Table"
+        FROM pg_catalog.pg_class c
+             JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid
+             JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid
+             LEFT JOIN pg_catalog.pg_user u ON u.usesysid = c.relowner
+             LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+        WHERE c.relkind IN ('i','')
+              AND n.nspname NOT IN ('pg_catalog', 'pg_toast')
+              AND pg_catalog.pg_table_is_visible(c.oid)
+              AND c2.relname NOT IN {ignore_tables}
+        ORDER BY 1,2;
+    '''.format(ignore_tables=startup['unused_tables'])
+    startup['engine_to_test_cur'].execute(stmt)
+    engine_to_test_cur_result = startup['engine_to_test_cur'].fetchall()
+    startup['engine_to_compare_cur'].execute(stmt)
+    engine_to_compare_cur_result = startup['engine_to_compare_cur'].fetchall()
+
+    unmatched_indexes = []
+    for old_row in engine_to_compare_cur_result:
+        #assert 1 == 2, (old_row, dir(old_row), old_row.items())
+        found = False
+        if found: continue
+        for new_row in engine_to_test_cur_result:
+            if new_row['Name'] == old_row['Name']:
+                found = True
+                break
+        if not found:
+            unmatched_indexes.append({'name': old_row['Name'], 'table': old_row['Table']})
+    assert len(unmatched_indexes) == 0, (unmatched_indexes)
+    # assert len(engine_to_test_cur_result) == len(engine_to_compare_cur_result), (
+    # "new db has {} more indexes.".format(len(engine_to_test_cur_result) - len(engine_to_compare_cur_result)))
+
+
+@pytest.fixture(scope="module")
+def teardown():
+    """Teardown fixture: close db connections"""
+    engine_to_test_cur = startup['engine_to_test_cur']
+    engine_to_compare_cur = startup['engine_to_compare_cur']
+    yield
+    engine_to_test_cur.close()
+    engine_to_compare_cur.close()
+    return (engine_to_test_cur, engine_to_compare_cur)
+
+
diff --git a/application.py b/application.py
index e3512f08..ca9c0ef9 100644
--- a/application.py
+++ b/application.py
@@ -1,9 +1,7 @@
-from ais import app, manager
+# Import ais from our ais/__init__.py file
+from ais import app as application
+# New flask 2.0 cli
+from flask.cli import FlaskGroup
 
 # Importing ais.api will initialize the app's routes.
 import ais.api.views
-
-if __name__ == '__main__':
-    manager.run()
-else:
-    application = app
diff --git a/bin/eb_env_utils.sh b/bin/eb_env_utils.sh
index 1cc92cf8..4cf3dc87 100644
--- a/bin/eb_env_utils.sh
+++ b/bin/eb_env_utils.sh
@@ -1,116 +1,5 @@
 #!/usr/bin/env bash
 
-EB_ENVS=$(eb list)
-
-get_prod_env() {
-  # Find the environment that is marked as production.
-  for env in $EB_ENVS ; do
-    # Trim carriage returns (\r) off of the env name. On windows, bash will 
-    # strip the new-line (\n) characters but leave the \r.
-    trimmed_env=$(echo $env | tr -d '\r')
-    url=$(eb status $trimmed_env)
-
-    echo "$url" | grep --quiet "ais-api-prod.us-east-1.elasticbeanstalk.com"
-    if [ $? -eq 0 ] ; then
-      echo $trimmed_env
-      return 0
-    fi
-  done
-  # If no environment is found, return with an error.
-  return 1
-}
-
-get_staging_env() {
-  # Find the environment that is either marked as staging or ready to swap in.
-  for env in $EB_ENVS ; do
-    # Trim carriage returns (\r) off of the env name. On windows, bash will
-    # strip the new-line (\n) characters but leave the \r.
-    trimmed_env=$(echo $env | tr -d '\r')
-    url=$(eb status $trimmed_env)
-
-    echo "$url" | grep --quiet "ais-api-staging.us-east-1.elasticbeanstalk.com"
-    if [ $? -eq 0 ] ; then
-      echo $trimmed_env
-      return 0
-    fi
-  done
-  # If no environment is found, return with an error.
-  return 1
-}
-
-get_test_env() {
-  __ENV_VAR_NAME=$1
-  __ENV_STATUS_NAME=$2
-
-  # Find the environment that is marked to swap in.
-  for env in $EB_ENVS ; do
-    # Trim carriage returns (\r) off of the env name. On windows, bash will
-    # strip the new-line (\n) characters but leave the \r.
-    trimmed_env=$(echo $env | tr -d '\r')
-    url=$(eb status $trimmed_env)
-    vars=$(eb printenv $trimmed_env)
-    echo "$url" | grep --quiet "ais-api-staging.us-east-1.elasticbeanstalk.com"
-    if [ $? -eq 0 ] ; then
-      eval "export $__ENV_VAR_NAME=$trimmed_env"
-      echo $vars | grep --quiet "SWAP = True"
-      if [ $? -eq 0 ] ; then
-          eval "export $__ENV_STATUS_NAME=Swap"
-          return 0
-      fi
-    fi
-  done
-
-  # If none is marked to swap, then use the environment marked as production (unless on branch staging):
-  target_url="ais-api-prod.us-east-1.elasticbeanstalk.com"
-  target_env_status="Production"
-  if [ $TRAVIS_BRANCH = "staging" ]; then
-    target_url="ais-api-staging.us-east-1.elasticbeanstalk.com"
-    target_env_status="Staging"
-  fi
-
-  for env in $EB_ENVS ; do
-    trimmed_env=$(echo $env | tr -d '\r')
-    url=$(eb status $trimmed_env)
-    vars=$(eb printenv $trimmed_env)
-
-    echo "$url" | grep --quiet $target_url
-    if [ $? -eq 0 ] ; then
-      eval "export $__ENV_VAR_NAME=$trimmed_env"
-      eval "export $__ENV_STATUS_NAME=$target_env_status"
-      return 0
-    fi
-  done
-
-  # If no environment is found, return with an error.
-  return 1
-}
-
-get_db_uri() {
-
-    trimmed_env=$(echo $1 | tr -d '\r')
-    vars=$(eb printenv $trimmed_env)
-    #uri=$(echo $vars | grep -Po 'SQLALCHEMY.*@.*?:')
-    uri=$(echo $vars | grep -Po 'postgresql://ais_engine:.*?:')
-    uri=${uri#*'@'}
-    uri=$(echo "${uri//:}")
-    #uri=$(echo "${uri//@}")
-    echo $uri
-    return 0
-}
-
-avoid_timeout() {
-    while true; do
-        echo -e "\a"
-        sleep 60
-    done
-}
-
-send_slack() {
-        message=$(echo $1)
-        payload='payload={"channel": "#ais_log", "username": "webhookbot", "text": "'"$message"'", "icon_emoji": ":ghost:"}'
-        curl -X POST --data-urlencode "${payload}" $SLACK_WEBHOOK_URL
-}
-
 send_teams() {
     TEXT=$(echo $1)
     WEBHOOK_URL='https://phila.webhook.office.com/webhookb2/763c9a83-0f38-4eb2-abfc-e0f2f41b6fbb@2046864f-68ea-497d-af34-a6629a6cd700/IncomingWebhook/d23e1f5b3aa54380843d2e7ab54dd689/99fdd4e1-23b3-4f5d-8398-06a885e26925'
diff --git a/build-test-compose.yml b/build-test-compose.yml
new file mode 100644
index 00000000..28f13457
--- /dev/null
+++ b/build-test-compose.yml
@@ -0,0 +1,16 @@
+version: '3.8'
+services:
+  ais:
+    container_name: ais
+    image: ais
+    build: 
+      context: .
+      dockerfile: Dockerfile
+    environment:
+      - ENGINE_DB_HOST
+      - ENGINE_DB_PASS
+      - BLUE_ENGINE_CNAME
+      - GREEN_ENGINE_CNAME
+    ports:
+      - "8080:8080"
+
diff --git a/config.py b/config.py
index e0c8baff..c6d82e58 100644
--- a/config.py
+++ b/config.py
@@ -12,9 +12,9 @@
 from passyunk.parser import PassyunkParser
 PARSER = PassyunkParser
 
-DATABASES = {
+#DATABASES = {
     # these are set in instance config or environment variables
-}
+#}
 
 DEBUG = (os.environ.get('DEBUG', 'False').title() == 'True')
 PROFILE = (os.environ.get('PROFILE', 'False').title() == 'True')
@@ -677,7 +677,7 @@ def make_eclipse_address(comps):
                     'value_field':          'district',
                 },
             },
-        },
+        },       
         {
             'layer_id':                     'political_ward',
             'name':                         'Ward',
diff --git a/docker-build-files/50x.html b/docker-build-files/50x.html
new file mode 100644
index 00000000..a57c2f93
--- /dev/null
+++ b/docker-build-files/50x.html
@@ -0,0 +1,19 @@
+
+
+
+Error
+
+
+
+
An error occurred.

+
Sorry, the page you are looking for is currently unavailable.

+Please try again later.

+
If you are the system administrator of this resource then you should check
+the error log for details.

+
Faithfully yours, nginx.

+
+
diff --git a/docker-build-files/entrypoint.sh b/docker-build-files/entrypoint.sh
new file mode 100644
index 00000000..ac03de2c
--- /dev/null
+++ b/docker-build-files/entrypoint.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+cd /ais
+
+
+# -z mean if unset
+# ! -z mean if set
+if [ -z "${ENGINE_DB_PASS}" ]; then
+    echo 'ENGINE_DB_PASS var not set!'
+    exit 1
+fi
+
+if [ -z "${ENGINE_DB_HOST}" ]; then
+    echo "Did not receive ENGINE_DB_HOST var, attempting to set manually.."
+    prod_color=$(dig ais-prod.phila.city +short | grep -o "blue\|green")
+    if [[ "$PROD_COLOR" -eq "blue" ]]; then
+	if [ -z "${BLUE_ENGINE_CNAME}" ]; then
+	    echo 'BLUE_ENGINE_CNAME var not set!'
+	    exit 1
+	fi
+        export ENGINE_DB_HOST=$BLUE_ENGINE_CNAME
+    fi
+    if [[ "$PROD_COLOR" -eq "green" ]]; then
+	if [ -z "${GREEN_ENGINE_CNAME}" ]; then
+	    echo 'GREEN_ENGINE_CNAME var not set!'
+	    exit 1
+	fi
+        export ENGINE_DB_HOST=$GREEN_ENGINE_CNAME
+    fi
+fi
+
+if [ -z "${ENGINE_DB_HOST}" ]; then
+    echo 'ENGINE_DB_HOST var not set!'
+    exit 1
+fi
+
+# This line has flask serve out the app directly, only for staging
+#python application.py runserver -h 0.0.0.0 -p 80
+
+# Create the configuration file that points ais at it's ais_engine database.
+echo "SQLALCHEMY_DATABASE_URI = \
+    'postgresql://ais_engine:$ENGINE_DB_PASS@$ENGINE_DB_HOST:5432/ais_engine'" >> /ais/instance/config.py
+
+
+#ls /ais/venv/lib/python3.10/site-packages/passyunk/pdata
+#ls /ais/venv/lib/python3.10/site-packages/passyunk_automation/pdata
+
+function fail {
+    printf '%s\n' "$1" >&2 ## Send message to stderr.
+    exit "${2-1}" ## Return a code specified by $2, or 1 by default.
+}
+
+declare -a pdata_files=('alias' 'alias_streets' 'apt' 'apt_std' 'apte'
+ 'centerline' 'centerline_streets' 'directional' 'landmarks' 'name_switch'
+ 'saint' 'std' 'suffix' )
+
+echo "Asserting private data is in passyunk site-package folder"
+for i in "${pdata_files[@]}"
+do
+  test -f /usr/local/lib/python3.10/site-packages/passyunk/pdata/$i.csv || fail "$i.csv does not exist in venv!"
+done
+
+
+declare -a pdata_files=('election_block' 'usps_alias' 'usps_cityzip' 'usps_zip4s')
+
+echo "Asserting private data is in passyunk_automation site-package folder"
+for i in "${pdata_files[@]}"
+do
+  test -f /usr/local/lib/python3.10/site-packages/passyunk_automation/pdata/$i.csv || fail "$i.csv does not exist in venv!"
+done
+echo 'All private data exists.'
+
+# Run nginx as proxy server to gunicorn
+# running like this will start in the background
+nginx
+
+# Gunicorn will be behind nginx, run on socket. Gunicorn must be run in the /ais folder.
+#gunicorn application --bind unix:/tmp/gunicorn.sock --worker-class=gevent --access-logfile '-' --log-level 'debug'
+#gunicorn application --bind unix:/tmp/gunicorn.sock --workers 4 --worker-class=gevent --access-logfile '-' --log-level 'notice'
+#gunicorn application --bind 0.0.0.0:8080 --workers 5 --threads 2 --worker-class gevent --access-logfile '-' --log-level 'notice'
+#gunicorn application --bind 0.0.0.0:8080 --worker-connections 512 --workers 2 --worker-class gevent --access-logfile '-' --log-level 'notice'
+# Nginx will proxy to the socket
+gunicorn application --bind unix:/tmp/gunicorn.sock --workers 4 --worker-class gevent --access-logfile '-' --log-level 'notice'
diff --git a/docker-build-files/nginx.conf b/docker-build-files/nginx.conf
new file mode 100644
index 00000000..30b88fd4
--- /dev/null
+++ b/docker-build-files/nginx.conf
@@ -0,0 +1,70 @@
+user www-data;
+worker_processes 1;
+pid /run/nginx.pid;
+include /etc/nginx/modules-enabled/*.conf;
+worker_rlimit_nofile 20480;
+
+events {
+        worker_connections 2048;
+        accept_mutex on;
+}
+
+http {
+  include mime.types;
+  # fallback in case we can't determine a type
+  default_type application/octet-stream;
+  access_log /var/log/nginx/access.log combined;
+  sendfile on;
+
+  upstream app_server {
+    # fail_timeout=0 means we always retry an upstream even if it failed
+    # to return a good HTTP response
+
+    # for UNIX domain socket setups
+    server unix:/tmp/gunicorn.sock fail_timeout=0;
+  }
+
+  server {
+    # if no Host match, close the connection to prevent host spoofing
+    listen 80 default_server;
+    return 444;
+  }
+
+  server {
+    # Allow loadbalancers in our VPC
+    allow 10.30.100.0/23;
+    # Allow loopback
+    allow 127.0.0.1/32;
+    # Allow docker networking on it's preferred subs
+    allow 172.0.0.0/8;
+    deny all;
+    # use 'listen 80 deferred;' for Linux
+    listen 8080 default_server;
+    client_max_body_size 128M;
+
+    keepalive_timeout 1;
+
+    # path for static files
+    root /var/www/html;
+
+    location / {
+      # checks for static file, if not found proxy to app
+      try_files $uri @proxy_to_app;
+    }
+
+    location @proxy_to_app {
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $http_host;
+      # we don't want nginx trying to do something clever with
+      # redirects, we set the Host: header above already.
+      proxy_redirect off;
+      proxy_pass http://app_server;
+    }
+
+    error_page 500 502 503 504 /50x.html;
+    location = /50x.html {
+      root /var/www/html;
+    }
+  }
+}
diff --git a/ecr-test-compose.yml b/ecr-test-compose.yml
new file mode 100644
index 00000000..dac12c4b
--- /dev/null
+++ b/ecr-test-compose.yml
@@ -0,0 +1,11 @@
+version: '3'
+services:
+  ais:
+    container_name: ais
+    image: 880708401960.dkr.ecr.us-east-1.amazonaws.com/ais
+    environment:
+      - ENGINE_DB_HOST
+      - ENGINE_DB_PASS
+    ports:
+      - "8080:8080"
+
diff --git a/env.example b/env.example
new file mode 100755
index 00000000..9cb985d4
--- /dev/null
+++ b/env.example
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Instructs the docker container to use the local database if true
+export DEV_TEST='false'
+# For the warmup_lb function to be able to make HTTP gets from behind our proxy to AWS, since the ALBs are public endpoints and not internal
+export PROXY_AUTH='http://user:password@proxy.address:8080'
+# Password for the local postgres instance we're building into.
+export LOCAL_PASSWORD=""
+# For warmup_lb.py
+export GATEKEEPER_KEY=""
+
+# Our Microsoft Teams webhook for sending alerts
+export TEAMS_WEBHOOK_URL="https://phila.webhook.office.com/webhookb2/stuff/IncomingWebhook/stuff"
+export TEAMS_WEBHOOK_URL=""
+
+# Hosted private zone IDs for phila.city in both citygeo and mulesoft accounts
+export PHILACITY_ZONE_ID=""
+export MULESOFT_PHILACITY_ZONE_ID=""
+
+# Route 53 DNS for accessing the cluster APIs
+export PROD_ENDPOINT=''
+export STAGE_ENDPOINT=''
+
+# Various db passwords needed throughout the process
+# We will switch between them in the build script depending on need by reassining them with:
+# export PGPASSWORD=$PASSWORD_VAR
+# RDS engine password for ais_engine
+export ENGINE_DB_PASS=""
+# RDS engine password for postgres
+export PG_ENGINE_DB_PASS=""
+# local postgres pass
+export LOCAL_ENGINE_DB_PASS=""
+
+# Access key that can pull from our S3 directory containing election_block.csv and usps_zip4s.csv 
+export AWS_ACCESS_KEY_ID=""
+export AWS_SECRET_ACCESS_KEY=""
diff --git a/manage.py b/manage.py
index 368b9c3f..a27d2af6 100644
--- a/manage.py
+++ b/manage.py
@@ -1,13 +1,19 @@
 import os
 from flask_script import Command, Option
-from ais import manager
+#from ais import manager
+from flask.cli import FlaskGroup
+from flask.cli import with_appcontext
 
+cli = FlaskGroup(app)
+
+@click.command(name='engine')
+@with_appcontext
 class EngineCommand(Command):
     """Management command for engine scripts."""
     def run(self):
         print('ok')
 
-manager.add_command('engine', EngineCommand)
+app.cli.add_command(create)
 
 
 # Loop over .py files in /scripts
@@ -20,4 +26,5 @@ def run(self):
         script_path = os.path.abspath(os.path.join(root, file))
         cmd = ScriptCommand(name, script_path)
 
-manager.run()
+if __name__ == "__main__":
+    cli()
diff --git a/migrations/README b/migrations/README
index 98e4f9c4..0e048441 100644
--- a/migrations/README
+++ b/migrations/README
@@ -1 +1 @@
-Generic single-database configuration.
\ No newline at end of file
+Single-database configuration for Flask.
diff --git a/migrations/alembic.ini b/migrations/alembic.ini
index f8ed4801..ec9d45c2 100644
--- a/migrations/alembic.ini
+++ b/migrations/alembic.ini
@@ -11,7 +11,7 @@
 
 # Logging configuration
 [loggers]
-keys = root,sqlalchemy,alembic
+keys = root,sqlalchemy,alembic,flask_migrate
 
 [handlers]
 keys = console
@@ -34,6 +34,11 @@ level = INFO
 handlers =
 qualname = alembic
 
+[logger_flask_migrate]
+level = INFO
+handlers =
+qualname = flask_migrate
+
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
diff --git a/migrations/env.py b/migrations/env.py
index 45938160..55e9df94 100644
--- a/migrations/env.py
+++ b/migrations/env.py
@@ -1,8 +1,11 @@
 from __future__ import with_statement
-from alembic import context
-from sqlalchemy import engine_from_config, pool
-from logging.config import fileConfig
+
 import logging
+from logging.config import fileConfig
+
+from flask import current_app
+
+from alembic import context
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -17,10 +20,11 @@
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
-from flask import current_app
-config.set_main_option('sqlalchemy.url',
-                       current_app.config.get('SQLALCHEMY_DATABASE_URI'))
-target_metadata = current_app.extensions['migrate'].db.metadata
+config.set_main_option(
+    'sqlalchemy.url',
+    str(current_app.extensions['migrate'].db.get_engine().url).replace(
+        '%', '%%'))
+target_db = current_app.extensions['migrate'].db
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
@@ -28,6 +32,12 @@
 # ... etc.
 
 
+def get_metadata():
+    if hasattr(target_db, 'metadatas'):
+        return target_db.metadatas[None]
+    return target_db.metadata
+
+
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
 
@@ -41,7 +51,9 @@ def run_migrations_offline():
 
     """
     url = config.get_main_option("sqlalchemy.url")
-    context.configure(url=url)
+    context.configure(
+        url=url, target_metadata=get_metadata(), literal_binds=True
+    )
 
     with context.begin_transaction():
         context.run_migrations()
@@ -57,7 +69,7 @@ def run_migrations_online():
 
     # this callback is used to prevent an auto-migration from being generated
     # when there are no changes to the schema
-    # reference: http://alembic.readthedocs.org/en/latest/cookbook.html
+    # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
     def process_revision_directives(context, revision, directives):
         if getattr(config.cmd_opts, 'autogenerate', False):
             script = directives[0]
@@ -65,21 +77,19 @@ def process_revision_directives(context, revision, directives):
                 directives[:] = []
                 logger.info('No changes in schema detected.')
 
-    engine = engine_from_config(config.get_section(config.config_ini_section),
-                                prefix='sqlalchemy.',
-                                poolclass=pool.NullPool)
+    connectable = current_app.extensions['migrate'].db.get_engine()
 
-    connection = engine.connect()
-    context.configure(connection=connection,
-                      target_metadata=target_metadata,
-                      process_revision_directives=process_revision_directives,
-                      **current_app.extensions['migrate'].configure_args)
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=get_metadata(),
+            process_revision_directives=process_revision_directives,
+            **current_app.extensions['migrate'].configure_args
+        )
 
-    try:
         with context.begin_transaction():
             context.run_migrations()
-    finally:
-        connection.close()
+
 
 if context.is_offline_mode():
     run_migrations_offline()
diff --git a/migrations/script.py.mako b/migrations/script.py.mako
index 95702017..2c015630 100644
--- a/migrations/script.py.mako
+++ b/migrations/script.py.mako
@@ -1,18 +1,20 @@
 """${message}
 
 Revision ID: ${up_revision}
-Revises: ${down_revision}
+Revises: ${down_revision | comma,n}
 Create Date: ${create_date}
 
 """
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
 
 # revision identifiers, used by Alembic.
 revision = ${repr(up_revision)}
 down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
 
-from alembic import op
-import sqlalchemy as sa
-${imports if imports else ""}
 
 def upgrade():
     ${upgrades if upgrades else "pass"}
diff --git a/migrations/versions/112f42c5fd48_new_initial_migration.py b/migrations/versions/112f42c5fd48_new_initial_migration.py
deleted file mode 100644
index 3c9fb151..00000000
--- a/migrations/versions/112f42c5fd48_new_initial_migration.py
+++ /dev/null
@@ -1,474 +0,0 @@
-"""new initial migration
-
-Revision ID: 112f42c5fd48
-Revises: None
-Create Date: 2017-10-26 15:19:13.759396
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = '112f42c5fd48'
-down_revision = None
-
-from alembic import op
-import sqlalchemy as sa
-import geoalchemy2
-
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('address',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('street_address', sa.Text(), nullable=True),
-    sa.Column('address_low', sa.Integer(), nullable=True),
-    sa.Column('address_low_suffix', sa.Text(), nullable=True),
-    sa.Column('address_low_frac', sa.Text(), nullable=True),
-    sa.Column('address_high', sa.Integer(), nullable=True),
-    sa.Column('street_predir', sa.Text(), nullable=True),
-    sa.Column('street_name', sa.Text(), nullable=True),
-    sa.Column('street_suffix', sa.Text(), nullable=True),
-    sa.Column('street_postdir', sa.Text(), nullable=True),
-    sa.Column('unit_type', sa.Text(), nullable=True),
-    sa.Column('unit_num', sa.Text(), nullable=True),
-    sa.Column('street_full', sa.Text(), nullable=True),
-    sa.Column('zip_code', sa.Text(), nullable=True),
-    sa.Column('zip_4', sa.Text(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('address_error',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('source_name', sa.Text(), nullable=True),
-    sa.Column('source_address', sa.Text(), nullable=True),
-    sa.Column('street_address', sa.Text(), nullable=True),
-    sa.Column('level', sa.Text(), nullable=True),
-    sa.Column('reason', sa.Text(), nullable=True),
-    sa.Column('notes', sa.Text(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('address_link',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('address_1', sa.Text(), nullable=True),
-    sa.Column('relationship', sa.Text(), nullable=True),
-    sa.Column('address_2', sa.Text(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('address_parcel',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('street_address', sa.Text(), nullable=True),
-    sa.Column('parcel_source', sa.Text(), nullable=True),
-    sa.Column('parcel_row_id', sa.Integer(), nullable=True),
-    sa.Column('match_type', sa.Text(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('address_property',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('street_address', sa.Text(), nullable=True),
-    sa.Column('opa_account_num', sa.Text(), nullable=True),
-    sa.Column('match_type', sa.Text(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('address_street',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('street_address', sa.Text(), nullable=True),
-    sa.Column('seg_id', sa.Integer(), nullable=True),
-    sa.Column('seg_side', sa.Text(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('address_summary',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('street_address', sa.Text(), nullable=True),
-    sa.Column('address_low', sa.Integer(), nullable=True),
-    sa.Column('address_low_suffix', sa.Text(), nullable=True),
-    sa.Column('address_low_frac', sa.Text(), nullable=True),
-    sa.Column('address_high', sa.Integer(), nullable=True),
-    sa.Column('street_predir', sa.Text(), nullable=True),
-    sa.Column('street_name', sa.Text(), nullable=True),
-    sa.Column('street_suffix', sa.Text(), nullable=True),
-    sa.Column('street_postdir', sa.Text(), nullable=True),
-    sa.Column('unit_type', sa.Text(), nullable=True),
-    sa.Column('unit_num', sa.Text(), nullable=True),
-    sa.Column('street_full', sa.Text(), nullable=True),
-    sa.Column('zip_code', sa.Text(), nullable=True),
-    sa.Column('zip_4', sa.Text(), nullable=True),
-    sa.Column('usps_bldgfirm', sa.Text(), nullable=True),
-    sa.Column('usps_type', sa.Text(), nullable=True),
-    sa.Column('election_block_id', sa.Text(), nullable=True),
-    sa.Column('election_precinct', sa.Text(), nullable=True),
-    sa.Column('street_code', sa.Integer(), nullable=True),
-    sa.Column('seg_id', sa.Integer(), nullable=True),
-    sa.Column('seg_side', sa.Text(), nullable=True),
-    sa.Column('pwd_parcel_id', sa.Text(), nullable=True),
-    sa.Column('dor_parcel_id', sa.Text(), nullable=True),
-    sa.Column('opa_account_num', sa.Text(), nullable=True),
-    sa.Column('opa_owners', sa.Text(), nullable=True),
-    sa.Column('opa_address', sa.Text(), nullable=True),
-    sa.Column('info_residents', sa.Text(), nullable=True),
-    sa.Column('info_companies', sa.Text(), nullable=True),
-    sa.Column('pwd_account_nums', sa.Text(), nullable=True),
-    sa.Column('li_address_key', sa.Text(), nullable=True),
-    sa.Column('eclipse_location_id', sa.Text(), nullable=True),
-    sa.Column('bin', sa.Text(), nullable=True),
-    sa.Column('zoning_document_ids', sa.Text(), nullable=True),
-    sa.Column('voters', sa.Text(), nullable=True),
-    sa.Column('geocode_type', sa.Text(), nullable=True),
-    sa.Column('geocode_x', sa.Float(), nullable=True),
-    sa.Column('geocode_y', sa.Float(), nullable=True),
-    sa.Column('geocode_street_x', sa.Float(), nullable=True),
-    sa.Column('geocode_street_y', sa.Float(), nullable=True),
-    sa.PrimaryKeyConstraint('id'),
-    sa.UniqueConstraint('street_address')
-    )
-    op.create_index('address_summary_sort_idx', 'address_summary', ['street_name', 'street_suffix', 'street_predir', 'street_postdir', 'address_low', 'address_high', 'unit_num'], unique=False, postgresql_using='btree')
-    op.create_index(op.f('ix_address_summary_dor_parcel_id'), 'address_summary', ['dor_parcel_id'], unique=False)
-    op.create_index(op.f('ix_address_summary_opa_account_num'), 'address_summary', ['opa_account_num'], unique=False)
-    op.create_index(op.f('ix_address_summary_pwd_parcel_id'), 'address_summary', ['pwd_parcel_id'], unique=False)
-    op.create_table('address_tag',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('street_address', sa.Text(), nullable=True),
-    sa.Column('key', sa.Text(), nullable=True),
-    sa.Column('value', sa.Text(), nullable=True),
-    sa.Column('linked_address', sa.Text(), nullable=True),
-    sa.Column('linked_path', sa.Text(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('address_zip',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('street_address', sa.Text(), nullable=True),
-    sa.Column('usps_id', sa.Text(), nullable=True),
-    sa.Column('match_type', sa.Text(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('curb',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('curb_id', sa.Integer(), nullable=True),
-    sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='MULTIPOLYGON', srid=2272), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('dor_parcel',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('parcel_id', sa.Text(), nullable=True),
-    sa.Column('street_address', sa.Text(), nullable=True),
-    sa.Column('address_low', sa.Integer(), nullable=True),
-    sa.Column('address_low_suffix', sa.Text(), nullable=True),
-    sa.Column('address_low_frac', sa.Text(), nullable=True),
-    sa.Column('address_high', sa.Integer(), nullable=True),
-    sa.Column('street_predir', sa.Text(), nullable=True),
-    sa.Column('street_name', sa.Text(), nullable=True),
-    sa.Column('street_suffix', sa.Text(), nullable=True),
-    sa.Column('street_postdir', sa.Text(), nullable=True),
-    sa.Column('unit_type', sa.Text(), nullable=True),
-    sa.Column('unit_num', sa.Text(), nullable=True),
-    sa.Column('street_full', sa.Text(), nullable=True),
-    sa.Column('source_object_id', sa.Integer(), nullable=True),
-    sa.Column('source_address', sa.Text(), nullable=True),
-    sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='MULTIPOLYGON', srid=2272), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_index(op.f('ix_dor_parcel_parcel_id'), 'dor_parcel', ['parcel_id'], unique=False)
-    op.create_table('dor_parcel_address_analysis',
-    sa.Column('objectid', sa.Integer(), nullable=False),
-    sa.Column('mapreg', sa.Text(), nullable=True),
-    sa.Column('stcod', sa.Integer(), nullable=True),
-    sa.Column('house', sa.Integer(), nullable=True),
-    sa.Column('suf', sa.Text(), nullable=True),
-    sa.Column('unit', sa.Text(), nullable=True),
-    sa.Column('stex', sa.Integer(), nullable=True),
-    sa.Column('stdir', sa.Text(), nullable=True),
-    sa.Column('stnam', sa.Text(), nullable=True),
-    sa.Column('stdes', sa.Text(), nullable=True),
-    sa.Column('stdessuf', sa.Text(), nullable=True),
-    sa.Column('concatenated_address', sa.Text(), nullable=True),
-    sa.Column('std_street_address', sa.Text(), nullable=True),
-    sa.Column('std_address_low', sa.Integer(), nullable=True),
-    sa.Column('std_address_low_suffix', sa.Text(), nullable=True),
-    sa.Column('std_high_num', sa.Integer(), nullable=True),
-    sa.Column('std_street_predir', sa.Text(), nullable=True),
-    sa.Column('std_street_name', sa.Text(), nullable=True),
-    sa.Column('std_street_suffix', sa.Text(), nullable=True),
-    sa.Column('std_address_postdir', sa.Text(), nullable=True),
-    sa.Column('std_unit_type', sa.Text(), nullable=True),
-    sa.Column('std_unit_num', sa.Text(), nullable=True),
-    sa.Column('std_street_code', sa.Integer(), nullable=True),
-    sa.Column('std_seg_id', sa.Integer(), nullable=True),
-    sa.Column('cl_addr_match', sa.Text(), nullable=True),
-    sa.Column('change_stcod', sa.Integer(), nullable=True),
-    sa.Column('change_house', sa.Integer(), nullable=True),
-    sa.Column('change_suf', sa.Integer(), nullable=True),
-    sa.Column('change_unit', sa.Integer(), nullable=True),
-    sa.Column('change_stex', sa.Integer(), nullable=True),
-    sa.Column('change_stdir', sa.Integer(), nullable=True),
-    sa.Column('change_stnam', sa.Integer(), nullable=True),
-    sa.Column('change_stdes', sa.Integer(), nullable=True),
-    sa.Column('change_stdessuf', sa.Integer(), nullable=True),
-    sa.Column('no_address', sa.Integer(), nullable=True),
-    sa.Column('shape', geoalchemy2.types.Geometry(geometry_type='MULTIPOLYGON', srid=2272), nullable=True),
-    sa.PrimaryKeyConstraint('objectid')
-    )
-    op.create_table('dor_parcel_error',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('objectid', sa.Integer(), nullable=True),
-    sa.Column('mapreg', sa.Text(), nullable=True),
-    sa.Column('stcod', sa.Integer(), nullable=True),
-    sa.Column('house', sa.Integer(), nullable=True),
-    sa.Column('suf', sa.Text(), nullable=True),
-    sa.Column('stex', sa.Text(), nullable=True),
-    sa.Column('stdir', sa.Text(), nullable=True),
-    sa.Column('stnam', sa.Text(), nullable=True),
-    sa.Column('stdes', sa.Text(), nullable=True),
-    sa.Column('stdessuf', sa.Text(), nullable=True),
-    sa.Column('unit', sa.Text(), nullable=True),
-    sa.Column('level', sa.Text(), nullable=True),
-    sa.Column('reason', sa.Text(), nullable=True),
-    sa.Column('notes', sa.Text(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('dor_parcel_error_polygon',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('objectid', sa.Integer(), nullable=True),
-    sa.Column('mapreg', sa.Text(), nullable=True),
-    sa.Column('stcod', sa.Integer(), nullable=True),
-    sa.Column('house', sa.Integer(), nullable=True),
-    sa.Column('suf', sa.Text(), nullable=True),
-    sa.Column('stex', sa.Text(), nullable=True),
-    sa.Column('stdir', sa.Text(), nullable=True),
-    sa.Column('stnam', sa.Text(), nullable=True),
-    sa.Column('stdes', sa.Text(), nullable=True),
-    sa.Column('stdessuf', sa.Text(), nullable=True),
-    sa.Column('unit', sa.Text(), nullable=True),
-    sa.Column('shape', geoalchemy2.types.Geometry(geometry_type='MULTIPOLYGON', srid=2272), nullable=True),
-    sa.Column('reasons', sa.Text(), nullable=True),
-    sa.Column('reason_count', sa.Integer(), nullable=True),
-    sa.Column('notes', sa.Text(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('geocode',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('street_address', sa.Text(), nullable=True),
-    sa.Column('geocode_type', sa.Integer(), nullable=True),
-    sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='POINT', srid=2272), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('multiple_seg_line',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('street_address', sa.Text(), nullable=True),
-    sa.Column('parent_address', sa.Text(), nullable=True),
-    sa.Column('seg_id', sa.Integer(), nullable=True),
-    sa.Column('parcel_source', sa.Text(), nullable=True),
-    sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='LINESTRING', srid=2272), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('opa_property',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('account_num', sa.Text(), nullable=True),
-    sa.Column('street_address', sa.Text(), nullable=True),
-    sa.Column('address_low', sa.Integer(), nullable=True),
-    sa.Column('address_low_suffix', sa.Text(), nullable=True),
-    sa.Column('address_low_frac', sa.Text(), nullable=True),
-    sa.Column('address_high', sa.Integer(), nullable=True),
-    sa.Column('street_predir', sa.Text(), nullable=True),
-    sa.Column('street_name', sa.Text(), nullable=True),
-    sa.Column('street_suffix', sa.Text(), nullable=True),
-    sa.Column('street_postdir', sa.Text(), nullable=True),
-    sa.Column('unit_type', sa.Text(), nullable=True),
-    sa.Column('unit_num', sa.Text(), nullable=True),
-    sa.Column('street_full', sa.Text(), nullable=True),
-    sa.Column('source_address', sa.Text(), nullable=True),
-    sa.Column('tencode', sa.Text(), nullable=True),
-    sa.Column('owners', sa.Text(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_index(op.f('ix_opa_property_account_num'), 'opa_property', ['account_num'], unique=False)
-    op.create_table('parcel_curb',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('parcel_source', sa.Text(), nullable=True),
-    sa.Column('parcel_row_id', sa.Text(), nullable=True),
-    sa.Column('curb_id', sa.Integer(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('pwd_parcel',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('parcel_id', sa.Integer(), nullable=True),
-    sa.Column('street_address', sa.Text(), nullable=True),
-    sa.Column('address_low', sa.Integer(), nullable=True),
-    sa.Column('address_low_suffix', sa.Text(), nullable=True),
-    sa.Column('address_low_frac', sa.Text(), nullable=True),
-    sa.Column('address_high', sa.Integer(), nullable=True),
-    sa.Column('street_predir', sa.Text(), nullable=True),
-    sa.Column('street_name', sa.Text(), nullable=True),
-    sa.Column('street_suffix', sa.Text(), nullable=True),
-    sa.Column('street_postdir', sa.Text(), nullable=True),
-    sa.Column('unit_type', sa.Text(), nullable=True),
-    sa.Column('unit_num', sa.Text(), nullable=True),
-    sa.Column('street_full', sa.Text(), nullable=True),
-    sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='MULTIPOLYGON', srid=2272), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_index(op.f('ix_pwd_parcel_parcel_id'), 'pwd_parcel', ['parcel_id'], unique=False)
-    op.create_table('service_area_diff',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('street_address', sa.Text(), nullable=True),
-    sa.Column('layer_id', sa.Text(), nullable=True),
-    sa.Column('ais_value', sa.Text(), nullable=True),
-    sa.Column('ulrs_value', sa.Text(), nullable=True),
-    sa.Column('distance', sa.Float(), nullable=True),
-    sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='POINT', srid=2272), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('service_area_layer',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('layer_id', sa.Text(), nullable=True),
-    sa.Column('name', sa.Text(), nullable=True),
-    sa.Column('description', sa.Text(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('service_area_line_dual',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('layer_id', sa.Text(), nullable=True),
-    sa.Column('source_object_id', sa.Integer(), nullable=True),
-    sa.Column('seg_id', sa.Integer(), nullable=True),
-    sa.Column('left_value', sa.Text(), nullable=True),
-    sa.Column('right_value', sa.Text(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('service_area_line_single',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('layer_id', sa.Text(), nullable=True),
-    sa.Column('source_object_id', sa.Integer(), nullable=True),
-    sa.Column('seg_id', sa.Integer(), nullable=True),
-    sa.Column('value', sa.Text(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('service_area_point',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('layer_id', sa.Text(), nullable=True),
-    sa.Column('source_object_id', sa.Integer(), nullable=True),
-    sa.Column('seg_id', sa.Integer(), nullable=True),
-    sa.Column('value', sa.Text(), nullable=True),
-    sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='MULTIPOINT', srid=2272), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('service_area_polygon',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('layer_id', sa.Text(), nullable=True),
-    sa.Column('source_object_id', sa.Integer(), nullable=True),
-    sa.Column('value', sa.Text(), nullable=True),
-    sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='MULTIPOLYGON', srid=2272), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('source_address',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('source_name', sa.Text(), nullable=True),
-    sa.Column('source_address', sa.Text(), nullable=True),
-    sa.Column('street_address', sa.Text(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('street_alias',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('street_predir', sa.Text(), nullable=True),
-    sa.Column('street_name', sa.Text(), nullable=True),
-    sa.Column('street_suffix', sa.Text(), nullable=True),
-    sa.Column('street_postdir', sa.Text(), nullable=True),
-    sa.Column('street_full', sa.Text(), nullable=True),
-    sa.Column('seg_id', sa.Integer(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('street_intersection',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('node_id', sa.Integer(), nullable=True),
-    sa.Column('int_id', sa.Integer(), nullable=True),
-    sa.Column('street_1_full', sa.Text(), nullable=True),
-    sa.Column('street_1_name', sa.Text(), nullable=True),
-    sa.Column('street_1_code', sa.Text(), nullable=True),
-    sa.Column('street_1_predir', sa.Text(), nullable=True),
-    sa.Column('street_1_postdir', sa.Text(), nullable=True),
-    sa.Column('street_1_suffix', sa.Text(), nullable=True),
-    sa.Column('street_2_full', sa.Text(), nullable=True),
-    sa.Column('street_2_name', sa.Text(), nullable=True),
-    sa.Column('street_2_code', sa.Text(), nullable=True),
-    sa.Column('street_2_predir', sa.Text(), nullable=True),
-    sa.Column('street_2_postdir', sa.Text(), nullable=True),
-    sa.Column('street_2_suffix', sa.Text(), nullable=True),
-    sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='POINT', srid=2272), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_index(op.f('ix_street_intersection_street_1_code'), 'street_intersection', ['street_1_code'], unique=False)
-    op.create_index(op.f('ix_street_intersection_street_2_code'), 'street_intersection', ['street_2_code'], unique=False)
-    op.create_table('street_segment',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('seg_id', sa.Integer(), nullable=True),
-    sa.Column('street_code', sa.Integer(), nullable=True),
-    sa.Column('street_predir', sa.Text(), nullable=True),
-    sa.Column('street_name', sa.Text(), nullable=True),
-    sa.Column('street_suffix', sa.Text(), nullable=True),
-    sa.Column('street_postdir', sa.Text(), nullable=True),
-    sa.Column('street_full', sa.Text(), nullable=True),
-    sa.Column('left_from', sa.Integer(), nullable=True),
-    sa.Column('left_to', sa.Integer(), nullable=True),
-    sa.Column('right_from', sa.Integer(), nullable=True),
-    sa.Column('right_to', sa.Integer(), nullable=True),
-    sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='LINESTRING', srid=2272), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('zip_range',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('usps_id', sa.Text(), nullable=True),
-    sa.Column('address_low', sa.Integer(), nullable=True),
-    sa.Column('address_high', sa.Integer(), nullable=True),
-    sa.Column('address_oeb', sa.Text(), nullable=True),
-    sa.Column('street_predir', sa.Text(), nullable=True),
-    sa.Column('street_name', sa.Text(), nullable=True),
-    sa.Column('street_suffix', sa.Text(), nullable=True),
-    sa.Column('street_postdir', sa.Text(), nullable=True),
-    sa.Column('unit_type', sa.Text(), nullable=True),
-    sa.Column('unit_low', sa.Text(), nullable=True),
-    sa.Column('unit_high', sa.Text(), nullable=True),
-    sa.Column('unit_oeb', sa.Text(), nullable=True),
-    sa.Column('zip_code', sa.Text(), nullable=True),
-    sa.Column('zip_4', sa.Text(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('zip_range')
-    op.drop_table('street_segment')
-    op.drop_index(op.f('ix_street_intersection_street_2_code'), table_name='street_intersection')
-    op.drop_index(op.f('ix_street_intersection_street_1_code'), table_name='street_intersection')
-    op.drop_table('street_intersection')
-    op.drop_table('street_alias')
-    op.drop_table('source_address')
-    op.drop_table('service_area_polygon')
-    op.drop_table('service_area_point')
-    op.drop_table('service_area_line_single')
-    op.drop_table('service_area_line_dual')
-    op.drop_table('service_area_layer')
-    op.drop_table('service_area_diff')
-    op.drop_index(op.f('ix_pwd_parcel_parcel_id'), table_name='pwd_parcel')
-    op.drop_table('pwd_parcel')
-    op.drop_table('parcel_curb')
-    op.drop_index(op.f('ix_opa_property_account_num'), table_name='opa_property')
-    op.drop_table('opa_property')
-    op.drop_table('multiple_seg_line')
-    op.drop_table('geocode')
-    op.drop_table('dor_parcel_error_polygon')
-    op.drop_table('dor_parcel_error')
-    op.drop_table('dor_parcel_address_analysis')
-    op.drop_index(op.f('ix_dor_parcel_parcel_id'), table_name='dor_parcel')
-    op.drop_table('dor_parcel')
-    op.drop_table('curb')
-    op.drop_table('address_zip')
-    op.drop_table('address_tag')
-    op.drop_index(op.f('ix_address_summary_pwd_parcel_id'), table_name='address_summary')
-    op.drop_index(op.f('ix_address_summary_opa_account_num'), table_name='address_summary')
-    op.drop_index(op.f('ix_address_summary_dor_parcel_id'), table_name='address_summary')
-    op.drop_index('address_summary_sort_idx', table_name='address_summary')
-    op.drop_table('address_summary')
-    op.drop_table('address_street')
-    op.drop_table('address_property')
-    op.drop_table('address_parcel')
-    op.drop_table('address_link')
-    op.drop_table('address_error')
-    op.drop_table('address')
-    ### end Alembic commands ###
diff --git a/migrations/versions/1adfc3a48581_.py b/migrations/versions/1adfc3a48581_.py
deleted file mode 100644
index 790f834b..00000000
--- a/migrations/versions/1adfc3a48581_.py
+++ /dev/null
@@ -1,26 +0,0 @@
-"""empty message
-
-Revision ID: 1adfc3a48581
-Revises: a31ad02fb246
-Create Date: 2019-01-14 17:15:18.765425
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = '1adfc3a48581'
-down_revision = 'a31ad02fb246'
-
-from alembic import op
-import sqlalchemy as sa
-
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('address_summary', sa.Column('bin_parcel_id', sa.Text(), nullable=True))
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.drop_column('address_summary', 'bin_parcel_id')
-    ### end Alembic commands ###
diff --git a/migrations/versions/4c665272f1fc_.py b/migrations/versions/4c665272f1fc_.py
deleted file mode 100644
index f1de72a4..00000000
--- a/migrations/versions/4c665272f1fc_.py
+++ /dev/null
@@ -1,35 +0,0 @@
-"""empty message
-
-Revision ID: 4c665272f1fc
-Revises: 9bda5e5e605c
-Create Date: 2018-04-24 20:52:09.959157
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = '4c665272f1fc'
-down_revision = '9bda5e5e605c'
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('dor_condominium_error',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('parcel_id', sa.Text(), nullable=True),
-    sa.Column('unit_num', sa.Text(), nullable=True),
-    sa.Column('source_object_id', sa.Integer(), nullable=True),
-    sa.Column('reason', sa.Text(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_index(op.f('ix_dor_condominium_error_parcel_id'), 'dor_condominium_error', ['parcel_id'], unique=False)
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.drop_index(op.f('ix_dor_condominium_error_parcel_id'), table_name='dor_condominium_error')
-    op.drop_table('dor_condominium_error')
-    ### end Alembic commands ###
diff --git a/migrations/versions/4cb88ab1c025_.py b/migrations/versions/4cb88ab1c025_.py
deleted file mode 100644
index 8ad52ae6..00000000
--- a/migrations/versions/4cb88ab1c025_.py
+++ /dev/null
@@ -1,30 +0,0 @@
-"""empty message
-
-Revision ID: 4cb88ab1c025
-Revises: 112f42c5fd48
-Create Date: 2018-02-23 14:38:24.184815
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = '4cb88ab1c025'
-down_revision = '112f42c5fd48'
-
-from alembic import op
-import sqlalchemy as sa
-
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('dor_parcel_address_analysis', sa.Column('num_parcels_w_address', sa.Integer(), nullable=True))
-    op.add_column('dor_parcel_address_analysis', sa.Column('num_parcels_w_mapreg', sa.Integer(), nullable=True))
-    op.add_column('dor_parcel_address_analysis', sa.Column('opa_account_nums', sa.Text(), nullable=True))
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.drop_column('dor_parcel_address_analysis', 'opa_account_nums')
-    op.drop_column('dor_parcel_address_analysis', 'num_parcels_w_mapreg')
-    op.drop_column('dor_parcel_address_analysis', 'num_parcels_w_address')
-    ### end Alembic commands ###
diff --git a/migrations/versions/61c98f1023cc_.py b/migrations/versions/61c98f1023cc_.py
deleted file mode 100644
index ddf080c9..00000000
--- a/migrations/versions/61c98f1023cc_.py
+++ /dev/null
@@ -1,26 +0,0 @@
-"""empty message
-
-Revision ID: 61c98f1023cc
-Revises: 4cb88ab1c025
-Create Date: 2018-03-22 12:57:34.052836
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = '61c98f1023cc'
-down_revision = '4cb88ab1c025'
-
-from alembic import op
-import sqlalchemy as sa
-
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.create_index(op.f('ix_address_summary_seg_id'), 'address_summary', ['seg_id'], unique=False)
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.drop_index(op.f('ix_address_summary_seg_id'), table_name='address_summary')
-    ### end Alembic commands ###
diff --git a/migrations/versions/9bda5e5e605c_.py b/migrations/versions/9bda5e5e605c_.py
deleted file mode 100644
index 3b1bbabe..00000000
--- a/migrations/versions/9bda5e5e605c_.py
+++ /dev/null
@@ -1,44 +0,0 @@
-"""empty message
-
-Revision ID: 9bda5e5e605c
-Revises: c98c0dbcdf02
-Create Date: 2018-04-23 16:07:06.803130
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = '9bda5e5e605c'
-down_revision = '61c98f1023cc'
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('dor_condominium',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('parcel_id', sa.Text(), nullable=True),
-    sa.Column('street_address', sa.Text(), nullable=True),
-    sa.Column('address_low', sa.Integer(), nullable=True),
-    sa.Column('address_low_suffix', sa.Text(), nullable=True),
-    sa.Column('address_low_frac', sa.Text(), nullable=True),
-    sa.Column('address_high', sa.Integer(), nullable=True),
-    sa.Column('street_predir', sa.Text(), nullable=True),
-    sa.Column('street_name', sa.Text(), nullable=True),
-    sa.Column('street_suffix', sa.Text(), nullable=True),
-    sa.Column('street_postdir', sa.Text(), nullable=True),
-    sa.Column('unit_num', sa.Text(), nullable=True),
-    sa.Column('street_full', sa.Text(), nullable=True),
-    sa.Column('source_object_id', sa.Integer(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_index(op.f('ix_dor_condominium_parcel_id'), 'dor_condominium', ['parcel_id'], unique=False)
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.drop_index(op.f('ix_dor_condominium_parcel_id'), table_name='dor_condominium')
-    op.drop_table('dor_condominium')
-    ### end Alembic commands ###
diff --git a/migrations/versions/a31ad02fb246_.py b/migrations/versions/a31ad02fb246_.py
deleted file mode 100644
index f6275033..00000000
--- a/migrations/versions/a31ad02fb246_.py
+++ /dev/null
@@ -1,26 +0,0 @@
-"""empty message
-
-Revision ID: a31ad02fb246
-Revises: 4c665272f1fc
-Create Date: 2018-05-30 14:08:46.399360
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = 'a31ad02fb246'
-down_revision = '4c665272f1fc'
-
-from alembic import op
-import sqlalchemy as sa
-
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('dor_parcel_address_analysis', sa.Column('status', sa.Integer(), nullable=True))
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.drop_column('dor_parcel_address_analysis', 'status')
-    ### end Alembic commands ###
diff --git a/requirements.app.txt b/requirements.app.txt
deleted file mode 100644
index 1c55cbd2..00000000
--- a/requirements.app.txt
+++ /dev/null
@@ -1,35 +0,0 @@
-setuptools==54.1.2
-alembic==1.0.0
--e git+https://github.com/CityOfPhiladelphia/datum.git#egg=datum
--e git+https://github.com/CityOfPhiladelphia/passyunk.git@pre-automation-stable#egg=passyunk
-# flasgger==0.5.14
-Flask==1.0.2
-Flask-Migrate==2.5.3
-Flask-Script==2.0.6
-Flask-SQLAlchemy==2.4.1
-Flask-Cors==3.0.8
-Flask-CacheControl==0.1.2
-GeoAlchemy2==0.8.2
-itsdangerous==0.24
-Jinja2==2.10.1
-Mako==1.0.3
-MarkupSafe==0.23
-normality==0.2.4
-psycopg2-binary==2.8.5
-pyproj==2.6.1.post1
-python-editor==0.5
-#PyYAML==3.11
-pyyaml>=4.2b1
-raven[flask]==5.21.0
-Shapely>=1.5.13
-SQLAlchemy==1.3.17
-Werkzeug==1.0.1
-zope.interface==5.1.0
-#petl==1.6.8
-#-e git+https://github.com/CityOfPhiladelphia/geopetl.git@4d77021c164d5b6fd8db62c30d44106a93204a72#egg=geopetl
-pathlib2==2.3.5
-
-# For passyunk
-fuzzywuzzy==0.11.1
-python-levenshtein==0.12.0
-
diff --git a/requirements.server.txt b/requirements.server.txt
deleted file mode 100644
index c0ade865..00000000
--- a/requirements.server.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-# Deployment requirements
-gunicorn==19.5.0
-psycogreen==1.0.2
-greenlet==1.1.2
-gevent==1.5.0
-honcho==0.7.1
-pytest==3.2.2
diff --git a/requirements.txt b/requirements.txt
index c4241df4..cb3890f8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,91 @@
--r requirements.app.txt
--r requirements.server.txt
+git+https://github.com/CityOfPhiladelphia/ais.git#egg=ais
+git+https://github.com/CityOfPhiladelphia/datum.git#egg=datum
+git+https://github.com/CityOfPhiladelphia/geopetl.git#egg=geopetl
+git+https://github.com/CityOfPhiladelphia/passyunk
+git+ssh://git@private-git/CityOfPhiladelphia/passyunk_automation.git
+alembic==1.8.1
+asttokens==2.2.1
+attrs==22.1.0
+backcall==0.2.0
+banal==1.0.6
+blinker==1.5
+certifi==2022.9.24
+chardet==5.0.0
+charset-normalizer==2.1.1
+click==8.1.3
+config==0.5.1
+cx-Oracle==8.3.0
+debugpy==1.6.6
+decorator==5.1.1
+dnspython==2.2.1
+exceptiongroup==1.0.4
+executing==1.2.0
+Flask==2.2.2
+Flask-CacheControl==0.3.0
+Flask-Cors==3.0.10
+Flask-Login==0.6.2
+Flask-Migrate==4.0.0
+Flask-SQLAlchemy==3.0.2
+fuzzywuzzy==0.18.0
+GeoAlchemy2==0.12.5
+geopetl==0.0.1
+gevent==22.10.2
+greenlet==2.0.1
+gunicorn==20.1.0
+guppy3==3.1.2
+idna==3.4
+importlib-metadata==6.1.0
+iniconfig==1.1.1
+ipython==8.11.0
+itsdangerous==2.1.2
+jedi==0.18.2
+Jinja2==3.1.2
+Levenshtein==0.20.8
+Mako==1.2.4
+MarkupSafe==2.1.1
+matplotlib-inline==0.1.6
+normality==2.4.0
+numpy==1.24.2
+packaging==21.3
+pandas==1.5.3
+parso==0.8.3
+pathlib2==2.3.7.post1
+petl==1.7.12
+pexpect==4.8.0
+pickleshare==0.7.5
+pluggy==1.0.0
+prompt-toolkit==3.0.38
+psutil==5.9.4
+psycogreen==1.0.2
+psycopg==3.1.8
+psycopg-binary==3.1.8
+psycopg2-binary==2.9.5
+ptyprocess==0.7.0
+pure-eval==0.2.2
+Pygments==2.14.0
+pyparsing==3.0.9
+pyproj==3.4.0
+pytest==7.2.0
+python-dateutil==2.8.1
+python-dotenv==0.21.0
+python-editor==1.0.4
+python-Levenshtein==0.20.8
+pytz==2022.7.1
+PyYAML==6.0
+rapidfuzz==2.13.2
+raven==6.10.0
+requests==2.28.1
+Shapely==1.8.5.post1
+six==1.10.0
+SQLAlchemy==1.4.44
+stack-data==0.6.2
+text-unidecode==1.3
+tomli==2.0.1
+traitlets==5.9.0
+typing_extensions==4.5.0
+urllib3==1.26.13
+wcwidth==0.2.6
+Werkzeug==2.2.2
+zipp==3.15.0
+zope.event==4.5.0
+zope.interface==5.5.2
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 2cfc573a..506389c7 100644
--- a/setup.py
+++ b/setup.py
@@ -7,6 +7,10 @@
       author='City of Philadelphia',
       author_email='maps@phila.gov',
       license='MIT',
-      packages=['ais'],
-      entry_points={'console_scripts': ['ais=ais:manager.run']},
-      zip_safe=False)
\ No newline at end of file
+      packages=['ais','ais.engine','ais.api','ais.engine.scripts'],
+      # console_script arg format is =:
+      # See application.py and setup.py for the responsible functions
+      #entry_points={'console_scripts': ['ais=ais:cli']},
+      #entry_points={'console_scripts': ['ais=ais.commands:cli']},
+      entry_points={'console_scripts': ['ais=ais.commands:cli']},
+      zip_safe=False)
diff --git a/ssh-config b/ssh-config
new file mode 100644
index 00000000..4a982b1c
--- /dev/null
+++ b/ssh-config
@@ -0,0 +1,21 @@
+# Deploy key for pulling our passyunk repo
+Host private-git
+  User Auto-CityGeo
+  Hostname github.com
+  PreferredAuthentications publickey
+  IdentityFile ~/.ssh/passyunk-private.key
+
+# James' key for quality assurance work
+Host github.com-ais_qa
+  User git
+  Hostname github.com
+  IdentityFile ~/.ssh/ais_qa
+  IdentitiesOnly true
+
+# Deploy key for the ais repo itself
+Host github.com github-ais
+  User git
+  Hostname github.com
+  IdentityFile ~/.ssh/ais
+  IdentitiesOnly true
+
diff --git a/write-secrets-to-env.py b/write-secrets-to-env.py
new file mode 100644
index 00000000..c4054807
--- /dev/null
+++ b/write-secrets-to-env.py
@@ -0,0 +1,80 @@
+import citygeo_secrets as cgs
+import os
+from os.path import expanduser
+
+cgs.set_config(
+    log_level='error',
+    keeper_dir='~')
+
+TEMP_ENV='citygeo_secrets_env_vars.bash'
+INSERT_MARKER='# Below is automatically inserted by write-secrets-to-env.py'
+
+cgs.generate_env_file('keeper', 
+    RDS_ENGINE_DB_PASS = (
+        'ais-engine (green and blue) - ais_engine',
+        'password'),
+    RDS_SUPER_ENGINE_DB_PASS = (
+        'ais-engine (green and blue) - postgres',
+        'password'),
+    LOCAL_POSTGRES_ENGINE_DB_PASS = (
+        'AIS local build postgres',
+        'password'),
+    LOCAL_ENGINE_DB_PASS = (
+        'ais_engine/on-prem',
+        'password'),
+    AWS_ACCESS_KEY_ID = (
+        'Citygeo AWS Key Pair PROD', 
+        'access_key'), 
+    AWS_SECRET_ACCESS_KEY = (
+        'Citygeo AWS Key Pair PROD', 
+        'secret_key')
+    )
+
+with open('.env', 'r') as f:
+    lines = f.readlines()
+
+# Find the index of the line that matches the insert_marker
+insert_index = -1
+for i, line in enumerate(lines):
+    if line.strip() == INSERT_MARKER:
+        insert_index = i
+        break
+
+# Truncate the file from the matched line onward
+with open('.env', 'w') as f:
+    f.writelines(lines[:insert_index+1])
+
+# Append contents of the new file
+with open(TEMP_ENV, 'r') as new_file:
+    new_contents = new_file.read()
+
+with open('.env', 'a') as f:
+    f.write('\n' + new_contents)
+
+os.remove(TEMP_ENV)
+
+##############################
+# Update ~/.aws/credentials
+
+aws_creds = cgs.get_secrets('Citygeo AWS Key Pair PROD')
+access_key_id       = aws_creds["Citygeo AWS Key Pair PROD"]['access_key']
+secret_access_key   = aws_creds["Citygeo AWS Key Pair PROD"]['secret_key']
+
+aws_creds = cgs.get_secrets('Mulesoft AWS Key Pair PROD')
+ms_access_key_id       = aws_creds["Mulesoft AWS Key Pair PROD"]['login']
+ms_secret_access_key   = aws_creds["Mulesoft AWS Key Pair PROD"]['password']
+
+home = expanduser("~")
+
+aws_credentials_path = os.path.join(home, '.aws/credentials')
+
+# Open the file in write mode ('w') to ensure it will be overwritten if it exists
+with open(aws_credentials_path, 'w') as file:
+    file.write(f"[default]\n")
+    file.write(f"aws_access_key_id = {access_key_id}\n")
+    file.write(f"aws_secret_access_key = {secret_access_key}\n")
+    file.write(f"[mulesoft]\n")
+    file.write(f"aws_access_key_id = {ms_access_key_id}\n")
+    file.write(f"aws_secret_access_key = {ms_secret_access_key}\n")
+
+print("AWS credentials file created and overwritten successfully.")