Merge pull request #117 from codeforboston/dev_postgres

Merge dev_postgres to master

docs/docs/admin.md

@@ -3,7 +3,7 @@
 ## Admin Panel
 
 ???+ Note
-    This section discusses how to use the admin panel for the website. For how to set up the admin page username and password during deployment, see the [Heroku deployment](cloud/heroku_deployment) documentation.
+    This section discusses how to use the admin panel for the website. For how to set up the admin page username and password during deployment, see the [Heroku deployment](cloud/heroku_deployment.md) documentation.
 
 The admin panel is used to manually override the model outputs during events and advisories that would adversely effect the river quality.
 

docs/docs/development_resources/shell.md

@@ -78,8 +78,6 @@ hobolink_data = get_live_hobolink_data()
 hobolink_data.to_csv('path/where/to/save/my-CSV-file.csv')
 ```
 
-Downloading the data may be useful if you want to see 
-
 ## Example 2: Preview Tweet
 
 Let's say you want to preview a Tweet that would be sent out without actually sending it. The `compose_tweet()` function returns a string of this message:

docs/docs/setup.md

@@ -19,7 +19,7 @@ Install all of the following programs onto your computer:
 **Recommended:**
 
 - A good text editor or IDE, such as [Atom.io](https://atom.io/) (which is lightweight and beginner friendly) or [PyCharm](https://www.jetbrains.com/pycharm/) (which is powerful but bulky and geared toward advanced users).
-- [Heroku CLI](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) _(required for remote deployment to Heroku.)_
+- [Heroku CLI](https://devcenter.heroku.com/articles/heroku-cli) _(required for remote deployment to Heroku.)_
 
 **Other:**
 
@@ -160,6 +160,9 @@ After you get everything set up, you should run the website at least once. The p
      * Restarting with stat
     ```
 
+???+ error
+    If you get an error that says something like "`Microsoft Visual C++ 14.0 or greater is required.`," you need to follow the link provided by the error message, download and install it, then reboot your computer.
+
 4. Point your browser of choice to the URL shown in the terminal output. If everything worked out, the website should be running on your local computer!
 
 ![](img/successful_run.png)

flagging_site/admin.py

@@ -1,17 +1,19 @@
-import os
-from inspect import cleandoc
+import io
+import datetime
+
 from flask import Flask
 from flask import redirect
 from flask import request
 from flask import Response
+from flask import send_file
 from flask_admin import Admin
 from flask_admin import BaseView
 from flask_admin import expose
 from flask_admin.contrib import sqla
-from werkzeug.exceptions import HTTPException
 
 from flask_basicauth import BasicAuth
 from werkzeug.exceptions import HTTPException
+from sqlalchemy.exc import ProgrammingError
 
 from .data import db
 
@@ -41,18 +43,85 @@ def init_admin(app: Flask):
     basic_auth.init_app(app)
     admin.init_app(app)
 
+    @app.before_request
+    def auth():
+        """Authorize all paths that start with /admin/."""
+        if request.path.startswith('/admin/'):
+            validate_credentials()
+
     with app.app_context():
         # Register /admin sub-views
-        from .data.cyano_overrides import CyanoOverridesModelView
-        admin.add_view(CyanoOverridesModelView(db.session))
-        admin.add_view(LogoutView(name="Logout"))
+        from .data.cyano_overrides import ManualOverridesModelView
+        admin.add_view(ManualOverridesModelView(db.session))
+
+        # Database functions
+        admin.add_view(DatabaseView(
+            name='Update Database', endpoint='update-db', category='Database'
+        ))
+
+        # Download data
+        admin.add_view(DownloadHobolinkView(
+            name='HOBOlink', endpoint='data/hobolink', category='Download Data'
+        ))
+        admin.add_view(DownloadUsgsView(
+            name='USGS', endpoint='data/usgs', category='Download Data'
+        ))
+        admin.add_view(DownloadProcessedDataView(
+            name='Processed Data', endpoint='data/processed-data', category='Download Data'
+        ))
+        admin.add_view(DownloadModelOutputsView(
+            name='Model Outputs', endpoint='data/model-outputs', category='Download Data'
+        ))
+        admin.add_view(DownloadBoathousesView(
+            name='Boathouses', endpoint='data/boathouses', category='Download Data'
+        ))
+
+        admin.add_view(LogoutView(name='Logout'))
+
+
+def validate_credentials() -> bool:
+    """
+    Protect admin pages with basic_auth.
+    If logged out and current page is /admin/, then ask for credentials.
+    Otherwise, raises HTTP 401 error and redirects user to /admin/ on the
+    frontend (redirecting with HTTP redirect causes user to always be
+    redirected to /admin/ even after logging in).
+
+    We redirect to /admin/ because our logout method only works if the path to
+    /logout is the same as the path to where we put in our credentials. So if
+    we put in credentials at /admin/cyanooverride, then we would need to logout
+    at /admin/cyanooverride/logout, which would be difficult to arrange. Instead,
+    we always redirect to /admin/ to put in credentials, and then logout at
+    /admin/logout.
+    """
+    if not basic_auth.authenticate():
+        if request.path.startswith('/admin/'):
+            raise AuthException('Not authenticated. Refresh the page.')
+        else:
+            raise HTTPException(
+                'Attempted to visit admin page but not authenticated.',
+                Response(
+                    '''
+                    Not authenticated. Navigate to /admin/ to login.
+                    <script>window.location = "/admin/";</script>
+                    ''',
+                    status=401  # 'Forbidden' status
+                )
+            )
+    return True
+
 
-    # Add an endpoint to the app that lets the database be updated manually.
-    app.add_url_rule('/admin/update-db', 'admin.update_db', update_database_manually)
+class AdminBaseView(BaseView):
+    def is_accessible(self):
+        return validate_credentials()
+
+    def inaccessible_callback(self, name, **kwargs):
+        """Ask for credentials when access fails"""
+        return redirect(basic_auth.challenge())
 
 
 # Adapted from https://computableverse.com/blog/flask-admin-using-basicauth
-class AdminModelView(sqla.ModelView):
+class AdminModelView(sqla.ModelView, AdminBaseView):
     """
     Extension of SQLAlchemy ModelView that requires BasicAuth authentication,
     and shows all columns in the form (including primary keys).
@@ -62,47 +131,10 @@ def __init__(self, model, *args, **kwargs):
         # Show all columns in form
         self.column_list = [c.key for c in model.__table__.columns]
         self.form_columns = self.column_list
-
         super().__init__(model, *args, **kwargs)
 
-    def is_accessible(self):
-        """
-        Protect admin pages with basic_auth.
-        If logged out and current page is /admin/, then ask for credentials.
-        Otherwise, raises HTTP 401 error and redirects user to /admin/ on the
-        frontend (redirecting with HTTP redirect causes user to always be
-        redirected to /admin/ even after logging in).
-
-        We redirect to /admin/ because our logout method only works if the path to
-        /logout is the same as the path to where we put in our credentials. So if
-        we put in credentials at /admin/cyanooverride, then we would need to logout
-        at /admin/cyanooverride/logout, which would be difficult to arrange. Instead,
-        we always redirect to /admin/ to put in credentials, and then logout at
-        /admin/logout.
-        """
-        if not basic_auth.authenticate():
-            if '/admin/' == request.path:
-                raise AuthException('Not authenticated. Refresh the page.')
-            else:
-                raise HTTPException(
-                    'Attempted to visit admin page but not authenticated.',
-                    Response(
-                        '''
-                        Not authenticated. Navigate to /admin/ to login.
-                        <script>window.location = "/admin/";</script>
-                        ''',
-                        status=401  # 'Forbidden' status
-                    )
-                )
-        else:
-            return True
-
-    def inaccessible_callback(self, name, **kwargs):
-        """Ask for credentials when access fails"""
-        return redirect(basic_auth.challenge())
 
-
-class LogoutView(BaseView):
+class LogoutView(AdminBaseView):
     @expose('/')
     def index(self):
         """
@@ -122,28 +154,92 @@ def index(self):
         )
 
 
-def update_database_manually():
-    """When this function is called, the database updates. This function is
-    designed to be available in the app during runtime, and is protected by
-    BasicAuth so that only administrators can run it.
+class DatabaseView(AdminBaseView):
+    @expose('/')
+    def update_db(self):
+        """When this function is called, the database updates. This function is
+        designed to be available in the app during runtime, and is protected by
+        BasicAuth so that only administrators can run it.
+        """
+        # If auth passed, then update database.
+        from .data.database import update_database
+        update_database()
+
+        # Notify the user that the update was successful, then redirect:
+        return '''<!DOCTYPE html>
+            <html>
+                <body>
+                    <script>
+                        setTimeout(function(){
+                            window.location.href = '/admin/';
+                        }, 3000);
+                    </script>
+                    <p>Databases updated. Redirecting in 3 seconds...</p>
+                </body>
+            </html>
+        '''
+
+
+def create_download_data_view(sql_table_name: str) -> type:
+    """This is a factory function that makes the data downloadable in the admin
+    panel. It returns a type that can be initialized later as an admin view.
+
+    Args:
+        sql_table_name: (str) A valid name for a PostgreSQL table in the
+                        database. It will cause an error if the table is
+                        invalid.
+
+    Returns:
+        A class that can be initialized as an admin view.
+
     """
-    if not basic_auth.authenticate():
-        raise AuthException('Not authenticated. Refresh the page.')
-
-    # If auth passed, then update database.
-    from .data.database import update_database
-    update_database()
-
-    # Notify the user that the update was successful, then redirect:
-    return '''<!DOCTYPE html>
-        <html>
-            <body>
-                <script>
-                    setTimeout(function(){
-                        window.location.href = '/';
-                    }, 5000);
-                </script>
-                <p>Databases updated. Redirecting in 5 seconds...</p>
-            </body>
-        </html>
-    '''
+    class _DownloadView(AdminBaseView):
+        @expose('/')
+        def download(self):
+            from .data.database import execute_sql
+
+            # WARNING:
+            # Be careful when parameterizing queries like how we do it below.
+            # The reason it's OK in this case is because users don't touch it.
+            # However it is dangerous to do this in some other contexts.
+            # We are doing it like this to avoid needing to utilize sessions.
+            query = f'''SELECT * FROM {sql_table_name}'''
+            try:
+                df = execute_sql(query)
+            except ProgrammingError:
+                raise HTTPException(
+                    'Invalid SQL.',
+                    Response(
+                        f'<b>Invalid SQL query:</b> <tt>{query}</tt>',
+                        status=500
+                    )
+                )
+
+            strio = io.StringIO()
+            df.to_csv(strio)
+
+            # Convert to bytes
+            bytesio = io.BytesIO()
+            bytesio.write(strio.getvalue().encode('utf-8'))
+            # seeking was necessary. Python 3.5.2, Flask 0.12.2
+            bytesio.seek(0)
+            strio.close()
+
+            return send_file(
+                bytesio,
+                as_attachment=True,
+                attachment_filename=f'{sql_table_name}.csv',
+                mimetype='text/csv'
+            )
+
+    pascal_case_tbl = sql_table_name.title().replace('_', '')
+    _DownloadView.__name__ = f'Download{pascal_case_tbl}View'
+    return _DownloadView
+
+
+# All of these
+DownloadHobolinkView = create_download_data_view('hobolink')
+DownloadUsgsView = create_download_data_view('usgs')
+DownloadProcessedDataView = create_download_data_view('processed_data')
+DownloadModelOutputsView = create_download_data_view('model_outputs')
+DownloadBoathousesView = create_download_data_view('boathouses')

flagging_site/app.py

@@ -6,12 +6,14 @@
 import time
 import json
 import decimal
+
 import datetime
 from typing import Optional
 from typing import Dict
 from typing import Union
 
 from flask import Flask
+from flask import current_app
 from flask.json import JSONEncoder
 
 import py7zr
@@ -115,20 +117,28 @@ def init_db_command():
     def update_db_command():
         """Update the database with the latest live data."""
         from .data.database import update_database
-        updated = update_database()
-        click.echo('Updated the database.')
-        if not updated:
-            click.echo('Note: while updating database, the predictive model '
-                       'did not run.')
+        try:
+            update_database()
+            click.echo('Updated the database.')
+            updated = True
+        except Exception as e:
+            click.echo("Note: while updating database, something didn't "
+                       f'work: {e}')
+            updated = False
         return updated
 
     @app.cli.command('update-website')
     @click.pass_context
     def update_website_command(ctx):
         """Updates the database, then Tweets a message."""
         updated = ctx.invoke(update_db_command)
-        # If the model updated, send tweet. Otherwise do nothing.
-        if updated:
+        # If the model updated and it's boating season, send a tweet.
+        # Otherwise, do nothing.
+        if (
+                updated
+                and current_app.config['BOATING_SEASON']
+                and current_app.config['SEND_TWEETS']
+        ):
             from .twitter import tweet_current_status
             msg = tweet_current_status()
             click.echo(f'Sent out tweet: {msg!r}')
@@ -193,9 +203,10 @@ def init_swagger(app: Flask):
                 "API for the Charles River Watershed Association's predictive "
                 'models, and the data used for those models.',
             'contact': {
-                'responsibleOrganization': 'Charles River Watershed Association',
-                'responsibleDeveloper': 'Code for Boston',
+                'x-responsibleOrganization': 'Charles River Watershed Association',
+                'x-responsibleDeveloper': 'Code for Boston',
             },
+            'version': '1.0',
         }
     }
     app.config['SWAGGER'] = {