Merge pull request #249 from dwreeves/add-sentry

add sentry
codeforboston · Jun 19, 2024 · 02a0996 · 02a0996
2 parents b2910c9 + 168c654
commit 02a0996
Show file tree

Hide file tree

Showing 11 changed files with 68 additions and 88 deletions.
diff --git a/app/admin/views/data.py b/app/admin/views/data.py
@@ -117,50 +117,48 @@ def download_from_db(self, sql_table_name: str):
 
     @expose("/csv/src/hobolink_source")
     def source_hobolink(self):
-        async_result = live_hobolink_data_task.s(export_name="code_for_boston_export_90d").delay()
+        async_result = live_hobolink_data_task.delay(export_name="code_for_boston_export_90d")
         return redirect(
             url_for("admin_downloadview.csv_wait", task_id=async_result.id, data_source="hobolink")
         )
 
     @expose("/csv/src/usgs_source")
     def source_usgs(self):
-        async_result = live_usgs_data_task.s(days_ago=90).delay()
+        async_result = live_usgs_data_task.delay(days_ago=90)
         return redirect(
             url_for("admin_downloadview.csv_wait", task_id=async_result.id, data_source="usgs")
         )
 
     @expose("/csv/src/processed_data_v1_source")
     def source_combine_data_v1(self):
-        async_result = combine_data_v1_task.s(
+        async_result = combine_data_v1_task.delay(
             export_name="code_for_boston_export_90d", days_ago=90
-        ).delay()
+        )
         return redirect(
             url_for("admin_downloadview.csv_wait", task_id=async_result.id, data_source="combined")
         )
 
     @expose("/csv/src/processed_data_v2_source")
     def source_combine_data_v2(self):
-        async_result = combine_data_v2_task.s(
+        async_result = combine_data_v2_task.delay(
             export_name="code_for_boston_export_90d", days_ago=90
-        ).delay()
+        )
         return redirect(
             url_for("admin_downloadview.csv_wait", task_id=async_result.id, data_source="combined")
         )
 
     @expose("/csv/src/processed_data_v3_source")
     def source_combine_data_v3(self):
-        async_result = combine_data_v3_task.s(
+        async_result = combine_data_v3_task.delay(
             export_name="code_for_boston_export_90d", days_ago=90
-        ).delay()
+        )
         return redirect(
             url_for("admin_downloadview.csv_wait", task_id=async_result.id, data_source="combined")
         )
 
     @expose("/csv/src/prediction_v1_source")
     def source_prediction_v1(self):
-        async_result = predict_v1_task.s(
-            export_name="code_for_boston_export_90d", days_ago=90
-        ).delay()
+        async_result = predict_v1_task.delay(export_name="code_for_boston_export_90d", days_ago=90)
         return redirect(
             url_for(
                 "admin_downloadview.csv_wait", task_id=async_result.id, data_source="prediction"
@@ -169,9 +167,7 @@ def source_prediction_v1(self):
 
     @expose("/csv/src/prediction_v2_source")
     def source_prediction_v2(self):
-        async_result = predict_v2_task.s(
-            export_name="code_for_boston_export_90d", days_ago=90
-        ).delay()
+        async_result = predict_v2_task.delay(export_name="code_for_boston_export_90d", days_ago=90)
         return redirect(
             url_for(
                 "admin_downloadview.csv_wait", task_id=async_result.id, data_source="prediction"
@@ -180,9 +176,7 @@ def source_prediction_v2(self):
 
     @expose("/csv/src/prediction_v3_source")
     def source_prediction_v3(self):
-        async_result = predict_v3_task.s(
-            export_name="code_for_boston_export_90d", days_ago=90
-        ).delay()
+        async_result = predict_v3_task.delay(export_name="code_for_boston_export_90d", days_ago=90)
         return redirect(
             url_for(
                 "admin_downloadview.csv_wait", task_id=async_result.id, data_source="prediction"

diff --git a/app/config.py b/app/config.py
@@ -202,6 +202,9 @@ def SQLALCHEMY_DATABASE_URI(self) -> str:
         "pk.eyJ1IjoibWFwYm94IiwiYSI6ImNpejY4NXVycTA2emYycXBndHRqcmZ3N3gifQ.rJcFIG214AriISLbB6B5aw",
     )  # noqa
 
+    SENTRY_DSN: str | None = os.getenv("SENTRY_DSN")
+    SENTRY_ENVIRONMENT: str | None = os.getenv("SENTRY_ENVIRONMENT")
+
 
 class ProductionConfig(Config):
     """The Production Config is used for deployment of the website to the

diff --git a/app/data/celery.py b/app/data/celery.py
@@ -125,20 +125,14 @@ def predict_v3_task(*args, **kwargs) -> RecordsType:
 
 
 @celery_app.task
-def update_db_task() -> None:
-    from app.data.processing.core import update_db
-
-    update_db()
-
-
-@celery_app.task
-def update_website_task() -> None:
+def update_db_task(tweet_status: bool = False) -> None:
     from app.data.globals import website_options
     from app.data.processing.core import update_db
-    from app.twitter import tweet_current_status
 
     update_db()
-    if website_options.boating_season:
+    if tweet_status and website_options.boating_season:
+        from app.twitter import tweet_current_status
+
         tweet_current_status()
 
 
@@ -161,5 +155,4 @@ def send_database_exports_task() -> None:
 predict_v2_task: WithAppContextTask
 predict_v3_task: WithAppContextTask
 update_db_task: WithAppContextTask
-update_website_task: WithAppContextTask
 send_database_exports_task: WithAppContextTask
diff --git a/app/data/processing/predictive_models/v1.py b/app/data/processing/predictive_models/v1.py
@@ -56,16 +56,16 @@ def process_data(df_hobolink: pd.DataFrame, df_usgs: pd.DataFrame) -> pd.DataFra
         df_hobolink.groupby("time")
         .agg(
             {
-                "pressure": np.mean,
-                "par": np.mean,
-                "rain": np.sum,
-                "rh": np.mean,
-                "dew_point": np.mean,
-                "wind_speed": np.mean,
-                "gust_speed": np.mean,
-                "wind_dir": np.mean,
+                "pressure": "mean",
+                "par": "mean",
+                "rain": "sum",
+                "rh": "mean",
+                "dew_point": "mean",
+                "wind_speed": "mean",
+                "gust_speed": "mean",
+                "wind_dir": "mean",
                 # 'water_temp': np.mean,
-                "air_temp": np.mean,
+                "air_temp": "mean",
             }
         )
         .reset_index()

diff --git a/app/data/processing/predictive_models/v2.py b/app/data/processing/predictive_models/v2.py
@@ -62,14 +62,14 @@ def process_data(df_hobolink: pd.DataFrame, df_usgs: pd.DataFrame) -> pd.DataFra
         df_usgs.groupby("time")
         .agg(
             {
-                "log_stream_flow": np.mean,
+                "log_stream_flow": "mean",
             }
         )
         .reset_index()
     )
     df_hobolink = (
         df_hobolink.groupby("time")
-        .agg({"rain": np.sum, "log_air_temp": np.mean, "days_since_sig_rain": np.min})
+        .agg({"rain": "sum", "log_air_temp": "mean", "days_since_sig_rain": "min"})
         .reset_index()
     )
 

diff --git a/app/data/processing/predictive_models/v3.py b/app/data/processing/predictive_models/v3.py
@@ -43,16 +43,16 @@ def process_data(df_hobolink: pd.DataFrame, df_usgs: pd.DataFrame) -> pd.DataFra
         df_hobolink.groupby("time")
         .agg(
             {
-                "pressure": np.mean,
-                "par": np.mean,
-                "rain": np.sum,
-                "rh": np.mean,
-                "dew_point": np.mean,
-                "wind_speed": np.mean,
-                "gust_speed": np.mean,
-                "wind_dir": np.mean,
+                "pressure": "mean",
+                "par": "mean",
+                "rain": "sum",
+                "rh": "mean",
+                "dew_point": "mean",
+                "wind_speed": "mean",
+                "gust_speed": "mean",
+                "wind_dir": "mean",
                 # 'water_temp': np.mean,
-                "air_temp": np.mean,
+                "air_temp": "mean",
             }
         )
         .reset_index()

diff --git a/app/main.py b/app/main.py
@@ -87,6 +87,16 @@ def register_extensions(app: Flask):
     cors = CORS(resources={"/api/*": {"origins": "*"}, "/flags": {"origins": "*"}})
     cors.init_app(app)
 
+    if app.config.get("SENTRY_DSN"):
+        import sentry_sdk
+
+        sentry_sdk.init(
+            dsn=app.config["SENTRY_DSN"],
+            environment=app.config.get("SENTRY_ENVIRONMENT") or app.config.get["FLASK_ENV"],
+            traces_sample_rate=1.0,
+            profiles_sample_rate=1.0,
+        )
+
 
 def register_blueprints(app: Flask):
     """Register the "blueprints." Blueprints are basically like mini web apps
@@ -262,42 +272,26 @@ def _wrap(*args, **kwargs):
         help="If set, then run this command in Celery."
         " This can help save a bit of money on Heroku compute.",
     )
+    @click.option(
+        "--tweet-status/--dont-tweet-status",
+        is_flag=True,
+        default=False,
+        help="If set, then send a tweet indicating the status of the flags.",
+    )
     @with_appcontext
     @mail_on_fail
-    def update_db_command(async_: bool = False):
+    def update_db_command(async_: bool = False, tweet_status: bool = False):
         """Update the database with the latest live data."""
         from app.data.celery import update_db_task
 
         if async_:
-            res = update_db_task.delay()
+            res = update_db_task.delay(tweet_status=tweet_status)
             click.echo(f"Started update database task ({res.id!r}).")
         else:
             click.echo("Updating the database...")
-            update_db_task.run()
+            update_db_task.run(tweet_status=tweet_status)
             click.echo("Updated the database successfully.")
 
-    @app.cli.command("update-website")
-    @click.option(
-        "--async",
-        "async_",
-        is_flag=True,
-        default=False,
-        help="If set, then run this command in Celery."
-        " This can help save a bit of money on Heroku compute.",
-    )
-    @mail_on_fail
-    def update_website_command(async_: bool = False):
-        """Updates the database, then Tweets a message."""
-        from app.data.celery import update_website_task
-
-        if async_:
-            res = update_website_task.delay()
-            click.echo(f"Started update website task ({res.id!r}).")
-        else:
-            click.echo("Updating the website...")
-            update_website_task.run()
-            click.echo("Updated the website successfully.")
-
     @app.cli.command("gen-mock-data")
     @dev_only
     def generate_mock_data():

diff --git a/pytest.ini b/pytest.ini
@@ -1,5 +1,4 @@
 [pytest]
-mocked-sessions = app.data.database.db.session
 addopts = --cov=app
 testpaths =
     tests

diff --git a/requirements.in b/requirements.in
@@ -27,6 +27,7 @@ redis
 requests
 rich-click
 SQLAlchemy
+sentry-sdk[flask]
 tenacity
 tweepy
 

diff --git a/requirements.txt b/requirements.txt
@@ -21,6 +21,7 @@ blinker==1.8.2
     # via
     #   flask
     #   flask-mail
+    #   sentry-sdk
 cachelib==0.9.0
     # via flask-caching
 celery==5.4.0
@@ -33,6 +34,7 @@ certifi==2024.2.2
     #   httpcore
     #   httpx
     #   requests
+    #   sentry-sdk
 cfgv==3.4.0
     # via pre-commit
 charset-normalizer==3.3.2
@@ -84,6 +86,7 @@ flask==3.0.3
     #   flask-mail
     #   flask-postgres
     #   flask-sqlalchemy
+    #   sentry-sdk
 flask-admin==1.6.1
     # via -r requirements.in
 flask-basicauth==0.2.0
@@ -169,6 +172,7 @@ markupsafe==2.1.5
     # via
     #   jinja2
     #   mako
+    #   sentry-sdk
     #   werkzeug
     #   wtforms
 mdurl==0.1.2
@@ -288,6 +292,8 @@ ruff==0.4.5
     # via -r requirements.in
 schemathesis==3.28.1
     # via -r requirements.in
+sentry-sdk==2.6.0
+    # via -r requirements.in
 setuptools==70.0.0
     # via
     #   flask-db
@@ -343,6 +349,7 @@ urllib3==2.2.1
     # via
     #   docker
     #   requests
+    #   sentry-sdk
 uv==0.2.3
     # via -r requirements.in
 vine==5.1.0

diff --git a/tests/test_cli.py b/tests/test_cli.py
@@ -30,19 +30,8 @@ def mail_send():
         yield mocked_func
 
 
-@pytest.mark.parametrize("cmd", ["update-db", "update-website"])
-def test_update_runs(cmd, app, cli_runner, mock_update_db, mock_send_tweet):
-    res = cli_runner.invoke(app.cli, [cmd])
-
-    assert res.exit_code == 0
-    assert mock_update_db.call_count == 1
-
-    if cmd == "update-db":
-        assert mock_send_tweet.call_count == 0
-
-
 def test_mail_when_error_raised(mail_send, app, cli_runner, monkeypatch, db_session):
-    # This should not cause an email to be send:
+    # This should not cause an email to be sent:
     monkeypatch.setattr(core, "update_db", lambda: None)
     cli_runner.invoke(app.cli, ["update-db"])
     assert mail_send.call_count == 0
@@ -59,7 +48,7 @@ def raise_an_error():
 def test_no_tweet_off_season(app, db_session, cli_runner, mock_update_db, mock_send_tweet):
     # Default database state (i.e. during testing) is boating_season is true.
     # So "update-website" should send a tweet.
-    res = cli_runner.invoke(app.cli, ["update-website"])
+    res = cli_runner.invoke(app.cli, ["update-db", "--tweet-status"])
     assert res.exit_code == 0
     assert mock_update_db.call_count == 1
     assert mock_send_tweet.call_count == 1
@@ -72,7 +61,7 @@ def test_no_tweet_off_season(app, db_session, cli_runner, mock_update_db, mock_s
 
     # No tweets should go out when it's not boating season.
     # The call count should not have gone up since the previous assert.
-    res = cli_runner.invoke(app.cli, ["update-website"])
+    res = cli_runner.invoke(app.cli, ["update-db", "--tweet-status"])
     assert res.exit_code == 0
     assert mock_update_db.call_count == 2
     assert mock_send_tweet.call_count == 1