From 52adf3e489d38df394ac2bf245246ec55cdd8296 Mon Sep 17 00:00:00 2001 From: --global Date: Wed, 19 Jun 2024 14:53:57 -0400 Subject: [PATCH 1/3] add sentry --- app/config.py | 2 ++ app/data/celery.py | 15 ++++----------- app/main.py | 43 ++++++++++++++++++------------------------- requirements.in | 1 + requirements.txt | 7 +++++++ 5 files changed, 32 insertions(+), 36 deletions(-) diff --git a/app/config.py b/app/config.py index e6c536d..e5598df 100644 --- a/app/config.py +++ b/app/config.py @@ -202,6 +202,8 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: "pk.eyJ1IjoibWFwYm94IiwiYSI6ImNpejY4NXVycTA2emYycXBndHRqcmZ3N3gifQ.rJcFIG214AriISLbB6B5aw", ) # noqa + SENTRY_DSN: str | None = os.getenv("SENTRY_DSN") + class ProductionConfig(Config): """The Production Config is used for deployment of the website to the diff --git a/app/data/celery.py b/app/data/celery.py index 0a4ca85..2fb3ec3 100644 --- a/app/data/celery.py +++ b/app/data/celery.py @@ -125,20 +125,14 @@ def predict_v3_task(*args, **kwargs) -> RecordsType: @celery_app.task -def update_db_task() -> None: - from app.data.processing.core import update_db - - update_db() - - -@celery_app.task -def update_website_task() -> None: +def update_db_task(tweet_status: bool = False) -> None: from app.data.globals import website_options from app.data.processing.core import update_db - from app.twitter import tweet_current_status update_db() - if website_options.boating_season: + if tweet_status and website_options.boating_season: + from app.twitter import tweet_current_status + tweet_current_status() @@ -161,5 +155,4 @@ def send_database_exports_task() -> None: predict_v2_task: WithAppContextTask predict_v3_task: WithAppContextTask update_db_task: WithAppContextTask -update_website_task: WithAppContextTask send_database_exports_task: WithAppContextTask diff --git a/app/main.py b/app/main.py index f587c6a..fe377f6 100644 --- a/app/main.py +++ b/app/main.py @@ -87,6 +87,15 @@ def register_extensions(app: Flask): cors = CORS(resources={"/api/*": {"origins": "*"}, "/flags": {"origins": "*"}}) cors.init_app(app) + if app.config.get("SENTRY_DSN"): + import sentry_sdk + + sentry_sdk.init( + dsn=app.config["SENTRY_DSN"], + traces_sample_rate=1.0, + profiles_sample_rate=1.0, + ) + def register_blueprints(app: Flask): """Register the "blueprints." Blueprints are basically like mini web apps @@ -262,42 +271,26 @@ def _wrap(*args, **kwargs): help="If set, then run this command in Celery." " This can help save a bit of money on Heroku compute.", ) + @click.option( + "--tweet-status/--dont-tweet-status", + is_flag=True, + default=False, + help="If set, then send a tweet indicating the status of the flags.", + ) @with_appcontext @mail_on_fail - def update_db_command(async_: bool = False): + def update_db_command(async_: bool = False, tweet_status: bool = False): """Update the database with the latest live data.""" from app.data.celery import update_db_task if async_: - res = update_db_task.delay() + res = update_db_task.delay(tweet_status=tweet_status) click.echo(f"Started update database task ({res.id!r}).") else: click.echo("Updating the database...") - update_db_task.run() + update_db_task.run(tweet_status=tweet_status) click.echo("Updated the database successfully.") - @app.cli.command("update-website") - @click.option( - "--async", - "async_", - is_flag=True, - default=False, - help="If set, then run this command in Celery." - " This can help save a bit of money on Heroku compute.", - ) - @mail_on_fail - def update_website_command(async_: bool = False): - """Updates the database, then Tweets a message.""" - from app.data.celery import update_website_task - - if async_: - res = update_website_task.delay() - click.echo(f"Started update website task ({res.id!r}).") - else: - click.echo("Updating the website...") - update_website_task.run() - click.echo("Updated the website successfully.") - @app.cli.command("gen-mock-data") @dev_only def generate_mock_data(): diff --git a/requirements.in b/requirements.in index a01ee43..973e694 100644 --- a/requirements.in +++ b/requirements.in @@ -27,6 +27,7 @@ redis requests rich-click SQLAlchemy +sentry-sdk[flask] tenacity tweepy diff --git a/requirements.txt b/requirements.txt index 9d14304..cbf3c73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ blinker==1.8.2 # via # flask # flask-mail + # sentry-sdk cachelib==0.9.0 # via flask-caching celery==5.4.0 @@ -33,6 +34,7 @@ certifi==2024.2.2 # httpcore # httpx # requests + # sentry-sdk cfgv==3.4.0 # via pre-commit charset-normalizer==3.3.2 @@ -84,6 +86,7 @@ flask==3.0.3 # flask-mail # flask-postgres # flask-sqlalchemy + # sentry-sdk flask-admin==1.6.1 # via -r requirements.in flask-basicauth==0.2.0 @@ -169,6 +172,7 @@ markupsafe==2.1.5 # via # jinja2 # mako + # sentry-sdk # werkzeug # wtforms mdurl==0.1.2 @@ -288,6 +292,8 @@ ruff==0.4.5 # via -r requirements.in schemathesis==3.28.1 # via -r requirements.in +sentry-sdk==2.6.0 + # via -r requirements.in setuptools==70.0.0 # via # flask-db @@ -343,6 +349,7 @@ urllib3==2.2.1 # via # docker # requests + # sentry-sdk uv==0.2.3 # via -r requirements.in vine==5.1.0 From 9087ff052fc0f9bb666f278c55cf480bc2097049 Mon Sep 17 00:00:00 2001 From: --global Date: Wed, 19 Jun 2024 15:03:49 -0400 Subject: [PATCH 2/3] fix --- app/config.py | 1 + app/main.py | 1 + tests/test_cli.py | 13 +------------ 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/app/config.py b/app/config.py index e5598df..d75c4ac 100644 --- a/app/config.py +++ b/app/config.py @@ -203,6 +203,7 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: ) # noqa SENTRY_DSN: str | None = os.getenv("SENTRY_DSN") + SENTRY_ENVIRONMENT: str | None = os.getenv("SENTRY_ENVIRONMENT") class ProductionConfig(Config): diff --git a/app/main.py b/app/main.py index fe377f6..36f1269 100644 --- a/app/main.py +++ b/app/main.py @@ -92,6 +92,7 @@ def register_extensions(app: Flask): sentry_sdk.init( dsn=app.config["SENTRY_DSN"], + environment=app.config.get("SENTRY_ENVIRONMENT") or app.config.get["FLASK_ENV"], traces_sample_rate=1.0, profiles_sample_rate=1.0, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 153349c..4964630 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -30,17 +30,6 @@ def mail_send(): yield mocked_func -@pytest.mark.parametrize("cmd", ["update-db", "update-website"]) -def test_update_runs(cmd, app, cli_runner, mock_update_db, mock_send_tweet): - res = cli_runner.invoke(app.cli, [cmd]) - - assert res.exit_code == 0 - assert mock_update_db.call_count == 1 - - if cmd == "update-db": - assert mock_send_tweet.call_count == 0 - - def test_mail_when_error_raised(mail_send, app, cli_runner, monkeypatch, db_session): # This should not cause an email to be send: monkeypatch.setattr(core, "update_db", lambda: None) @@ -59,7 +48,7 @@ def raise_an_error(): def test_no_tweet_off_season(app, db_session, cli_runner, mock_update_db, mock_send_tweet): # Default database state (i.e. during testing) is boating_season is true. # So "update-website" should send a tweet. - res = cli_runner.invoke(app.cli, ["update-website"]) + res = cli_runner.invoke(app.cli, ["update-db", "--tweet-status"]) assert res.exit_code == 0 assert mock_update_db.call_count == 1 assert mock_send_tweet.call_count == 1 From 168c654a34a5f1a9222e3f9cfc5bfe4d477580c1 Mon Sep 17 00:00:00 2001 From: --global Date: Wed, 19 Jun 2024 15:58:49 -0400 Subject: [PATCH 3/3] update --- app/admin/views/data.py | 28 ++++++++------------- app/data/processing/predictive_models/v1.py | 18 ++++++------- app/data/processing/predictive_models/v2.py | 4 +-- app/data/processing/predictive_models/v3.py | 18 ++++++------- pytest.ini | 1 - tests/test_cli.py | 4 +-- 6 files changed, 33 insertions(+), 40 deletions(-) diff --git a/app/admin/views/data.py b/app/admin/views/data.py index 4c82a56..fdf6e2e 100644 --- a/app/admin/views/data.py +++ b/app/admin/views/data.py @@ -117,50 +117,48 @@ def download_from_db(self, sql_table_name: str): @expose("/csv/src/hobolink_source") def source_hobolink(self): - async_result = live_hobolink_data_task.s(export_name="code_for_boston_export_90d").delay() + async_result = live_hobolink_data_task.delay(export_name="code_for_boston_export_90d") return redirect( url_for("admin_downloadview.csv_wait", task_id=async_result.id, data_source="hobolink") ) @expose("/csv/src/usgs_source") def source_usgs(self): - async_result = live_usgs_data_task.s(days_ago=90).delay() + async_result = live_usgs_data_task.delay(days_ago=90) return redirect( url_for("admin_downloadview.csv_wait", task_id=async_result.id, data_source="usgs") ) @expose("/csv/src/processed_data_v1_source") def source_combine_data_v1(self): - async_result = combine_data_v1_task.s( + async_result = combine_data_v1_task.delay( export_name="code_for_boston_export_90d", days_ago=90 - ).delay() + ) return redirect( url_for("admin_downloadview.csv_wait", task_id=async_result.id, data_source="combined") ) @expose("/csv/src/processed_data_v2_source") def source_combine_data_v2(self): - async_result = combine_data_v2_task.s( + async_result = combine_data_v2_task.delay( export_name="code_for_boston_export_90d", days_ago=90 - ).delay() + ) return redirect( url_for("admin_downloadview.csv_wait", task_id=async_result.id, data_source="combined") ) @expose("/csv/src/processed_data_v3_source") def source_combine_data_v3(self): - async_result = combine_data_v3_task.s( + async_result = combine_data_v3_task.delay( export_name="code_for_boston_export_90d", days_ago=90 - ).delay() + ) return redirect( url_for("admin_downloadview.csv_wait", task_id=async_result.id, data_source="combined") ) @expose("/csv/src/prediction_v1_source") def source_prediction_v1(self): - async_result = predict_v1_task.s( - export_name="code_for_boston_export_90d", days_ago=90 - ).delay() + async_result = predict_v1_task.delay(export_name="code_for_boston_export_90d", days_ago=90) return redirect( url_for( "admin_downloadview.csv_wait", task_id=async_result.id, data_source="prediction" @@ -169,9 +167,7 @@ def source_prediction_v1(self): @expose("/csv/src/prediction_v2_source") def source_prediction_v2(self): - async_result = predict_v2_task.s( - export_name="code_for_boston_export_90d", days_ago=90 - ).delay() + async_result = predict_v2_task.delay(export_name="code_for_boston_export_90d", days_ago=90) return redirect( url_for( "admin_downloadview.csv_wait", task_id=async_result.id, data_source="prediction" @@ -180,9 +176,7 @@ def source_prediction_v2(self): @expose("/csv/src/prediction_v3_source") def source_prediction_v3(self): - async_result = predict_v3_task.s( - export_name="code_for_boston_export_90d", days_ago=90 - ).delay() + async_result = predict_v3_task.delay(export_name="code_for_boston_export_90d", days_ago=90) return redirect( url_for( "admin_downloadview.csv_wait", task_id=async_result.id, data_source="prediction" diff --git a/app/data/processing/predictive_models/v1.py b/app/data/processing/predictive_models/v1.py index 6ee2c01..efc540e 100644 --- a/app/data/processing/predictive_models/v1.py +++ b/app/data/processing/predictive_models/v1.py @@ -56,16 +56,16 @@ def process_data(df_hobolink: pd.DataFrame, df_usgs: pd.DataFrame) -> pd.DataFra df_hobolink.groupby("time") .agg( { - "pressure": np.mean, - "par": np.mean, - "rain": np.sum, - "rh": np.mean, - "dew_point": np.mean, - "wind_speed": np.mean, - "gust_speed": np.mean, - "wind_dir": np.mean, + "pressure": "mean", + "par": "mean", + "rain": "sum", + "rh": "mean", + "dew_point": "mean", + "wind_speed": "mean", + "gust_speed": "mean", + "wind_dir": "mean", # 'water_temp': np.mean, - "air_temp": np.mean, + "air_temp": "mean", } ) .reset_index() diff --git a/app/data/processing/predictive_models/v2.py b/app/data/processing/predictive_models/v2.py index 5054aeb..ab87133 100644 --- a/app/data/processing/predictive_models/v2.py +++ b/app/data/processing/predictive_models/v2.py @@ -62,14 +62,14 @@ def process_data(df_hobolink: pd.DataFrame, df_usgs: pd.DataFrame) -> pd.DataFra df_usgs.groupby("time") .agg( { - "log_stream_flow": np.mean, + "log_stream_flow": "mean", } ) .reset_index() ) df_hobolink = ( df_hobolink.groupby("time") - .agg({"rain": np.sum, "log_air_temp": np.mean, "days_since_sig_rain": np.min}) + .agg({"rain": "sum", "log_air_temp": "mean", "days_since_sig_rain": "min"}) .reset_index() ) diff --git a/app/data/processing/predictive_models/v3.py b/app/data/processing/predictive_models/v3.py index 21f1cbc..6facdc7 100644 --- a/app/data/processing/predictive_models/v3.py +++ b/app/data/processing/predictive_models/v3.py @@ -43,16 +43,16 @@ def process_data(df_hobolink: pd.DataFrame, df_usgs: pd.DataFrame) -> pd.DataFra df_hobolink.groupby("time") .agg( { - "pressure": np.mean, - "par": np.mean, - "rain": np.sum, - "rh": np.mean, - "dew_point": np.mean, - "wind_speed": np.mean, - "gust_speed": np.mean, - "wind_dir": np.mean, + "pressure": "mean", + "par": "mean", + "rain": "sum", + "rh": "mean", + "dew_point": "mean", + "wind_speed": "mean", + "gust_speed": "mean", + "wind_dir": "mean", # 'water_temp': np.mean, - "air_temp": np.mean, + "air_temp": "mean", } ) .reset_index() diff --git a/pytest.ini b/pytest.ini index 2df2815..f39fde7 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,4 @@ [pytest] -mocked-sessions = app.data.database.db.session addopts = --cov=app testpaths = tests diff --git a/tests/test_cli.py b/tests/test_cli.py index 4964630..551cbf5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -31,7 +31,7 @@ def mail_send(): def test_mail_when_error_raised(mail_send, app, cli_runner, monkeypatch, db_session): - # This should not cause an email to be send: + # This should not cause an email to be sent: monkeypatch.setattr(core, "update_db", lambda: None) cli_runner.invoke(app.cli, ["update-db"]) assert mail_send.call_count == 0 @@ -61,7 +61,7 @@ def test_no_tweet_off_season(app, db_session, cli_runner, mock_update_db, mock_s # No tweets should go out when it's not boating season. # The call count should not have gone up since the previous assert. - res = cli_runner.invoke(app.cli, ["update-website"]) + res = cli_runner.invoke(app.cli, ["update-db", "--tweet-status"]) assert res.exit_code == 0 assert mock_update_db.call_count == 2 assert mock_send_tweet.call_count == 1