diff --git a/.github/actions/setup-project/action.yml b/.github/actions/setup-project/action.yml index c095bd595..b2821a5b6 100644 --- a/.github/actions/setup-project/action.yml +++ b/.github/actions/setup-project/action.yml @@ -15,4 +15,4 @@ runs: python-version: "3.12.3" - name: Install poetry shell: bash - run: pip install --upgrade poetry + run: pip install poetry==1.8.5 diff --git a/Makefile b/Makefile index acd31f390..1ffc0a725 100644 --- a/Makefile +++ b/Makefile @@ -9,10 +9,12 @@ GIT_COMMIT ?= $(shell git rev-parse HEAD) ## DEVELOPMENT +## TODO this line should go under `make generate-version-file` +## poetry self update + .PHONY: bootstrap bootstrap: ## Set up everything to run the app make generate-version-file - poetry self update poetry self add poetry-dotenv-plugin poetry lock --no-update poetry install --sync --no-root diff --git a/app/aws/s3.py b/app/aws/s3.py index d97c421f2..c33366a2c 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -1,7 +1,9 @@ +import csv import datetime import re import time from concurrent.futures import ThreadPoolExecutor +from io import StringIO import botocore from boto3 import Session @@ -395,31 +397,25 @@ def get_job_from_s3(service_id, job_id): def extract_phones(job): - job = job.split("\r\n") - first_row = job[0] - job.pop(0) - first_row = first_row.split(",") + job_csv_data = StringIO(job) + csv_reader = csv.reader(job_csv_data) + first_row = next(csv_reader) + phone_index = 0 - for item in first_row: - # Note: may contain a BOM and look like \ufeffphone number - if item.lower() in [ - "phone number", - "\\ufeffphone number", - "\\ufeffphone number\n", - "phone number\n", - ]: + for i, item in enumerate(first_row): + if item.lower().lstrip("\ufeff") == "phone number": + phone_index = i break - phone_index = phone_index + 1 phones = {} job_row = 0 - for row in job: - row = row.split(",") + for row in csv_reader: if phone_index >= len(row): phones[job_row] = "Unavailable" current_app.logger.error( - "Corrupt csv file, missing columns or possibly a byte order mark in the file", + f"Corrupt csv file, missing columns or\ + possibly a byte order mark in the file, row looks like {row}", ) else: diff --git a/poetry.lock b/poetry.lock index e0096a7de..fadc6cde7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2829,14 +2829,18 @@ version = "1.3.0" description = "TLS (SSL) sockets, key generation, encryption, decryption, signing, verification and KDFs using the OS crypto libraries. Does not require a compiler, and relies on the OS for patching. Works on Windows, OS X and Linux/BSD." optional = false python-versions = "*" -files = [ - {file = "oscrypto-1.3.0-py2.py3-none-any.whl", hash = "sha256:2b2f1d2d42ec152ca90ccb5682f3e051fb55986e1b170ebde472b133713e7085"}, - {file = "oscrypto-1.3.0.tar.gz", hash = "sha256:6f5fef59cb5b3708321db7cca56aed8ad7e662853351e7991fcf60ec606d47a4"}, -] +files = [] +develop = false [package.dependencies] asn1crypto = ">=1.5.1" +[package.source] +type = "git" +url = "https://github.com/wbond/oscrypto.git" +reference = "1547f53" +resolved_reference = "1547f535001ba568b239b8797465536759c742a3" + [[package]] name = "packageurl-python" version = "0.16.0" @@ -4947,4 +4951,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "271d8e0f25856f45970e5a9cc3b8871a01b732226ab3ed68ea426912b5117fcf" +content-hash = "81a109693e74d2ffa3be7098e629050f25090c6a08bab57056b9a4a35283ea6f" diff --git a/pyproject.toml b/pyproject.toml index dc568d89e..d29ff84f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ marshmallow = "==3.22.0" marshmallow-sqlalchemy = "==1.0.0" newrelic = "*" notifications-python-client = "==10.0.0" -oscrypto = "==1.3.0" +oscrypto = { git = "https://github.com/wbond/oscrypto.git", rev = "1547f53" } packaging = "==24.1" poetry-dotenv-plugin = "==0.2.0" psycopg2-binary = "==2.9.9"