diff --git a/cumulus_etl/cli_utils.py b/cumulus_etl/cli_utils.py index 1ba5bea..4151361 100644 --- a/cumulus_etl/cli_utils.py +++ b/cumulus_etl/cli_utils.py @@ -62,7 +62,7 @@ def make_export_dir(export_to: str | None = None) -> common.Directory: """Makes a temporary directory to drop exported ndjson files into""" # Handle the easy case -- just a random temp dir if not export_to: - return tempfile.TemporaryDirectory() # pylint: disable=consider-using-with + return tempfile.TemporaryDirectory() # OK the user has a specific spot in mind. Let's do some quality checks. It must be local and empty. diff --git a/cumulus_etl/deid/scrubber.py b/cumulus_etl/deid/scrubber.py index 6a75b79..c13037f 100644 --- a/cumulus_etl/deid/scrubber.py +++ b/cumulus_etl/deid/scrubber.py @@ -44,7 +44,7 @@ async def scrub_bulk_data(input_dir: str) -> tempfile.TemporaryDirectory: :returns: a temporary directory holding the de-identified results, in FHIR ndjson format """ - tmpdir = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with + tmpdir = tempfile.TemporaryDirectory() await mstool.run_mstool(input_dir, tmpdir.name) return tmpdir diff --git a/cumulus_etl/etl/studies/covid_symptom/covid_ctakes.py b/cumulus_etl/etl/studies/covid_symptom/covid_ctakes.py index 7fe4dd1..da5e110 100644 --- a/cumulus_etl/etl/studies/covid_symptom/covid_ctakes.py +++ b/cumulus_etl/etl/studies/covid_symptom/covid_ctakes.py @@ -63,7 +63,7 @@ async def covid_symptoms_extract( ctakes_json = await nlp.ctakes_extract( cache, ctakes_namespace, clinical_note, client=ctakes_http_client ) - except Exception as exc: # pylint: disable=broad-except + except Exception as exc: logging.warning( "Could not extract symptoms for docref %s (%s): %s", docref_id, type(exc).__name__, exc ) @@ -96,7 +96,7 @@ def is_covid_match(m: ctakesclient.typesystem.MatchText): model=polarity_model, client=cnlp_http_client, ) - except Exception as exc: # pylint: disable=broad-except + except Exception as exc: logging.warning( "Could not check polarity for docref %s (%s): %s", docref_id, type(exc).__name__, exc ) diff --git a/cumulus_etl/etl/tasks/basic_tasks.py b/cumulus_etl/etl/tasks/basic_tasks.py index 009b502..2dacc97 100644 --- a/cumulus_etl/etl/tasks/basic_tasks.py +++ b/cumulus_etl/etl/tasks/basic_tasks.py @@ -149,7 +149,7 @@ async def fetch_medication(self, resource: dict) -> dict | None: try: medication = await fhir.download_reference(self.task_config.client, reference) - except Exception as exc: # pylint: disable=broad-except + except Exception as exc: logging.warning("Could not download Medication reference: %s", exc) self.summaries[1].had_errors = True diff --git a/cumulus_etl/etl/tasks/nlp_task.py b/cumulus_etl/etl/tasks/nlp_task.py index ae0fdb0..0150e33 100644 --- a/cumulus_etl/etl/tasks/nlp_task.py +++ b/cumulus_etl/etl/tasks/nlp_task.py @@ -94,7 +94,7 @@ async def read_notes( warned_connection_error = True self.add_error(orig_docref) continue - except Exception as exc: # pylint: disable=broad-except + except Exception as exc: logging.warning("Error getting text for docref %s: %s", docref["id"], exc) self.add_error(orig_docref) continue diff --git a/cumulus_etl/formats/base.py b/cumulus_etl/formats/base.py index 20ed511..8f56dd4 100644 --- a/cumulus_etl/formats/base.py +++ b/cumulus_etl/formats/base.py @@ -62,7 +62,7 @@ def write_records(self, batch: Batch) -> bool: try: self._write_one_batch(batch) return True - except Exception: # pylint: disable=broad-except + except Exception: logging.exception("Could not process data records") return False diff --git a/cumulus_etl/formats/deltalake.py b/cumulus_etl/formats/deltalake.py index 459dfbf..1e8fc60 100644 --- a/cumulus_etl/formats/deltalake.py +++ b/cumulus_etl/formats/deltalake.py @@ -139,7 +139,7 @@ def finalize(self) -> None: table = delta.DeltaTable.forPath(self.spark, full_path) except AnalysisException: return # if the table doesn't exist because we didn't write anything, that's fine - just bail - except Exception: # pylint: disable=broad-except + except Exception: logging.exception("Could not finalize Delta Lake table %s", self.dbname) return @@ -147,7 +147,7 @@ def finalize(self) -> None: table.optimize().executeCompaction() # pool small files for better query performance table.generate("symlink_format_manifest") table.vacuum() # Clean up unused data files older than retention policy (default 7 days) - except Exception: # pylint: disable=broad-except + except Exception: logging.exception("Could not finalize Delta Lake table %s", self.dbname) def _table_path(self, dbname: str) -> str: diff --git a/cumulus_etl/loaders/fhir/ndjson_loader.py b/cumulus_etl/loaders/fhir/ndjson_loader.py index f8767ea..d3be266 100644 --- a/cumulus_etl/loaders/fhir/ndjson_loader.py +++ b/cumulus_etl/loaders/fhir/ndjson_loader.py @@ -67,7 +67,7 @@ async def load_all(self, resources: list[str]) -> common.Directory: # This uses more disk space temporarily (copied files will get deleted once the MS tool is done and this # TemporaryDirectory gets discarded), but that seems reasonable. print("Copying ndjson input files…") - tmpdir = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with + tmpdir = tempfile.TemporaryDirectory() filenames = common.ls_resources(input_root, set(resources), warn_if_empty=True) for filename in filenames: input_root.get(filename, f"{tmpdir.name}/") diff --git a/cumulus_etl/loaders/i2b2/oracle/query.py b/cumulus_etl/loaders/i2b2/oracle/query.py index 952ed46..0394b06 100644 --- a/cumulus_etl/loaders/i2b2/oracle/query.py +++ b/cumulus_etl/loaders/i2b2/oracle/query.py @@ -120,11 +120,11 @@ def where(expression=None) -> str: return "\n WHERE " + expression if expression else "" -def AND(expression: str) -> str: # pylint: disable=invalid-name +def AND(expression: str) -> str: return f"\n AND ({expression})" -def OR(expression: str) -> str: # pylint: disable=invalid-name +def OR(expression: str) -> str: return f"\n OR ({expression})" diff --git a/cumulus_etl/nlp/extract.py b/cumulus_etl/nlp/extract.py index e23b9fb..fdc0f61 100644 --- a/cumulus_etl/nlp/extract.py +++ b/cumulus_etl/nlp/extract.py @@ -29,7 +29,7 @@ async def ctakes_extract( try: cached_response = common.read_json(full_path) result = ctakesclient.typesystem.CtakesJSON(source=cached_response) - except Exception: # pylint: disable=broad-except + except Exception: result = await ctakesclient.client.extract(sentence, client=client) cache.makedirs(os.path.dirname(full_path)) common.write_json(full_path, result.as_json()) @@ -58,7 +58,7 @@ async def list_polarity( try: result = [ctakesclient.typesystem.Polarity(x) for x in common.read_json(full_path)] - except Exception: # pylint: disable=broad-except + except Exception: result = await ctakesclient.transformer.list_polarity( sentence, spans, client=client, model=model ) diff --git a/cumulus_etl/upload_notes/selector.py b/cumulus_etl/upload_notes/selector.py index 2d3c6ca..8dee196 100644 --- a/cumulus_etl/upload_notes/selector.py +++ b/cumulus_etl/upload_notes/selector.py @@ -43,7 +43,7 @@ def _create_docref_filter( else: # Just accept everything (we still want to read them though, to copy them to a possible export folder). # So this lambda just returns an iterator over its input. - return lambda x: iter(x) # pylint: disable=unnecessary-lambda + return lambda x: iter(x) def _filter_real_docrefs(docrefs_csv: str, docrefs: Iterable[dict]) -> Iterator[dict]: diff --git a/tests/covid_symptom/test_covid_results.py b/tests/covid_symptom/test_covid_results.py index 4cb488f..c2bfe8c 100644 --- a/tests/covid_symptom/test_covid_results.py +++ b/tests/covid_symptom/test_covid_results.py @@ -92,10 +92,10 @@ async def test_ed_note_filtering_for_nlp(self, codings, expected): async def test_non_ed_visit_is_skipped_for_covid_symptoms(self): """Verify we ignore non ED visits for the covid symptoms NLP""" docref0 = i2b2_mock_data.documentreference() - docref0["type"]["coding"][0]["code"] = "NOTE:nope" # pylint: disable=unsubscriptable-object + docref0["type"]["coding"][0]["code"] = "NOTE:nope" self.make_json("DocumentReference", "skipped", **docref0) docref1 = i2b2_mock_data.documentreference() - docref1["type"]["coding"][0]["code"] = "NOTE:149798455" # pylint: disable=unsubscriptable-object + docref1["type"]["coding"][0]["code"] = "NOTE:149798455" self.make_json("DocumentReference", "present", **docref1) await covid_symptom.CovidSymptomNlpResultsTask(self.job_config, self.scrubber).run() diff --git a/tests/ctakesmock.py b/tests/ctakesmock.py index 39bb661..381dba5 100644 --- a/tests/ctakesmock.py +++ b/tests/ctakesmock.py @@ -34,7 +34,7 @@ def setUp(self): CtakesMixin.ctakes_port += 1 os.environ["URL_CTAKES_REST"] = f"http://localhost:{CtakesMixin.ctakes_port}/" - self.ctakes_overrides = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with + self.ctakes_overrides = tempfile.TemporaryDirectory() self._run_fake_ctakes_server(f"{self.ctakes_overrides.name}/symptoms.bsv") cnlp_patcher = mock.patch( @@ -131,7 +131,7 @@ class FakeCTakesHandler(http.server.BaseHTTPRequestHandler): # We don't want that behavior, because if our mock code is misbehaving and not detecting an overrides file change, # if we time out a request on our end, it will look like a successful file change detection and mask a testing bug. - def do_POST(self): # pylint: disable=invalid-name + def do_POST(self): """Serve a POST request.""" self.server.was_called.value = 1 # signal to test framework that we were actually called diff --git a/tests/formats/test_deltalake.py b/tests/formats/test_deltalake.py index aca0e8f..e1cfb89 100644 --- a/tests/formats/test_deltalake.py +++ b/tests/formats/test_deltalake.py @@ -27,7 +27,7 @@ class TestDeltaLake(utils.AsyncTestCase): @classmethod def setUpClass(cls): super().setUpClass() - output_tempdir = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with + output_tempdir = tempfile.TemporaryDirectory() cls.output_tempdir = output_tempdir cls.output_dir = output_tempdir.name cls.root = store.Root(output_tempdir.name) diff --git a/tests/loaders/i2b2/test_i2b2_oracle_extract.py b/tests/loaders/i2b2/test_i2b2_oracle_extract.py index 0f3d349..75f2d34 100644 --- a/tests/loaders/i2b2/test_i2b2_oracle_extract.py +++ b/tests/loaders/i2b2/test_i2b2_oracle_extract.py @@ -15,7 +15,7 @@ class TestOracleExtraction(AsyncTestCase): def setUp(self) -> None: super().setUp() - self.maxDiff = None # pylint: disable=invalid-name + self.maxDiff = None # Mock all the sql connection/cursor/execution stuff connect_patcher = mock.patch("cumulus_etl.loaders.i2b2.oracle.extract.connect") diff --git a/tests/loaders/i2b2/test_i2b2_oracle_query.py b/tests/loaders/i2b2/test_i2b2_oracle_query.py index e0df9d4..edb3bf7 100644 --- a/tests/loaders/i2b2/test_i2b2_oracle_query.py +++ b/tests/loaders/i2b2/test_i2b2_oracle_query.py @@ -50,7 +50,7 @@ class TestOracleQueries(utils.AsyncTestCase): def setUp(self) -> None: super().setUp() - self.maxDiff = None # pylint: disable=invalid-name + self.maxDiff = None def test_list_patient(self): common.print_header("# patient") diff --git a/tests/loaders/i2b2/test_i2b2_transform.py b/tests/loaders/i2b2/test_i2b2_transform.py index 87eac13..d6065ae 100644 --- a/tests/loaders/i2b2/test_i2b2_transform.py +++ b/tests/loaders/i2b2/test_i2b2_transform.py @@ -18,7 +18,6 @@ def test_to_fhir_patient(self): self.assertEqual(str(12345), subject["id"]) self.assertEqual("2005-06-07", subject["birthDate"]) self.assertEqual("female", subject["gender"]) - # pylint: disable-next=unsubscriptable-object self.assertEqual("02115", subject["address"][0]["postalCode"]) @ddt.data( diff --git a/tests/loaders/ndjson/test_bulk_export.py b/tests/loaders/ndjson/test_bulk_export.py index 50f91fc..6f13fad 100644 --- a/tests/loaders/ndjson/test_bulk_export.py +++ b/tests/loaders/ndjson/test_bulk_export.py @@ -472,7 +472,7 @@ async def test_delay(self, mock_sleep): await self.export() # 86760 == 24 hours + six minutes - self.assertEqual(86760, self.exporter._total_wait_time) # pylint: disable=protected-access + self.assertEqual(86760, self.exporter._total_wait_time) self.assertListEqual( [ diff --git a/tests/loaders/ndjson/test_ndjson_loader.py b/tests/loaders/ndjson/test_ndjson_loader.py index 4c2c718..a44b877 100644 --- a/tests/loaders/ndjson/test_ndjson_loader.py +++ b/tests/loaders/ndjson/test_ndjson_loader.py @@ -21,7 +21,7 @@ class TestNdjsonLoader(AsyncTestCase): def setUp(self): super().setUp() - self.jwks_file = tempfile.NamedTemporaryFile() # pylint: disable=consider-using-with + self.jwks_file = tempfile.NamedTemporaryFile() self.jwks_path = self.jwks_file.name self.jwks_file.write(b'{"fake":"jwks"}') self.jwks_file.flush() diff --git a/tests/s3mock.py b/tests/s3mock.py index befdc93..2c33c75 100644 --- a/tests/s3mock.py +++ b/tests/s3mock.py @@ -54,7 +54,7 @@ def setUp(self): try: self.s3fs.mkdir(self.bucket) # create the bucket as a quickstart - except Exception: # pylint: disable=broad-except + except Exception: self._kill_moto_server() self.fail("Stale moto server") diff --git a/tests/utils.py b/tests/utils.py index 74dc97b..f62372c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -57,7 +57,7 @@ def setUp(self): def make_tempdir(self) -> str: """Creates a temporary dir that will be automatically cleaned up""" - tempdir = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with + tempdir = tempfile.TemporaryDirectory() self.addCleanup(tempdir.cleanup) return tempdir.name @@ -120,7 +120,7 @@ def setUp(self): filecmp.clear_cache() # you'll always want this when debugging - self.maxDiff = None # pylint: disable=invalid-name + self.maxDiff = None def assert_etl_output_equal(self, left: str, right: str): """Compares the etl output with the expected json structure""" @@ -188,7 +188,7 @@ def setUp(self): ).export(as_dict=True) self.fhir_jwks = {"keys": [jwk_token]} - self._fhir_jwks_file = tempfile.NamedTemporaryFile() # pylint: disable=consider-using-with + self._fhir_jwks_file = tempfile.NamedTemporaryFile() self._fhir_jwks_file.write(json.dumps(self.fhir_jwks).encode("utf8")) self._fhir_jwks_file.flush() self.addCleanup(self._fhir_jwks_file.close)