From a75a2ddfee212c47123afd7987fe723df2122fff Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 21 Nov 2023 13:03:14 +0100 Subject: [PATCH 01/12] poetry - use acacore 1.1.2 --- poetry.lock | 14 +++++++------- pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/poetry.lock b/poetry.lock index 78a260d9..4537c940 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2,7 +2,7 @@ [[package]] name = "acacore" -version = "1.1.1" +version = "1.1.2" description = "" optional = false python-versions = "^3.9" @@ -18,8 +18,8 @@ pyyaml = "^6.0.1" [package.source] type = "git" url = "https://github.com/aarhusstadsarkiv/acacore.git" -reference = "v1.1.1" -resolved_reference = "f484826756f4eeec63e238514ba77b7ebdb3f6b1" +reference = "v1.1.2" +resolved_reference = "bff3199b6fcd0d8617a297b203d3cd9b1277d1ca" [[package]] name = "annotated-types" @@ -158,13 +158,13 @@ toml = ["tomli"] [[package]] name = "exceptiongroup" -version = "1.1.3" +version = "1.2.0" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"}, - {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"}, + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, ] [package.extras] @@ -579,4 +579,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "95814b5aaa6aae56cffa9de1782416a101f8dc02d74474756e2b09817a2be624" +content-hash = "fcc904e146184e65a7fece47c0e9b3fe06afbac8ec0d36cc2386fe585a162a55" diff --git a/pyproject.toml b/pyproject.toml index af895daf..efa3d5f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ include = ["pyproject.toml"] [tool.poetry.dependencies] python = "^3.9" -acacore = {git = "https://github.com/aarhusstadsarkiv/acacore.git", tag = "v1.1.1"} +acacore = {git = "https://github.com/aarhusstadsarkiv/acacore.git", tag = "v1.1.2"} [tool.poetry.group.dev.dependencies] pytest = "^7.0" From 457fdd206d127f011c2f757e54b32f0a6ff05012 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 21 Nov 2023 13:06:10 +0100 Subject: [PATCH 02/12] cli:identify - catch Exception and DecompressionBombError but raise OSError --- digiarch/cli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/digiarch/cli.py b/digiarch/cli.py index 5971dd25..9efa5a59 100644 --- a/digiarch/cli.py +++ b/digiarch/cli.py @@ -37,7 +37,7 @@ from click import pass_context from click import Path as ClickPath from click import version_option -from PIL import UnidentifiedImageError +from PIL.Image import DecompressionBombError from pydantic import TypeAdapter from .__version__ import __version__ @@ -196,15 +196,15 @@ def app_identify( file_history: list[HistoryEntry] = [] - with ExceptionManager(UnidentifiedImageError) as image_exception: + with ExceptionManager(Exception, DecompressionBombError, allow=[OSError]) as identify_error: file = File.from_file(path, root, siegfried, actions, custom_signatures) - if image_exception.exception: + if identify_error.exception: file = File.from_file(path, root, siegfried) file.action = "manual" file.action_data = ActionData( manual=ManualAction( - reason=image_exception.exception.__class__.__name__, + reason=identify_error.exception.__class__.__name__, process="Identify and fix error.", ), ) @@ -213,7 +213,7 @@ def app_identify( ctx, "file:identify:error", file.uuid, - repr(image_exception.exception), + repr(identify_error.exception), "".join(format_tb(exception.traceback)) if exception.traceback else None, ), ) From 94183e92b393c905d513b9b656e40e06ee3db692 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 21 Nov 2023 13:07:16 +0100 Subject: [PATCH 03/12] cli - better end history entry on error --- digiarch/cli.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/digiarch/cli.py b/digiarch/cli.py index 9efa5a59..4183019b 100644 --- a/digiarch/cli.py +++ b/digiarch/cli.py @@ -72,7 +72,11 @@ def handle_end(ctx: Context, database: FileDB, exception: ExceptionManager, *log ctx, "end", data=1 if exception.exception else 0, - reason="".join(format_tb(exception.traceback)) if exception.traceback else None, + reason=("".join(format_tb(exception.traceback)) + f"\n\n{exception.exception!r}") + if exception.traceback + else repr(exception.exception) + if exception.exception + else None, ) for logger in loggers: From 5786f0412ca2b679969ca692e81d3d7bed9412d5 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 21 Nov 2023 13:10:53 +0100 Subject: [PATCH 04/12] cli:identify - catch UnidentifiedImageError --- digiarch/cli.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/digiarch/cli.py b/digiarch/cli.py index 4183019b..22b9962f 100644 --- a/digiarch/cli.py +++ b/digiarch/cli.py @@ -9,6 +9,7 @@ from uuid import UUID import yaml +from PIL import UnidentifiedImageError from acacore.models.file import File from acacore.models.history import HistoryEntry from acacore.models.reference_files import Action @@ -200,7 +201,12 @@ def app_identify( file_history: list[HistoryEntry] = [] - with ExceptionManager(Exception, DecompressionBombError, allow=[OSError]) as identify_error: + with ExceptionManager( + Exception, + UnidentifiedImageError, + DecompressionBombError, + allow=[OSError], + ) as identify_error: file = File.from_file(path, root, siegfried, actions, custom_signatures) if identify_error.exception: From 34f8c4a1e4f71b0aa7eafeb4dca8c953f7544815 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 21 Nov 2023 13:12:23 +0100 Subject: [PATCH 05/12] cli:identify - raise IOError --- digiarch/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/digiarch/cli.py b/digiarch/cli.py index 22b9962f..b8522145 100644 --- a/digiarch/cli.py +++ b/digiarch/cli.py @@ -9,7 +9,6 @@ from uuid import UUID import yaml -from PIL import UnidentifiedImageError from acacore.models.file import File from acacore.models.history import HistoryEntry from acacore.models.reference_files import Action @@ -38,6 +37,7 @@ from click import pass_context from click import Path as ClickPath from click import version_option +from PIL import UnidentifiedImageError from PIL.Image import DecompressionBombError from pydantic import TypeAdapter @@ -205,7 +205,7 @@ def app_identify( Exception, UnidentifiedImageError, DecompressionBombError, - allow=[OSError], + allow=[OSError, IOError], ) as identify_error: file = File.from_file(path, root, siegfried, actions, custom_signatures) From 67bd091b2bf09af6103420a86bc67f1fb2383b03 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 21 Nov 2023 13:20:51 +0100 Subject: [PATCH 06/12] cli - set max image size processed by pillow to 50e3^2 --- digiarch/cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/digiarch/cli.py b/digiarch/cli.py index b8522145..e4607b1d 100644 --- a/digiarch/cli.py +++ b/digiarch/cli.py @@ -37,6 +37,7 @@ from click import pass_context from click import Path as ClickPath from click import version_option +from PIL import Image from PIL import UnidentifiedImageError from PIL.Image import DecompressionBombError from pydantic import TypeAdapter @@ -44,6 +45,8 @@ from .__version__ import __version__ from .database import FileDB +Image.MAX_IMAGE_PIXELS = int(50e3**2) + def handle_rename(file: File, action: RenameAction) -> Union[tuple[Path, Path], tuple[None, None]]: old_path: Path = file.get_absolute_path() From dfc93e057ad911aa40e67cd264251340247b70ee Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 21 Nov 2023 13:33:53 +0100 Subject: [PATCH 07/12] cli - use None or exception repr for ending history event --- digiarch/cli.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/digiarch/cli.py b/digiarch/cli.py index e4607b1d..51747acc 100644 --- a/digiarch/cli.py +++ b/digiarch/cli.py @@ -75,12 +75,8 @@ def handle_end(ctx: Context, database: FileDB, exception: ExceptionManager, *log program_end: HistoryEntry = HistoryEntry.command_history( ctx, "end", - data=1 if exception.exception else 0, - reason=("".join(format_tb(exception.traceback)) + f"\n\n{exception.exception!r}") - if exception.traceback - else repr(exception.exception) - if exception.exception - else None, + data=repr(exception.exception), + reason="".join(format_tb(exception.traceback)) if exception.traceback else None, ) for logger in loggers: From b898a364d2e1611174179cc3f57c73d58705d01a Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 21 Nov 2023 13:38:00 +0100 Subject: [PATCH 08/12] tests:cli - update test for last history event --- tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index f923b171..c61c5aaf 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -98,7 +98,7 @@ def test_identify(tests_folder: Path, files_folder: Path, files_folder_copy: Pat with FileDB(files_folder_copy / "_metadata" / "files.db") as database: last_history: HistoryEntry = sorted(database.history, key=lambda h: h.time).pop() - assert last_history.data == 1 + assert isinstance(last_history.data, str) and last_history.data.startswith("FileNotFoundError") assert last_history.reason is not None From 0e57a38d607547bc3106356671dc1a2b068829db Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 21 Nov 2023 13:38:24 +0100 Subject: [PATCH 09/12] tests:cli - format --- tests/test_cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index c61c5aaf..ff9898b5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -98,7 +98,8 @@ def test_identify(tests_folder: Path, files_folder: Path, files_folder_copy: Pat with FileDB(files_folder_copy / "_metadata" / "files.db") as database: last_history: HistoryEntry = sorted(database.history, key=lambda h: h.time).pop() - assert isinstance(last_history.data, str) and last_history.data.startswith("FileNotFoundError") + assert isinstance(last_history.data, str) + assert last_history.data.startswith("FileNotFoundError") assert last_history.reason is not None From 6e0c3ee209a49810daadbcfe3d733bb21aa7d69c Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 21 Nov 2023 13:38:59 +0100 Subject: [PATCH 10/12] version - patch 1.0.3 > 1.0.4 --- digiarch/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/digiarch/__version__.py b/digiarch/__version__.py index 976498ab..92192eed 100644 --- a/digiarch/__version__.py +++ b/digiarch/__version__.py @@ -1 +1 @@ -__version__ = "1.0.3" +__version__ = "1.0.4" From 9e70e468ea765aae6ff2821daf417bf3dadeac94 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 21 Nov 2023 13:38:59 +0100 Subject: [PATCH 11/12] poetry - version patch 1.0.3 > 1.0.4 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index efa3d5f7..f14b291b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "digiarch" -version = "1.0.3" +version = "1.0.4" description = "Tools for the Digital Archive Project at Aarhus Stadsarkiv" authors = ["Aryan Muhammadi Landi ", "Nina Jensen ", "Aarhus Stadsarkiv "] license = "GPL-3.0" From 07860d9ee3f6202fd0d86a28e660818a1ca61e3a Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 21 Nov 2023 13:51:32 +0100 Subject: [PATCH 12/12] cli - fix incorrect value in end program event if no error occurred --- digiarch/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/digiarch/cli.py b/digiarch/cli.py index 51747acc..724f8941 100644 --- a/digiarch/cli.py +++ b/digiarch/cli.py @@ -75,7 +75,7 @@ def handle_end(ctx: Context, database: FileDB, exception: ExceptionManager, *log program_end: HistoryEntry = HistoryEntry.command_history( ctx, "end", - data=repr(exception.exception), + data=repr(exception.exception) if exception.exception else None, reason="".join(format_tb(exception.traceback)) if exception.traceback else None, )