From 7dc3e675519e25cf511fae583138d076e1b1bfc9 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Fri, 22 Mar 2024 10:18:18 +0100 Subject: [PATCH 1/3] added object import after an upload --- api/bro_import/bulk_import.py | 2 +- api/bro_upload/delivery.py | 7 +++++- ...o_id_alter_uploadtask_registration_type.py | 23 +++++++++++++++++++ api/models.py | 1 + api/tasks.py | 23 +++++++++++++++++-- 5 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 api/migrations/0023_uploadtask_bro_id_alter_uploadtask_registration_type.py diff --git a/api/bro_import/bulk_import.py b/api/bro_import/bulk_import.py index 14a22a4..acc736d 100644 --- a/api/bro_import/bulk_import.py +++ b/api/bro_import/bulk_import.py @@ -46,7 +46,7 @@ def run(self) -> None: data_importer.run() except requests.RequestException as e: logger.exception(e) - raise DataImportError(f"Error fetching BRO IDs from {url}: {e}") from e + raise DataImportError(f"Error while importing data for bro id: {bro_id}: {e}") from e def _create_bro_ids_import_url(self) -> str: """Creates the import url for a given bro object type and kvk combination.""" diff --git a/api/bro_upload/delivery.py b/api/bro_upload/delivery.py index 488b153..2237329 100644 --- a/api/bro_upload/delivery.py +++ b/api/bro_upload/delivery.py @@ -53,6 +53,7 @@ def __init__( self.upload_task_instance = upload_task_instance self.bro_username = bro_username self.bro_password = bro_password + self.bro_id = None def process(self) -> None: # Generate the XML file. @@ -69,7 +70,7 @@ def process(self) -> None: while retries_count < 4: if self._check_delivery(deliver_url): - return + return self.bro_id else: time.sleep(10) retries_count += 1 @@ -139,6 +140,7 @@ def _check_delivery(self, delivery_url: str) -> bool: delivery_url, self.bro_username, self.bro_password ) + errors = delivery_info["brondocuments"][0]["errors"] if errors: @@ -152,6 +154,9 @@ def _check_delivery(self, delivery_url: str) -> bool: delivery_status == "DOORGELEVERD" and delivery_brondocument_status == "OPGENOMEN_LVBRO" ): + # Set BRO id to self to enable an import task based on the bro id. This keeps the data up2date in the api. + self.bro_id = delivery_info["brondocuments"][0]["broId"] + return True else: diff --git a/api/migrations/0023_uploadtask_bro_id_alter_uploadtask_registration_type.py b/api/migrations/0023_uploadtask_bro_id_alter_uploadtask_registration_type.py new file mode 100644 index 0000000..a430c96 --- /dev/null +++ b/api/migrations/0023_uploadtask_bro_id_alter_uploadtask_registration_type.py @@ -0,0 +1,23 @@ +# Generated by Django 5.0.1 on 2024-03-22 09:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0022_alter_uploadtask_registration_type'), + ] + + operations = [ + migrations.AddField( + model_name='uploadtask', + name='bro_id', + field=models.CharField(blank=True, max_length=500, null=True), + ), + migrations.AlterField( + model_name='uploadtask', + name='registration_type', + field=models.CharField(choices=[('GMN_StartRegistration', 'GMN_StartRegistration'), ('GMN_MeasuringPoint', 'GMN_MeasuringPoint'), ('GMN_MeasuringPointEndDate', 'GMN_MeasuringPointEndDate'), ('GMN_TubeReference', 'GMN_TubeReference'), ('GMN_Closure', 'GMN_Closure'), ('GMW', 'GWM')], max_length=235), + ), + ] diff --git a/api/models.py b/api/models.py index b24e55a..9174aac 100644 --- a/api/models.py +++ b/api/models.py @@ -76,6 +76,7 @@ class UploadTask(models.Model): metadata = JSONField("Metadata", default=dict, blank=False) sourcedocument_data = JSONField("Sourcedocument data", default=dict, blank=False) status = models.CharField(max_length=500, blank=True, null=True) + bro_id = models.CharField(max_length=500, blank=True, null=True) log = models.TextField(blank=True) def __str__(self) -> str: diff --git a/api/tasks.py b/api/tasks.py index 31e1e2e..20dcbe6 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -1,9 +1,14 @@ +import logging + +import requests from celery import shared_task from . import models as api_models -from .bro_import import bulk_import +from .bro_import import bulk_import, config from .bro_upload.delivery import BRODelivery +logger = logging.getLogger(__name__) + @shared_task def import_bro_data_task(import_task_instance_uuid: str) -> None: @@ -58,13 +63,27 @@ def upload_bro_data_task( try: # The actual task - uploader.process() + bro_id = uploader.process() # Update upload task instance + upload_task_instance.bro_id = bro_id upload_task_instance.status = "COMPLETED" upload_task_instance.log = "The upload was done successfully" upload_task_instance.save() + # Start import task to keep the data up to date in the api + try: + object_importer_class = config.object_importer_mapping[upload_task_instance.bro_domain] + object_importer_class( + bro_domain = upload_task_instance.bro_domain, + bro_id = upload_task_instance.bro_id, + data_owner = upload_task_instance.data_owner + ) + except requests.RequestException as e: + logger.exception(e) + raise bulk_import.DataImportError(f"Error while importing data for bro id: {bro_id}: {e}") from e + + except Exception as e: upload_task_instance.log = e upload_task_instance.status = "FAILED" From aeee97480beb1bb1b53427737b037d6a1516b959 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Fri, 22 Mar 2024 10:37:44 +0100 Subject: [PATCH 2/3] adde dlogging --- .gitignore | 1 + bro_hub/settings.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/.gitignore b/.gitignore index 2a9ace3..231a73d 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,4 @@ docker-compose.override.yml ### Extra lines below are preserved ### /.env +*.log diff --git a/bro_hub/settings.py b/bro_hub/settings.py index d2d6ee3..5d2ff82 100644 --- a/bro_hub/settings.py +++ b/bro_hub/settings.py @@ -76,6 +76,25 @@ }, ] +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'file': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + 'filename': os.path.join(BASE_DIR, 'django_log.log'), + }, + }, + 'loggers': { + '': { + 'handlers': ['file'], + 'level': 'DEBUG', + 'propagate': True, + }, + }, +} + WSGI_APPLICATION = "bro_hub.wsgi.application" CORS_ALLOWED_ORIGINS = [ From 5fc4d982be721b7003c3cfc2020e626cc746bd4c Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Fri, 22 Mar 2024 11:11:33 +0100 Subject: [PATCH 3/3] precommit --- api/bro_import/bulk_import.py | 4 +++- api/bro_upload/delivery.py | 1 - ...o_id_alter_uploadtask_registration_type.py | 23 ++++++++++++------ api/tasks.py | 20 +++++++++------- bro_hub/settings.py | 24 +++++++++---------- 5 files changed, 43 insertions(+), 29 deletions(-) diff --git a/api/bro_import/bulk_import.py b/api/bro_import/bulk_import.py index acc736d..1b3f1aa 100644 --- a/api/bro_import/bulk_import.py +++ b/api/bro_import/bulk_import.py @@ -46,7 +46,9 @@ def run(self) -> None: data_importer.run() except requests.RequestException as e: logger.exception(e) - raise DataImportError(f"Error while importing data for bro id: {bro_id}: {e}") from e + raise DataImportError( + f"Error while importing data for bro id: {bro_id}: {e}" + ) from e def _create_bro_ids_import_url(self) -> str: """Creates the import url for a given bro object type and kvk combination.""" diff --git a/api/bro_upload/delivery.py b/api/bro_upload/delivery.py index 2237329..a07c066 100644 --- a/api/bro_upload/delivery.py +++ b/api/bro_upload/delivery.py @@ -140,7 +140,6 @@ def _check_delivery(self, delivery_url: str) -> bool: delivery_url, self.bro_username, self.bro_password ) - errors = delivery_info["brondocuments"][0]["errors"] if errors: diff --git a/api/migrations/0023_uploadtask_bro_id_alter_uploadtask_registration_type.py b/api/migrations/0023_uploadtask_bro_id_alter_uploadtask_registration_type.py index a430c96..229e950 100644 --- a/api/migrations/0023_uploadtask_bro_id_alter_uploadtask_registration_type.py +++ b/api/migrations/0023_uploadtask_bro_id_alter_uploadtask_registration_type.py @@ -4,20 +4,29 @@ class Migration(migrations.Migration): - dependencies = [ - ('api', '0022_alter_uploadtask_registration_type'), + ("api", "0022_alter_uploadtask_registration_type"), ] operations = [ migrations.AddField( - model_name='uploadtask', - name='bro_id', + model_name="uploadtask", + name="bro_id", field=models.CharField(blank=True, max_length=500, null=True), ), migrations.AlterField( - model_name='uploadtask', - name='registration_type', - field=models.CharField(choices=[('GMN_StartRegistration', 'GMN_StartRegistration'), ('GMN_MeasuringPoint', 'GMN_MeasuringPoint'), ('GMN_MeasuringPointEndDate', 'GMN_MeasuringPointEndDate'), ('GMN_TubeReference', 'GMN_TubeReference'), ('GMN_Closure', 'GMN_Closure'), ('GMW', 'GWM')], max_length=235), + model_name="uploadtask", + name="registration_type", + field=models.CharField( + choices=[ + ("GMN_StartRegistration", "GMN_StartRegistration"), + ("GMN_MeasuringPoint", "GMN_MeasuringPoint"), + ("GMN_MeasuringPointEndDate", "GMN_MeasuringPointEndDate"), + ("GMN_TubeReference", "GMN_TubeReference"), + ("GMN_Closure", "GMN_Closure"), + ("GMW", "GWM"), + ], + max_length=235, + ), ), ] diff --git a/api/tasks.py b/api/tasks.py index 20dcbe6..9f312c8 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -73,16 +73,20 @@ def upload_bro_data_task( # Start import task to keep the data up to date in the api try: - object_importer_class = config.object_importer_mapping[upload_task_instance.bro_domain] - object_importer_class( - bro_domain = upload_task_instance.bro_domain, - bro_id = upload_task_instance.bro_id, - data_owner = upload_task_instance.data_owner + object_importer_class = config.object_importer_mapping[ + upload_task_instance.bro_domain + ] + importer = object_importer_class( + bro_domain=upload_task_instance.bro_domain, + bro_id=upload_task_instance.bro_id, + data_owner=upload_task_instance.data_owner, ) + importer.run() except requests.RequestException as e: - logger.exception(e) - raise bulk_import.DataImportError(f"Error while importing data for bro id: {bro_id}: {e}") from e - + logger.exception(e) + raise bulk_import.DataImportError( + f"Error while importing data for bro id: {bro_id}: {e}" + ) from e except Exception as e: upload_task_instance.log = e diff --git a/bro_hub/settings.py b/bro_hub/settings.py index 5d2ff82..4cd244f 100644 --- a/bro_hub/settings.py +++ b/bro_hub/settings.py @@ -77,20 +77,20 @@ ] LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'handlers': { - 'file': { - 'level': 'DEBUG', - 'class': 'logging.FileHandler', - 'filename': os.path.join(BASE_DIR, 'django_log.log'), + "version": 1, + "disable_existing_loggers": False, + "handlers": { + "file": { + "level": "DEBUG", + "class": "logging.FileHandler", + "filename": os.path.join(BASE_DIR, "django_log.log"), }, }, - 'loggers': { - '': { - 'handlers': ['file'], - 'level': 'DEBUG', - 'propagate': True, + "loggers": { + "": { + "handlers": ["file"], + "level": "DEBUG", + "propagate": True, }, }, }