diff --git a/.gitignore b/.gitignore index 2a9ace3..231a73d 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,4 @@ docker-compose.override.yml ### Extra lines below are preserved ### /.env +*.log diff --git a/api/bro_import/bulk_import.py b/api/bro_import/bulk_import.py index 14a22a4..1b3f1aa 100644 --- a/api/bro_import/bulk_import.py +++ b/api/bro_import/bulk_import.py @@ -46,7 +46,9 @@ def run(self) -> None: data_importer.run() except requests.RequestException as e: logger.exception(e) - raise DataImportError(f"Error fetching BRO IDs from {url}: {e}") from e + raise DataImportError( + f"Error while importing data for bro id: {bro_id}: {e}" + ) from e def _create_bro_ids_import_url(self) -> str: """Creates the import url for a given bro object type and kvk combination.""" diff --git a/api/bro_upload/delivery.py b/api/bro_upload/delivery.py index 488b153..a07c066 100644 --- a/api/bro_upload/delivery.py +++ b/api/bro_upload/delivery.py @@ -53,6 +53,7 @@ def __init__( self.upload_task_instance = upload_task_instance self.bro_username = bro_username self.bro_password = bro_password + self.bro_id = None def process(self) -> None: # Generate the XML file. @@ -69,7 +70,7 @@ def process(self) -> None: while retries_count < 4: if self._check_delivery(deliver_url): - return + return self.bro_id else: time.sleep(10) retries_count += 1 @@ -152,6 +153,9 @@ def _check_delivery(self, delivery_url: str) -> bool: delivery_status == "DOORGELEVERD" and delivery_brondocument_status == "OPGENOMEN_LVBRO" ): + # Set BRO id to self to enable an import task based on the bro id. This keeps the data up2date in the api. + self.bro_id = delivery_info["brondocuments"][0]["broId"] + return True else: diff --git a/api/migrations/0023_uploadtask_bro_id_alter_uploadtask_registration_type.py b/api/migrations/0023_uploadtask_bro_id_alter_uploadtask_registration_type.py new file mode 100644 index 0000000..229e950 --- /dev/null +++ b/api/migrations/0023_uploadtask_bro_id_alter_uploadtask_registration_type.py @@ -0,0 +1,32 @@ +# Generated by Django 5.0.1 on 2024-03-22 09:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0022_alter_uploadtask_registration_type"), + ] + + operations = [ + migrations.AddField( + model_name="uploadtask", + name="bro_id", + field=models.CharField(blank=True, max_length=500, null=True), + ), + migrations.AlterField( + model_name="uploadtask", + name="registration_type", + field=models.CharField( + choices=[ + ("GMN_StartRegistration", "GMN_StartRegistration"), + ("GMN_MeasuringPoint", "GMN_MeasuringPoint"), + ("GMN_MeasuringPointEndDate", "GMN_MeasuringPointEndDate"), + ("GMN_TubeReference", "GMN_TubeReference"), + ("GMN_Closure", "GMN_Closure"), + ("GMW", "GWM"), + ], + max_length=235, + ), + ), + ] diff --git a/api/models.py b/api/models.py index b24e55a..9174aac 100644 --- a/api/models.py +++ b/api/models.py @@ -76,6 +76,7 @@ class UploadTask(models.Model): metadata = JSONField("Metadata", default=dict, blank=False) sourcedocument_data = JSONField("Sourcedocument data", default=dict, blank=False) status = models.CharField(max_length=500, blank=True, null=True) + bro_id = models.CharField(max_length=500, blank=True, null=True) log = models.TextField(blank=True) def __str__(self) -> str: diff --git a/api/tasks.py b/api/tasks.py index 31e1e2e..9f312c8 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -1,9 +1,14 @@ +import logging + +import requests from celery import shared_task from . import models as api_models -from .bro_import import bulk_import +from .bro_import import bulk_import, config from .bro_upload.delivery import BRODelivery +logger = logging.getLogger(__name__) + @shared_task def import_bro_data_task(import_task_instance_uuid: str) -> None: @@ -58,13 +63,31 @@ def upload_bro_data_task( try: # The actual task - uploader.process() + bro_id = uploader.process() # Update upload task instance + upload_task_instance.bro_id = bro_id upload_task_instance.status = "COMPLETED" upload_task_instance.log = "The upload was done successfully" upload_task_instance.save() + # Start import task to keep the data up to date in the api + try: + object_importer_class = config.object_importer_mapping[ + upload_task_instance.bro_domain + ] + importer = object_importer_class( + bro_domain=upload_task_instance.bro_domain, + bro_id=upload_task_instance.bro_id, + data_owner=upload_task_instance.data_owner, + ) + importer.run() + except requests.RequestException as e: + logger.exception(e) + raise bulk_import.DataImportError( + f"Error while importing data for bro id: {bro_id}: {e}" + ) from e + except Exception as e: upload_task_instance.log = e upload_task_instance.status = "FAILED" diff --git a/bro_hub/settings.py b/bro_hub/settings.py index d2d6ee3..4cd244f 100644 --- a/bro_hub/settings.py +++ b/bro_hub/settings.py @@ -76,6 +76,25 @@ }, ] +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "handlers": { + "file": { + "level": "DEBUG", + "class": "logging.FileHandler", + "filename": os.path.join(BASE_DIR, "django_log.log"), + }, + }, + "loggers": { + "": { + "handlers": ["file"], + "level": "DEBUG", + "propagate": True, + }, + }, +} + WSGI_APPLICATION = "bro_hub.wsgi.application" CORS_ALLOWED_ORIGINS = [