Merge pull request #11 from nens/single_import

import object after upload added

.gitignore

@@ -43,3 +43,4 @@ docker-compose.override.yml
 
 ### Extra lines below are preserved ###
 /.env
+*.log

api/bro_import/bulk_import.py

@@ -46,7 +46,9 @@ def run(self) -> None:
                 data_importer.run()
             except requests.RequestException as e:
                 logger.exception(e)
-                raise DataImportError(f"Error fetching BRO IDs from {url}: {e}") from e
+                raise DataImportError(
+                    f"Error while importing data for bro id: {bro_id}: {e}"
+                ) from e
 
     def _create_bro_ids_import_url(self) -> str:
         """Creates the import url for a given bro object type and kvk combination."""

api/bro_upload/delivery.py

@@ -53,6 +53,7 @@ def __init__(
         self.upload_task_instance = upload_task_instance
         self.bro_username = bro_username
         self.bro_password = bro_password
+        self.bro_id = None
 
     def process(self) -> None:
         # Generate the XML file.
@@ -69,7 +70,7 @@ def process(self) -> None:
 
         while retries_count < 4:
             if self._check_delivery(deliver_url):
-                return
+                return self.bro_id
             else:
                 time.sleep(10)
                 retries_count += 1
@@ -152,6 +153,9 @@ def _check_delivery(self, delivery_url: str) -> bool:
                 delivery_status == "DOORGELEVERD"
                 and delivery_brondocument_status == "OPGENOMEN_LVBRO"
             ):
+                # Set BRO id to self to enable an import task based on the bro id. This keeps the data up2date in the api.
+                self.bro_id = delivery_info["brondocuments"][0]["broId"]
+
                 return True
 
             else:

api/migrations/0023_uploadtask_bro_id_alter_uploadtask_registration_type.py

@@ -0,0 +1,32 @@
+# Generated by Django 5.0.1 on 2024-03-22 09:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("api", "0022_alter_uploadtask_registration_type"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="uploadtask",
+            name="bro_id",
+            field=models.CharField(blank=True, max_length=500, null=True),
+        ),
+        migrations.AlterField(
+            model_name="uploadtask",
+            name="registration_type",
+            field=models.CharField(
+                choices=[
+                    ("GMN_StartRegistration", "GMN_StartRegistration"),
+                    ("GMN_MeasuringPoint", "GMN_MeasuringPoint"),
+                    ("GMN_MeasuringPointEndDate", "GMN_MeasuringPointEndDate"),
+                    ("GMN_TubeReference", "GMN_TubeReference"),
+                    ("GMN_Closure", "GMN_Closure"),
+                    ("GMW", "GWM"),
+                ],
+                max_length=235,
+            ),
+        ),
+    ]

api/models.py

@@ -76,6 +76,7 @@ class UploadTask(models.Model):
     metadata = JSONField("Metadata", default=dict, blank=False)
     sourcedocument_data = JSONField("Sourcedocument data", default=dict, blank=False)
     status = models.CharField(max_length=500, blank=True, null=True)
+    bro_id = models.CharField(max_length=500, blank=True, null=True)
     log = models.TextField(blank=True)
 
     def __str__(self) -> str:

api/tasks.py

@@ -1,9 +1,14 @@
+import logging
+
+import requests
 from celery import shared_task
 
 from . import models as api_models
-from .bro_import import bulk_import
+from .bro_import import bulk_import, config
 from .bro_upload.delivery import BRODelivery
 
+logger = logging.getLogger(__name__)
+
 
 @shared_task
 def import_bro_data_task(import_task_instance_uuid: str) -> None:
@@ -58,13 +63,31 @@ def upload_bro_data_task(
 
     try:
         # The actual task
-        uploader.process()
+        bro_id = uploader.process()
 
         # Update upload task instance
+        upload_task_instance.bro_id = bro_id
         upload_task_instance.status = "COMPLETED"
         upload_task_instance.log = "The upload was done successfully"
         upload_task_instance.save()
 
+        # Start import task to keep the data up to date in the api
+        try:
+            object_importer_class = config.object_importer_mapping[
+                upload_task_instance.bro_domain
+            ]
+            importer = object_importer_class(
+                bro_domain=upload_task_instance.bro_domain,
+                bro_id=upload_task_instance.bro_id,
+                data_owner=upload_task_instance.data_owner,
+            )
+            importer.run()
+        except requests.RequestException as e:
+            logger.exception(e)
+            raise bulk_import.DataImportError(
+                f"Error while importing data for bro id: {bro_id}: {e}"
+            ) from e
+
     except Exception as e:
         upload_task_instance.log = e
         upload_task_instance.status = "FAILED"

bro_hub/settings.py

@@ -76,6 +76,25 @@
     },
 ]
 
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "handlers": {
+        "file": {
+            "level": "DEBUG",
+            "class": "logging.FileHandler",
+            "filename": os.path.join(BASE_DIR, "django_log.log"),
+        },
+    },
+    "loggers": {
+        "": {
+            "handlers": ["file"],
+            "level": "DEBUG",
+            "propagate": True,
+        },
+    },
+}
+
 WSGI_APPLICATION = "bro_hub.wsgi.application"
 
 CORS_ALLOWED_ORIGINS = [