From 47ca3e347bfce8ff825fa5630fe79da8b54ecdbc Mon Sep 17 00:00:00 2001 From: StevenHosper Date: Thu, 12 Dec 2024 08:18:40 +0100 Subject: [PATCH] Add to bulk-import for GLD --- CHANGES.md | 1 + api/bro_upload/upload_datamodels.py | 8 +- api/choices.py | 1 + api/serializers.py | 30 +++++++- api/tests/test_bulk_upload.py | 7 +- api/views.py | 2 +- manage.py | 1 + upload_examples.ipynb | 115 ++++++++++++---------------- 8 files changed, 92 insertions(+), 73 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4e98bc4..dd18563 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ - Update the docs. - Add the GLD Endpoints and models. +- Add GLD option to bulk endpoint. ## 0.66 (2024-12-05) diff --git a/api/bro_upload/upload_datamodels.py b/api/bro_upload/upload_datamodels.py index ab2da92..a715ed4 100644 --- a/api/bro_upload/upload_datamodels.py +++ b/api/bro_upload/upload_datamodels.py @@ -139,7 +139,9 @@ class GMWConstruction(BaseModel): deliveryContext: str constructionStandard: str initialFunction: str - numberOfMonitoringTubes: str | int # Should this not be derived from 'monitoringTubes' + numberOfMonitoringTubes: ( + str | int + ) # Should this not be derived from 'monitoringTubes' groundLevelStable: str wellStability: str | None = None owner: str | None = None @@ -478,4 +480,6 @@ class FRDGemMeasurement(BaseModel): determinationProcedure: str evaluationProcedure: str measurements: list[GemMeasurement] - relatedCalculatedApparentFormationResistance: RelatedCalculatedApparentFormationResistance | None = None + relatedCalculatedApparentFormationResistance: ( + RelatedCalculatedApparentFormationResistance | None + ) = None diff --git a/api/choices.py b/api/choices.py index b57a8d5..cdfa9aa 100644 --- a/api/choices.py +++ b/api/choices.py @@ -47,6 +47,7 @@ BULK_UPLOAD_TYPES = [ ("GAR", "GAR"), + ("GLD", "GLD"), ] # NOTE: Add new registration types to the registration_type_datamodel_mapping below diff --git a/api/serializers.py b/api/serializers.py index 3c8e108..74032ba 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -62,16 +62,37 @@ class Meta: class BulkUploadSerializer(UrlFieldMixin, serializers.ModelSerializer): - fieldwork_file = serializers.FileField(write_only=True, required=True) - lab_file = serializers.FileField(write_only=True, required=True) + fieldwork_file = serializers.FileField(write_only=True, required=False) + lab_file = serializers.FileField(write_only=True, required=False) + measurement_tvp_file = serializers.FileField(write_only=True, required=False) class Meta: model = api_models.BulkUpload fields = "__all__" + def validate(self, attrs): + fieldwork_file = attrs.get("fieldwork_file") + lab_file = attrs.get("lab_file") + measurement_tvp_file = attrs.get("measurement_tvp_file") + bulk_upload_type = attrs.get("bulk_upload_type") + + # Check if either both fieldwork_file and lab_file are present + # or measurement_tvp_file is present + if bulk_upload_type == "GAR" and not (fieldwork_file and lab_file): + raise serializers.ValidationError( + "Both 'fieldwork_file' and 'lab_file' must be provided, when a GAR bulk-upload is requested." + ) + elif bulk_upload_type == "GLD" and not measurement_tvp_file: + raise serializers.ValidationError( + "A 'measurement_tvp_file' must be provided, when a GLD bulk-upload is requested." + ) + return attrs + def create(self, validated_data): fieldwork_file = validated_data.pop("fieldwork_file", None) lab_file = validated_data.pop("lab_file", None) + measurement_tvp_file = validated_data.pop("measurement_tvp_file", None) + bulk_upload = api_models.BulkUpload.objects.create(**validated_data) if fieldwork_file: @@ -81,4 +102,9 @@ def create(self, validated_data): if lab_file: api_models.UploadFile.objects.create(bulk_upload=bulk_upload, file=lab_file) + if measurement_tvp_file: + api_models.UploadFile.objects.create( + bulk_upload=bulk_upload, file=measurement_tvp_file + ) + return bulk_upload diff --git a/api/tests/test_bulk_upload.py b/api/tests/test_bulk_upload.py index 5af5a58..c26aadf 100644 --- a/api/tests/test_bulk_upload.py +++ b/api/tests/test_bulk_upload.py @@ -71,9 +71,10 @@ def test_gar_bulk_upload_valid_input( } ) - with fieldwork_file_path.open("rb") as fp_fieldwork, lab_file_path.open( - "rb" - ) as fp_lab: + with ( + fieldwork_file_path.open("rb") as fp_fieldwork, + lab_file_path.open("rb") as fp_lab, + ): data = { "bulk_upload_type": "GAR", "project_number": 1, diff --git a/api/views.py b/api/views.py index 2077c35..7ec9666 100644 --- a/api/views.py +++ b/api/views.py @@ -476,7 +476,7 @@ class BulkUploadViewSet(mixins.UserOrganizationMixin, viewsets.ModelViewSet): This endpoint interfaces with the BulkUpload model and supports the following POST parameters: `bulk_upload_type`: - str (*required*): Options: ['GAR', 'GLD] + str (*required*): Options: ['GAR', 'GLD'] `metadata`: json (*optional*): Open json field that can be filled in with information that cannot be provided through the upload files diff --git a/manage.py b/manage.py index d8e580b..e9d06cc 100644 --- a/manage.py +++ b/manage.py @@ -1,5 +1,6 @@ #!/usr/bin/env python """Django's command-line utility for administrative tasks.""" + import os import sys diff --git a/upload_examples.ipynb b/upload_examples.ipynb index 75d43ef..9da8082 100644 --- a/upload_examples.ipynb +++ b/upload_examples.ipynb @@ -22,25 +22,25 @@ "url = \"http://brostar.nl/api/uploadtasks/\"\n", "\n", "metadata = {\n", - " \"requestReference\":\"test\",\n", - " \"deliveryAccountableParty\":\"12345678\",\n", - " \"qualityRegime\":\"IMBRO\"\n", - " }\n", + " \"requestReference\": \"test\",\n", + " \"deliveryAccountableParty\": \"12345678\",\n", + " \"qualityRegime\": \"IMBRO\",\n", + "}\n", "\n", "sourcedocument_data = {\n", - " \"objectIdAccountableParty\":\"test\",\n", - " \"name\":\"test\",\n", - " \"deliveryContext\":\"kaderrichtlijnWater\",\n", - " \"monitoringPurpose\":\"strategischBeheerKwaliteitRegionaal\",\n", - " \"groundwaterAspect\":\"kwantiteit\",\n", - " \"startDateMonitoring\":\"2024-01-01\",\n", - " \"measuringPoints\":[\n", + " \"objectIdAccountableParty\": \"test\",\n", + " \"name\": \"test\",\n", + " \"deliveryContext\": \"kaderrichtlijnWater\",\n", + " \"monitoringPurpose\": \"strategischBeheerKwaliteitRegionaal\",\n", + " \"groundwaterAspect\": \"kwantiteit\",\n", + " \"startDateMonitoring\": \"2024-01-01\",\n", + " \"measuringPoints\": [\n", " {\n", - " \"measuringPointCode\":\"PUT00001\",\n", - " \"broId\":\"GMW000000000001\",\n", - " \"tubeNumber\":\"1\"\n", + " \"measuringPointCode\": \"PUT00001\",\n", + " \"broId\": \"GMW000000000001\",\n", + " \"tubeNumber\": \"1\",\n", " }\n", - " ]\n", + " ],\n", "}\n", "\n", "payload = {\n", @@ -49,10 +49,10 @@ " \"registration_type\": \"GMN_StartRegistration\",\n", " \"request_type\": \"registration\",\n", " \"metadata\": metadata,\n", - " \"sourcedocument_data\": sourcedocument_data\n", + " \"sourcedocument_data\": sourcedocument_data,\n", "}\n", "\n", - "r = requests.post(url, json=payload)\n" + "r = requests.post(url, json=payload)" ] }, { @@ -118,10 +118,10 @@ "outputs": [], "source": [ "metadata = {\n", - " \"requestReference\":\"\",\n", - " \"deliveryAccountableParty\":\"\",\n", - " \"qualityRegime\":\"\",\n", - " \"underPrivilege\":\"\"\n", + " \"requestReference\": \"\",\n", + " \"deliveryAccountableParty\": \"\",\n", + " \"qualityRegime\": \"\",\n", + " \"underPrivilege\": \"\",\n", "}\n", "\n", "sourcedocument_data = {\n", @@ -169,19 +169,19 @@ " \"electrodeNumber\": \"\",\n", " \"electrodePackingMaterial\": \"\",\n", " \"electrodeStatus\": \"\",\n", - " \"electrodePosition\": \"\"\n", + " \"electrodePosition\": \"\",\n", " },\n", " {\n", " \"electrodeNumber\": \"\",\n", " \"electrodePackingMaterial\": \"\",\n", " \"electrodeStatus\": \"\",\n", - " \"electrodePosition\": \"\"\n", - " }\n", - " ]\n", + " \"electrodePosition\": \"\",\n", + " },\n", + " ],\n", " }\n", - " ]\n", + " ],\n", " }\n", - " ]\n", + " ],\n", "}" ] }, @@ -204,19 +204,15 @@ "#########################\n", "\n", "# Registration requests:\n", - "metadata = {\n", - " \"requestReference\":\"\",\n", - " \"deliveryAccountableParty\":\"\",\n", - " \"qualityRegime\":\"\"\n", - "}\n", + "metadata = {\"requestReference\": \"\", \"deliveryAccountableParty\": \"\", \"qualityRegime\": \"\"}\n", "\n", "# All other request types:\n", "metadata = {\n", - " \"requestReference\":\"\",\n", - " \"deliveryAccountableParty\":\"\",\n", - " \"broId\":\"\",\n", - " \"qualityRegime\":\"\",\n", - " \"correctionReason\":\"\"\n", + " \"requestReference\": \"\",\n", + " \"deliveryAccountableParty\": \"\",\n", + " \"broId\": \"\",\n", + " \"qualityRegime\": \"\",\n", + " \"correctionReason\": \"\",\n", "}\n", "\n", "#########################\n", @@ -226,47 +222,36 @@ "# GMN_StartRegistration:\n", "\n", "sourcedocument_data = {\n", - " \"objectIdAccountableParty\":\"\",\n", - " \"name\":\"\",\n", - " \"deliveryContext\":\"\",\n", - " \"monitoringPurpose\":\"\",\n", - " \"groundwaterAspect\":\"\",\n", - " \"startDateMonitoring\":\"\",\n", - " \"measuringPoints\":[\n", - " {\n", - " \"measuringPointCode\":\"\",\n", - " \"broId\":\"\",\n", - " \"tubeNumber\":\"\"\n", - " }\n", - " ]\n", + " \"objectIdAccountableParty\": \"\",\n", + " \"name\": \"\",\n", + " \"deliveryContext\": \"\",\n", + " \"monitoringPurpose\": \"\",\n", + " \"groundwaterAspect\": \"\",\n", + " \"startDateMonitoring\": \"\",\n", + " \"measuringPoints\": [{\"measuringPointCode\": \"\", \"broId\": \"\", \"tubeNumber\": \"\"}],\n", "}\n", "\n", "# GMN_MeasuringPoint\n", "sourcedocument_data = {\n", - " \"eventDate\":\"\",\n", - " \"measuringPointCode\":\"\",\n", - " \"broId\":\"\",\n", - " \"tubeNumber\":\"\"\n", + " \"eventDate\": \"\",\n", + " \"measuringPointCode\": \"\",\n", + " \"broId\": \"\",\n", + " \"tubeNumber\": \"\",\n", "}\n", "\n", "# GMN_TubeReference\n", "sourcedocument_data = {\n", - " \"eventDate\":\"\",\n", - " \"measuringPointCode\":\"\",\n", - " \"broId\":\"\",\n", - " \"tubeNumber\":\"\"\n", + " \"eventDate\": \"\",\n", + " \"measuringPointCode\": \"\",\n", + " \"broId\": \"\",\n", + " \"tubeNumber\": \"\",\n", "}\n", "\n", "# GMN_MeasuringPointEndDate\n", - "sourcedocument_data = {\n", - " \"eventDate\":\"\",\n", - " \"measuringPointCode\":\"\"\n", - "}\n", + "sourcedocument_data = {\"eventDate\": \"\", \"measuringPointCode\": \"\"}\n", "\n", "# GMN_Closure\n", - "sourcedocument_data = {\n", - " \"endDateMonitoring\":\"\"\n", - "}" + "sourcedocument_data = {\"endDateMonitoring\": \"\"}" ] } ],