🔀 Merge pull request #370 from COS301-SE-2024/dev/feat/update_birds

✨ Added automations needed to import birds from SABAB

.github/workflows/deploy_dotnet.yml

@@ -114,3 +114,9 @@ jobs:
       description: "${{ contains(needs.deploy.result, 'success') && 'Deployed:' || 'Deployment failed:' }} ${{ github.event.head_commit.message }}"
       color: ${{ contains(needs.deploy.result, 'success') && 65280 || 16711680 }}
     secrets: inherit
+
+  update_birds:
+    needs: [deploy]
+    name: 🦚 Update production birds
+    uses: ./.github/workflows/update_birds.yml
+    secrets: inherit

.github/workflows/update_birds.yml

@@ -0,0 +1,45 @@
+name: 🦜 Update Bird data
+
+on:
+  schedule:
+    - cron: "0 8 * * *"
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  update_data:
+    name: 🦜 Get bird data
+    outputs:
+      artifact_url: ${{ steps.artifact-upload-step.outputs.artifact-url}}
+    runs-on: ubuntu-latest
+    steps:
+
+    - uses: actions/checkout@v4
+
+    - name: 🪽 Run auto import script
+      if: ${{ !env.ACT }}
+      run: ./scripts/bash/auto_import.sh ./scripts/bash/provinces.txt ${{ secrets.DATA_URL }}
+
+    - name: 🧪 Test auto update script
+      if: ${{ env.ACT }}
+      run: ./scripts/bash/auto_import.sh ./scripts/bash/provinces.txt ${{ secrets.DATA_URL }} ./res/species_list/south_africa.csv
+
+    - uses: actions/upload-artifact@v4
+      id: artifact-upload-step
+      with:
+        name: bird-data
+        path: |
+            *.csv
+
+  notify_discord:
+    name: 🔔 Send Discord notification about deployment
+    needs: [update_data]
+    if: ${{ !cancelled() && (success() || failure()) }}
+    uses: ./.github/workflows/discord.yml
+    with:
+      content: "${{ contains(needs.update_data.result, 'success') && 'Successfully updated bird data' || 'Error during update of' }} ${{ github.ref_name }} for bird data"
+      title: "${{ contains(needs.update_data.result, 'success') && 'Successfully updated bird data' || 'Error during update of' }} ${{ github.ref_name }} for bird data"
+      url: ${{ needs.update_data.outputs.artifact_url }}
+      description: "${{ contains(needs.update_data.result, 'success') && 'Updated:' || 'Update failed:' }} ${{ github.event.head_commit.message }}"
+      color: ${{ contains(needs.update_data.result, 'success') && 65280 || 16711680 }}
+    secrets: inherit

dotnet/BirdApi/BeakPeekApi.Tests/Controllers/ImportControllerTest.cs

@@ -25,7 +25,7 @@ public async Task ImportData_ShouldReturnOk()
         var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(fileContent));
         var formFile = new FormFile(stream, 0, stream.Length, "file", fileName);
 
-        _mockCsvImporter.Setup(m => m.ImportCsvData(It.IsAny<string>(), It.IsAny<string>()));
+        _mockCsvImporter.Setup(m => m.ImportCsvData(It.IsAny<string>(), It.IsAny<string>())).Returns(Task.Delay(100));
 
         // Act
         var result = await _controller.ImportData(formFile, "TestProvince");

dotnet/BirdApi/BeakPeekApi/Controllers/ImportController.cs

@@ -28,7 +28,7 @@ public async Task<IActionResult> ImportData(IFormFile file, [FromQuery] string p
                 await file.CopyToAsync(stream);
             }
 
-            _csvImporter.ImportCsvData(filePath, province);
+            await _csvImporter.ImportCsvData(filePath, province);
 
             return Ok();
         }
@@ -40,5 +40,24 @@ public IActionResult ImportAll([FromQuery] string path)
 
             return Ok();
         }
+
+        [HttpPost("importBirds")]
+        public async Task<IActionResult> importBirds(IFormFile file)
+        {
+            if (file == null || file.Length == 0)
+            {
+                return BadRequest("File is empty");
+            }
+
+            var filePath = Path.GetTempFileName();
+            using (var stream = new FileStream(filePath, FileMode.Create))
+            {
+                await file.CopyToAsync(stream);
+            }
+
+            _csvImporter.ImportBirds(filePath);
+
+            return Ok();
+        }
     }
 }

dotnet/BirdApi/BeakPeekApi/Helpers/CsvImporter.cs

@@ -57,7 +57,7 @@ public virtual void ImportBirds(string filepath)
             }
         }
 
-        public virtual void ImportCsvData<T>(string filepath, string provinceName) where T : Province, new()
+        public virtual async Task ImportCsvData<T>(string filepath, string provinceName) where T : Province, new()
         {
             var province = _context.ProvincesList.FirstOrDefault(p => p.Name == provinceName);
 
@@ -104,8 +104,8 @@ public virtual void ImportBirds(string filepath)
 
                             pentads.Add(pentad_allocation);
 
-                            _context.Pentads.Add(new_pentad);
-                            _context.SaveChanges();
+                            await _context.Pentads.AddAsync(new_pentad);
+                            await _context.SaveChangesAsync();
                         }
                         tmp_pentad = new_pentad;
                     }
@@ -120,9 +120,15 @@ public virtual void ImportBirds(string filepath)
 
                     if (!does_bird_have_province)
                     {
-                        _context.Birds.Find(record.Spp)?.Bird_Provinces.Add(province);
-                        _context.SaveChanges();
-                        birds_in_province.Add(record.Spp);
+                        // await _context.Birds.FindAsync(record.Spp).Bird_Provinces.Add(province);
+                        var found_bird = await _context.Birds.FindAsync(record.Spp);
+                        if (found_bird != null)
+                        {
+                            found_bird.Bird_Provinces ??= new List<ProvinceList> { };
+                            found_bird.Bird_Provinces.Add(province);
+                            await _context.SaveChangesAsync();
+                            birds_in_province.Add(record.Spp);
+                        }
                     }
 
                     var bird_record = _context.Birds.Find(record.Spp);
@@ -155,92 +161,92 @@ public virtual void ImportBirds(string filepath)
                 {
                     case "easterncape":
                         // List<Easterncape> Easterncape_list_to_add = (List<Easterncape>)records_to_be_add.Cast<Easterncape>().ToList();
-                        _context.Easterncape.AddRange((IEnumerable<Easterncape>)records_to_be_add);
+                        await _context.Easterncape.AddRangeAsync((IEnumerable<Easterncape>)records_to_be_add);
                         break;
                     case "freestate":
                         // List<Freestate> Freestate_list_to_add = (List<Freestate>)records_to_be_add.Cast<Freestate>().ToList();
-                        _context.Freestate.AddRange((IEnumerable<Freestate>)records_to_be_add);
+                        await _context.Freestate.AddRangeAsync((IEnumerable<Freestate>)records_to_be_add);
                         break;
                     case "gauteng":
                         // List<Gauteng> Gauteng_list_to_add = (List<Gauteng>)records_to_be_add.Cast<Gauteng>();
-                        _context.Gauteng.AddRange((IEnumerable<Gauteng>)records_to_be_add);
+                        await _context.Gauteng.AddRangeAsync((IEnumerable<Gauteng>)records_to_be_add);
                         break;
                     case "kwazulunatal":
                         // List<Kwazulunatal> Kwazulunatal_list_to_add = (List<Kwazulunatal>)records_to_be_add.Cast<Kwazulunatal>();
-                        _context.Kwazulunatal.AddRange((IEnumerable<Kwazulunatal>)records_to_be_add);
+                        await _context.Kwazulunatal.AddRangeAsync((IEnumerable<Kwazulunatal>)records_to_be_add);
                         break;
                     case "limpopo":
                         // List<Limpopo> Limpopo_list_to_add = (List<Limpopo>)records_to_be_add.Cast<Limpopo>();
-                        _context.Limpopo.AddRange((IEnumerable<Limpopo>)records_to_be_add);
+                        await _context.Limpopo.AddRangeAsync((IEnumerable<Limpopo>)records_to_be_add);
                         break;
                     case "mpumalanga":
                         // List<Mpumalanga> Mpumalanga_list_to_add = (List<Mpumalanga>)records_to_be_add.Cast<Mpumalanga>();
-                        _context.Mpumalanga.AddRange((IEnumerable<Mpumalanga>)records_to_be_add);
+                        await _context.Mpumalanga.AddRangeAsync((IEnumerable<Mpumalanga>)records_to_be_add);
                         break;
                     case "northerncape":
                         // List<Northerncape> Northerncape_list_to_add = (List<Northerncape>)records_to_be_add.Cast<Northerncape>();
-                        _context.Northerncape.AddRange((IEnumerable<Northerncape>)records_to_be_add);
+                        await _context.Northerncape.AddRangeAsync((IEnumerable<Northerncape>)records_to_be_add);
                         break;
                     case "northwest":
                         // List<Northwest> Northwest_list_to_add = (List<Northwest>)records_to_be_add.Cast<Northwest>();
-                        _context.Northwest.AddRange((IEnumerable<Northwest>)records_to_be_add);
+                        await _context.Northwest.AddRangeAsync((IEnumerable<Northwest>)records_to_be_add);
                         break;
                     case "westerncape":
                         // List<Westerncape> Westerncape_list_to_add = (List<Westerncape>)records_to_be_add.Cast<Westerncape>();
-                        _context.Westerncape.AddRange((IEnumerable<Westerncape>)records_to_be_add);
+                        await _context.Westerncape.AddRangeAsync((IEnumerable<Westerncape>)records_to_be_add);
                         break;
                     default:
                         throw new Exception($"No province found that matches the province name given. {provinceName}");
                 }
 
-                _context.SaveChanges();
+                await _context.SaveChangesAsync();
             }
         }
 
-        public virtual void ImportCsvData(string filepath, string provinceName)
+        public virtual async Task ImportCsvData(string filepath, string provinceName)
         {
 
             switch (provinceName)
             {
                 case "easterncape":
-                    ImportCsvData<Easterncape>(filepath, provinceName);
+                    await ImportCsvData<Easterncape>(filepath, provinceName);
                     break;
                 case "freestate":
-                    ImportCsvData<Freestate>(filepath, provinceName);
+                    await ImportCsvData<Freestate>(filepath, provinceName);
                     break;
                 case "gauteng":
-                    ImportCsvData<Gauteng>(filepath, provinceName);
+                    await ImportCsvData<Gauteng>(filepath, provinceName);
                     break;
                 case "kwazulunatal":
-                    ImportCsvData<Kwazulunatal>(filepath, provinceName);
+                    await ImportCsvData<Kwazulunatal>(filepath, provinceName);
                     break;
                 case "limpopo":
-                    ImportCsvData<Limpopo>(filepath, provinceName);
+                    await ImportCsvData<Limpopo>(filepath, provinceName);
                     break;
                 case "mpumalanga":
-                    ImportCsvData<Mpumalanga>(filepath, provinceName);
+                    await ImportCsvData<Mpumalanga>(filepath, provinceName);
                     break;
                 case "northerncape":
-                    ImportCsvData<Northerncape>(filepath, provinceName);
+                    await ImportCsvData<Northerncape>(filepath, provinceName);
                     break;
                 case "northwest":
-                    ImportCsvData<Northwest>(filepath, provinceName);
+                    await ImportCsvData<Northwest>(filepath, provinceName);
                     break;
                 case "westerncape":
-                    ImportCsvData<Westerncape>(filepath, provinceName);
+                    await ImportCsvData<Westerncape>(filepath, provinceName);
                     break;
                 default:
                     throw new Exception("No province found that matches the province name given.");
             }
         }
 
-        public void ImportAllCsvData(string directoryPath)
+        public async Task ImportAllCsvData(string directoryPath)
         {
             var csvFiles = Directory.GetFiles(directoryPath, "*.csv");
             foreach (var csvFile in csvFiles)
             {
                 var province = Path.GetFileNameWithoutExtension(csvFile);
-                ImportCsvData(csvFile, province);
+                await ImportCsvData(csvFile, province);
             }
         }
     }

dotnet/BirdApi/BeakPeekApi/appsettings.Development.json

@@ -1,6 +1,6 @@
 {
   "ConnectionStrings": {
-    "DefaultConnection": "Server=db;Database=BeakPeek;User=sa;Password=Your_password123;TrustServerCertificate=true;",
+    "DefaultConnection": "Server=localhost,1433;Database=BirdDB;User=sa;Password=Your_password123;TrustServerCertificate=true;",
     "AZURE_SQL_CONNECTIONSTRING": "Server=tcp:beakpeek.database.windows.net,1433;Initial Catalog=BeakPeekDB;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;Authentication=\"Active Directory Default\";"
   },
   "Logging": {

scripts/bash/auto_import.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+if [[ -z "$1" ]]; then
+    echo "No proince list"
+    exit 1
+else
+    PROVINCE_LIST="$1"
+    echo "Given province list"
+fi
+
+if [[ -z "$2" ]]; then
+    URL="http://localhost:5050"
+    echo "Using default Url"
+else
+    URL="$2"
+    echo "Using given URL"
+fi
+
+BIRDS=birds.csv
+if [[ -z "$3" ]]; then
+    echo "Getting birds"
+    curl -L -o birds.csv "https://api.birdmap.africa/sabap2/v2/coverage/country/southafrica/species?format=csv"
+else
+    IS_TEST=true
+    BIRDS=$3
+    echo "Test mode"
+fi
+
+GOOD_BIRDS=$(awk -F, 'BEGIN {FS=","} {if ($6 > 0.01 ) print $0}' $BIRDS)
+echo "$GOOD_BIRDS" > good_birds.csv
+
+curl -X 'POST' \
+  "$URL/api/Import/importBirds" \
+  -H 'accept: */*' \
+  -H 'Content-Type: multipart/form-data' \
+  -F 'file=@good_birds.csv;type=text/csv'
+
+base_url="https://api.birdmap.africa/sabap2/v2/monthly/speciesbypentad/province/"
+
+set -o pipefail
+while IFS= read -r line
+do
+    FULL_URL="${base_url}${line}?period=&dates=&format=csv"
+
+    echo "Downloading ${line} CSV"
+
+    if curl -L "$FULL_URL" -o ${line}.csv  ; then
+    if [[ IS_TEST ]]; then
+        awk '{FS=","} {print } NR==10{exit}' ${line}.csv > ${line}_short.csv
+        cat ${line}_short.csv
+    fi
+    curl -X 'POST' \
+        "$URL/api/Import/import?province=${line}" \
+        -H 'accept: */*' \
+        -H 'Content-Type: multipart/form-data' \
+        -F "file=@${line}_short.csv;type=text/csv"
+    fi
+done < "$PROVINCE_LIST"

scripts/bash/download_regions.sh

@@ -3,7 +3,8 @@
 # curl -L -o gauteng.csv  https://api.birdmap.africa/sabap2/v2/monthly/speciesbypentad/province/gauteng\?period\=\&dates\=\&format\=csv
 # curl -L -o easterncape.csv 
 
-province_list="../../scripts/bash/provinces.txt"
+# province_list="../../scripts/bash/provinces.txt"
+province_list="provinces.txt"
 
 base_url="https://api.birdmap.africa/sabap2/v2/monthly/speciesbypentad/province/"