diff --git a/.github/workflows/deploy_dotnet.yml b/.github/workflows/deploy_dotnet.yml index 194dcc91..86f2d1fc 100644 --- a/.github/workflows/deploy_dotnet.yml +++ b/.github/workflows/deploy_dotnet.yml @@ -114,3 +114,9 @@ jobs: description: "${{ contains(needs.deploy.result, 'success') && 'Deployed:' || 'Deployment failed:' }} ${{ github.event.head_commit.message }}" color: ${{ contains(needs.deploy.result, 'success') && 65280 || 16711680 }} secrets: inherit + + update_birds: + needs: [deploy] + name: ๐Ÿฆš Update production birds + uses: ./.github/workflows/update_birds.yml + secrets: inherit diff --git a/.github/workflows/update_birds.yml b/.github/workflows/update_birds.yml new file mode 100644 index 00000000..bf5a0ad8 --- /dev/null +++ b/.github/workflows/update_birds.yml @@ -0,0 +1,45 @@ +name: ๐Ÿฆœ Update Bird data + +on: + schedule: + - cron: "0 8 * * *" + workflow_dispatch: + workflow_call: + +jobs: + update_data: + name: ๐Ÿฆœ Get bird data + outputs: + artifact_url: ${{ steps.artifact-upload-step.outputs.artifact-url}} + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v4 + + - name: ๐Ÿชฝ Run auto import script + if: ${{ !env.ACT }} + run: ./scripts/bash/auto_import.sh ./scripts/bash/provinces.txt ${{ secrets.DATA_URL }} + + - name: ๐Ÿงช Test auto update script + if: ${{ env.ACT }} + run: ./scripts/bash/auto_import.sh ./scripts/bash/provinces.txt ${{ secrets.DATA_URL }} ./res/species_list/south_africa.csv + + - uses: actions/upload-artifact@v4 + id: artifact-upload-step + with: + name: bird-data + path: | + *.csv + + notify_discord: + name: ๐Ÿ”” Send Discord notification about deployment + needs: [update_data] + if: ${{ !cancelled() && (success() || failure()) }} + uses: ./.github/workflows/discord.yml + with: + content: "${{ contains(needs.update_data.result, 'success') && 'Successfully updated bird data' || 'Error during update of' }} ${{ github.ref_name }} for bird data" + title: "${{ contains(needs.update_data.result, 'success') && 'Successfully updated bird data' || 'Error during update of' }} ${{ github.ref_name }} for bird data" + url: ${{ needs.update_data.outputs.artifact_url }} + description: "${{ contains(needs.update_data.result, 'success') && 'Updated:' || 'Update failed:' }} ${{ github.event.head_commit.message }}" + color: ${{ contains(needs.update_data.result, 'success') && 65280 || 16711680 }} + secrets: inherit diff --git a/dotnet/BirdApi/BeakPeekApi.Tests/Controllers/ImportControllerTest.cs b/dotnet/BirdApi/BeakPeekApi.Tests/Controllers/ImportControllerTest.cs index 2d28739c..f579cae6 100644 --- a/dotnet/BirdApi/BeakPeekApi.Tests/Controllers/ImportControllerTest.cs +++ b/dotnet/BirdApi/BeakPeekApi.Tests/Controllers/ImportControllerTest.cs @@ -25,7 +25,7 @@ public async Task ImportData_ShouldReturnOk() var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(fileContent)); var formFile = new FormFile(stream, 0, stream.Length, "file", fileName); - _mockCsvImporter.Setup(m => m.ImportCsvData(It.IsAny(), It.IsAny())); + _mockCsvImporter.Setup(m => m.ImportCsvData(It.IsAny(), It.IsAny())).Returns(Task.Delay(100)); // Act var result = await _controller.ImportData(formFile, "TestProvince"); diff --git a/dotnet/BirdApi/BeakPeekApi/Controllers/ImportController.cs b/dotnet/BirdApi/BeakPeekApi/Controllers/ImportController.cs index b13baba9..71f3a341 100644 --- a/dotnet/BirdApi/BeakPeekApi/Controllers/ImportController.cs +++ b/dotnet/BirdApi/BeakPeekApi/Controllers/ImportController.cs @@ -28,7 +28,7 @@ public async Task ImportData(IFormFile file, [FromQuery] string p await file.CopyToAsync(stream); } - _csvImporter.ImportCsvData(filePath, province); + await _csvImporter.ImportCsvData(filePath, province); return Ok(); } @@ -40,5 +40,24 @@ public IActionResult ImportAll([FromQuery] string path) return Ok(); } + + [HttpPost("importBirds")] + public async Task importBirds(IFormFile file) + { + if (file == null || file.Length == 0) + { + return BadRequest("File is empty"); + } + + var filePath = Path.GetTempFileName(); + using (var stream = new FileStream(filePath, FileMode.Create)) + { + await file.CopyToAsync(stream); + } + + _csvImporter.ImportBirds(filePath); + + return Ok(); + } } } diff --git a/dotnet/BirdApi/BeakPeekApi/Helpers/CsvImporter.cs b/dotnet/BirdApi/BeakPeekApi/Helpers/CsvImporter.cs index 5738a0f8..b540e8f5 100644 --- a/dotnet/BirdApi/BeakPeekApi/Helpers/CsvImporter.cs +++ b/dotnet/BirdApi/BeakPeekApi/Helpers/CsvImporter.cs @@ -57,7 +57,7 @@ public virtual void ImportBirds(string filepath) } } - public virtual void ImportCsvData(string filepath, string provinceName) where T : Province, new() + public virtual async Task ImportCsvData(string filepath, string provinceName) where T : Province, new() { var province = _context.ProvincesList.FirstOrDefault(p => p.Name == provinceName); @@ -104,8 +104,8 @@ public virtual void ImportBirds(string filepath) pentads.Add(pentad_allocation); - _context.Pentads.Add(new_pentad); - _context.SaveChanges(); + await _context.Pentads.AddAsync(new_pentad); + await _context.SaveChangesAsync(); } tmp_pentad = new_pentad; } @@ -120,9 +120,15 @@ public virtual void ImportBirds(string filepath) if (!does_bird_have_province) { - _context.Birds.Find(record.Spp)?.Bird_Provinces.Add(province); - _context.SaveChanges(); - birds_in_province.Add(record.Spp); + // await _context.Birds.FindAsync(record.Spp).Bird_Provinces.Add(province); + var found_bird = await _context.Birds.FindAsync(record.Spp); + if (found_bird != null) + { + found_bird.Bird_Provinces ??= new List { }; + found_bird.Bird_Provinces.Add(province); + await _context.SaveChangesAsync(); + birds_in_province.Add(record.Spp); + } } var bird_record = _context.Birds.Find(record.Spp); @@ -155,92 +161,92 @@ public virtual void ImportBirds(string filepath) { case "easterncape": // List Easterncape_list_to_add = (List)records_to_be_add.Cast().ToList(); - _context.Easterncape.AddRange((IEnumerable)records_to_be_add); + await _context.Easterncape.AddRangeAsync((IEnumerable)records_to_be_add); break; case "freestate": // List Freestate_list_to_add = (List)records_to_be_add.Cast().ToList(); - _context.Freestate.AddRange((IEnumerable)records_to_be_add); + await _context.Freestate.AddRangeAsync((IEnumerable)records_to_be_add); break; case "gauteng": // List Gauteng_list_to_add = (List)records_to_be_add.Cast(); - _context.Gauteng.AddRange((IEnumerable)records_to_be_add); + await _context.Gauteng.AddRangeAsync((IEnumerable)records_to_be_add); break; case "kwazulunatal": // List Kwazulunatal_list_to_add = (List)records_to_be_add.Cast(); - _context.Kwazulunatal.AddRange((IEnumerable)records_to_be_add); + await _context.Kwazulunatal.AddRangeAsync((IEnumerable)records_to_be_add); break; case "limpopo": // List Limpopo_list_to_add = (List)records_to_be_add.Cast(); - _context.Limpopo.AddRange((IEnumerable)records_to_be_add); + await _context.Limpopo.AddRangeAsync((IEnumerable)records_to_be_add); break; case "mpumalanga": // List Mpumalanga_list_to_add = (List)records_to_be_add.Cast(); - _context.Mpumalanga.AddRange((IEnumerable)records_to_be_add); + await _context.Mpumalanga.AddRangeAsync((IEnumerable)records_to_be_add); break; case "northerncape": // List Northerncape_list_to_add = (List)records_to_be_add.Cast(); - _context.Northerncape.AddRange((IEnumerable)records_to_be_add); + await _context.Northerncape.AddRangeAsync((IEnumerable)records_to_be_add); break; case "northwest": // List Northwest_list_to_add = (List)records_to_be_add.Cast(); - _context.Northwest.AddRange((IEnumerable)records_to_be_add); + await _context.Northwest.AddRangeAsync((IEnumerable)records_to_be_add); break; case "westerncape": // List Westerncape_list_to_add = (List)records_to_be_add.Cast(); - _context.Westerncape.AddRange((IEnumerable)records_to_be_add); + await _context.Westerncape.AddRangeAsync((IEnumerable)records_to_be_add); break; default: throw new Exception($"No province found that matches the province name given. {provinceName}"); } - _context.SaveChanges(); + await _context.SaveChangesAsync(); } } - public virtual void ImportCsvData(string filepath, string provinceName) + public virtual async Task ImportCsvData(string filepath, string provinceName) { switch (provinceName) { case "easterncape": - ImportCsvData(filepath, provinceName); + await ImportCsvData(filepath, provinceName); break; case "freestate": - ImportCsvData(filepath, provinceName); + await ImportCsvData(filepath, provinceName); break; case "gauteng": - ImportCsvData(filepath, provinceName); + await ImportCsvData(filepath, provinceName); break; case "kwazulunatal": - ImportCsvData(filepath, provinceName); + await ImportCsvData(filepath, provinceName); break; case "limpopo": - ImportCsvData(filepath, provinceName); + await ImportCsvData(filepath, provinceName); break; case "mpumalanga": - ImportCsvData(filepath, provinceName); + await ImportCsvData(filepath, provinceName); break; case "northerncape": - ImportCsvData(filepath, provinceName); + await ImportCsvData(filepath, provinceName); break; case "northwest": - ImportCsvData(filepath, provinceName); + await ImportCsvData(filepath, provinceName); break; case "westerncape": - ImportCsvData(filepath, provinceName); + await ImportCsvData(filepath, provinceName); break; default: throw new Exception("No province found that matches the province name given."); } } - public void ImportAllCsvData(string directoryPath) + public async Task ImportAllCsvData(string directoryPath) { var csvFiles = Directory.GetFiles(directoryPath, "*.csv"); foreach (var csvFile in csvFiles) { var province = Path.GetFileNameWithoutExtension(csvFile); - ImportCsvData(csvFile, province); + await ImportCsvData(csvFile, province); } } } diff --git a/dotnet/BirdApi/BeakPeekApi/appsettings.Development.json b/dotnet/BirdApi/BeakPeekApi/appsettings.Development.json index 22ba0813..ff97075f 100644 --- a/dotnet/BirdApi/BeakPeekApi/appsettings.Development.json +++ b/dotnet/BirdApi/BeakPeekApi/appsettings.Development.json @@ -1,6 +1,6 @@ { "ConnectionStrings": { - "DefaultConnection": "Server=db;Database=BeakPeek;User=sa;Password=Your_password123;TrustServerCertificate=true;", + "DefaultConnection": "Server=localhost,1433;Database=BirdDB;User=sa;Password=Your_password123;TrustServerCertificate=true;", "AZURE_SQL_CONNECTIONSTRING": "Server=tcp:beakpeek.database.windows.net,1433;Initial Catalog=BeakPeekDB;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;Authentication=\"Active Directory Default\";" }, "Logging": { diff --git a/scripts/bash/auto_import.sh b/scripts/bash/auto_import.sh new file mode 100755 index 00000000..038a4837 --- /dev/null +++ b/scripts/bash/auto_import.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +if [[ -z "$1" ]]; then + echo "No proince list" + exit 1 +else + PROVINCE_LIST="$1" + echo "Given province list" +fi + +if [[ -z "$2" ]]; then + URL="http://localhost:5050" + echo "Using default Url" +else + URL="$2" + echo "Using given URL" +fi + +BIRDS=birds.csv +if [[ -z "$3" ]]; then + echo "Getting birds" + curl -L -o birds.csv "https://api.birdmap.africa/sabap2/v2/coverage/country/southafrica/species?format=csv" +else + IS_TEST=true + BIRDS=$3 + echo "Test mode" +fi + +GOOD_BIRDS=$(awk -F, 'BEGIN {FS=","} {if ($6 > 0.01 ) print $0}' $BIRDS) +echo "$GOOD_BIRDS" > good_birds.csv + +curl -X 'POST' \ + "$URL/api/Import/importBirds" \ + -H 'accept: */*' \ + -H 'Content-Type: multipart/form-data' \ + -F 'file=@good_birds.csv;type=text/csv' + +base_url="https://api.birdmap.africa/sabap2/v2/monthly/speciesbypentad/province/" + +set -o pipefail +while IFS= read -r line +do + FULL_URL="${base_url}${line}?period=&dates=&format=csv" + + echo "Downloading ${line} CSV" + + if curl -L "$FULL_URL" -o ${line}.csv ; then + if [[ IS_TEST ]]; then + awk '{FS=","} {print } NR==10{exit}' ${line}.csv > ${line}_short.csv + cat ${line}_short.csv + fi + curl -X 'POST' \ + "$URL/api/Import/import?province=${line}" \ + -H 'accept: */*' \ + -H 'Content-Type: multipart/form-data' \ + -F "file=@${line}_short.csv;type=text/csv" + fi +done < "$PROVINCE_LIST" diff --git a/scripts/bash/download_regions.sh b/scripts/bash/download_regions.sh index 0ccc067a..a8681c02 100755 --- a/scripts/bash/download_regions.sh +++ b/scripts/bash/download_regions.sh @@ -3,7 +3,8 @@ # curl -L -o gauteng.csv https://api.birdmap.africa/sabap2/v2/monthly/speciesbypentad/province/gauteng\?period\=\&dates\=\&format\=csv # curl -L -o easterncape.csv -province_list="../../scripts/bash/provinces.txt" +# province_list="../../scripts/bash/provinces.txt" +province_list="provinces.txt" base_url="https://api.birdmap.africa/sabap2/v2/monthly/speciesbypentad/province/"