Merge pull request #189 from NavAbility/23Q1/list_blob_data

Project.toml

@@ -3,7 +3,7 @@ uuid = "f3e6a059-199c-4ada-8143-fcefb97e6165"
 keywords = ["navability", "navigation", "slam", "sdk", "robotics", "robots"]
 desc = "NavAbility SDK: Access NavAbility Cloud factor graph features. Note that this SDK and the related API are still in development. Please let us know if you have any issues at [email protected]."
 authors = ["NavAbility <[email protected]>"]
-version = "0.4.9"
+version = "0.4.10"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
@@ -21,7 +21,7 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 Diana = "0.2, 0.3"
 DocStringExtensions = "0.8, 0.9"
 Downloads = "1"
-HTTP = "0.9, 1"
+HTTP = "^1.7"
 JSON = "0.21"
 TensorCast = "0.4"
 julia = "1.6"

src/navability/graphql/DataBlobs.jl

@@ -95,4 +95,14 @@ query sdk_listdataentries(\$userId: ID!, \$robotId: ID!, \$sessionId: ID!, \$var
     }
   }
 }
+"""
+
+GQL_LISTDATABLOBS = """
+query sdk_listdatablobs {
+  files {
+    id
+    filename
+    filesize
+  }
+}
 """

src/navability/services/DataBlobs.jl

@@ -202,8 +202,9 @@ function addDataEvent(
   #
   io = IOBuffer(blob)
 
-  filesize = io.size
-  np = 1
+  filesize = length(blob)
+  # TODO: Use about a 50M file part here.
+  np = 1 # TODO: ceil(filesize / 50e6)
   # create the upload url destination
   d = NVA.createUploadEvent(client, blobname, filesize, np)
 
@@ -224,9 +225,7 @@ function addDataEvent(
   ]
   #
 
-  # fid = open(filepath,"r")
-  resp = HTTP.put(url, headers, io)
-  # close(fid)
+  resp = HTTP.put(url, headers, blob)
 
   # Extract eTag
   eTag = match(r"[a-zA-Z0-9]+",resp["eTag"]).match
@@ -343,6 +342,41 @@ listDataEntriesEvent(client::NavAbilityClient,
 
 listDataEntries(w...) = @async listDataEntriesEvent(w...)
 
+## 
+
+function listDataBlobsEvent(
+  navAbilityClient::NavAbilityClient, 
+)
+  #
+  response = navAbilityClient.query(QueryOptions(
+    "sdk_listdatablobs",
+    GQL_LISTDATABLOBS,
+    Dict()
+  )) |> fetch
+  rootData = JSON.parse(response.Data)
+  if haskey(rootData, "errors")
+    throw("Error: $(rootData["errors"])")
+  end
+  data = get(rootData,"data",nothing)
+  if data === nothing return "Error" end
+
+  listdata = data["files"]
+  ret = []
+  for d in listdata
+    tupk = Tuple(Symbol.(keys(d)))
+    nt = NamedTuple{tupk}( values(d) )
+    push!(ret,
+      nt
+    )
+  end
+
+  return ret
+end
+
+#
+
+listDataBlobs(w...) = @async listDataBlobsEvent(w...)
+
 
 
 """

test/load/blobload.jl

@@ -0,0 +1,73 @@
+## 
+## On-demand load testing script for range of blob upload/download scenarios 
+##
+using Test
+using Random
+using NavAbilitySDK
+using Base64
+using JSON3
+
+using TimerOutputs
+
+apiUrl = get(ENV,"API_URL","https://api.navability.io")
+userId = get(ENV,"USER_ID","[email protected]")
+robotId = get(ENV,"ROBOT_ID","IntegrationRobot")
+sessionId = get(ENV,"SESSION_ID","TestSession"*randstring(7))
+
+include("../integration/fixtures.jl")
+# @testset "Blob load test" begin
+client = NavAbilityHttpsClient(apiUrl)
+context = Client(userId,robotId,sessionId)
+  # client, context = createClients(apiUrl, userId, robotId, sessionId)
+
+to = TimerOutput()
+
+## 
+data1M = JSON3.write(rand(Float64, 512*1024))[1:1024*1024];
+@info "1Mb files"
+for i in 1:20
+  @info "Iteration $(i)"
+  fileId = @timeit to "[Serial] Add 1Mb Data" NVA.addData(client, "LoadTest", codeunits(data1M)) |> fetch
+  data1MRet = @timeit to "[Serial] Get 1Mb Data" NVA.getData(client, context, fileId) |> fetch |> take! 
+  # @timeit to "List files" NVA.li
+  @test codeunits(data1M) == data1MRet
+  @info "Validated data, file ID $(fileId)"
+end
+
+data10M = JSON3.write(rand(Float64, 512*1024*10))[1:10*1024*1024];
+@info "10Mb files"
+for i in 1:20
+  @info "Iteration $(i)"
+  fileId = @timeit to "[Serial] Add 10Mb Data" NVA.addData(client, "LoadTest", codeunits(data10M)) |> fetch
+  data10MRet = @timeit to "[Serial] Get 10Mb Data" NVA.getData(client, context, fileId) |> fetch |> take! 
+  # @timeit to "List files" NVA.li
+  @test codeunits(data10M) == data10MRet
+  @info "Validated data, file ID $(fileId)"
+end
+
+data100M = JSON3.write(rand(Float64, 512*1024*100))[1:1024*1024*100];
+@info "100Mb files"
+for i in 1:10
+  @info "Iteration $(i)"
+  fileId = @timeit to "[Serial] Add 100Mb Data" NVA.addData(client, "LoadTest", codeunits(data100M)) |> fetch
+  data100MRet = @timeit to "[Serial] Get 100Mb Data" NVA.getData(client, context, fileId) |> fetch |> take! 
+  # @timeit to "List files" NVA.li
+  @test codeunits(data100M) == data100MRet
+  @info "Validated data, file ID $(fileId)"
+end
+
+for i in 1:20
+  @timeit to "List Blobs" NVA.listDataBlobs(client) |> fetch
+end
+
+## Parallel file uploads
+parallelSet = 1:20
+fileIds = @timeit to "[Parallel] Add 1Mb Data [20 files total]" (
+    map(r -> (NVA.addData(client, "LoadTest", codeunits(data1M))), parallelSet) .|> fetch)
+data1MRets = @timeit to "[Parallel] Get 1Mb Data [20 files total]" (
+    map(fileId -> NVA.getData(client, context, fileId), fileIds) .|> fetch .|> take!);
+@test all(map(dRet -> codeunits(data1M) == dRet, data1MRets))
+
+to
+
+# end