From 2f7ccb2ef85fd70b08b25f5fdc4f9671db193c62 Mon Sep 17 00:00:00 2001 From: cybermaggedon Date: Wed, 27 Nov 2024 17:24:27 +0000 Subject: [PATCH] - Reduce back-pressure on tg-load-kg-core (#179) - Save entity in correct format in tg-save-core --- trustgraph-cli/scripts/tg-load-kg-core | 6 ++++-- trustgraph-flow/trustgraph/api/gateway/service.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/trustgraph-cli/scripts/tg-load-kg-core b/trustgraph-cli/scripts/tg-load-kg-core index 13fac153..e2d0a405 100755 --- a/trustgraph-cli/scripts/tg-load-kg-core +++ b/trustgraph-cli/scripts/tg-load-kg-core @@ -104,8 +104,10 @@ async def loader(ge_queue, t_queue, path, format, user, collection): async def run(**args): - ge_q = asyncio.Queue() - t_q = asyncio.Queue() + # Maxsize on queues reduces back-pressure so tg-load-kg-core doesn't + # grow to eat all memory + ge_q = asyncio.Queue(maxsize=500) + t_q = asyncio.Queue(maxsize=500) load_task = asyncio.create_task( loader( diff --git a/trustgraph-flow/trustgraph/api/gateway/service.py b/trustgraph-flow/trustgraph/api/gateway/service.py index 0ae01d3a..7b12e1a2 100755 --- a/trustgraph-flow/trustgraph/api/gateway/service.py +++ b/trustgraph-flow/trustgraph/api/gateway/service.py @@ -231,7 +231,7 @@ def serialize_graph_embeddings(message): "collection": message.metadata.collection, }, "vectors": message.vectors, - "entity": message.entity.value, + "entity": message.entity, } class Api: