Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Maint/knowledge load collections #132

Merged
merged 2 commits into from
Oct 31, 2024
Merged

Maint/knowledge load collections #132

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7c1568b
Add user/collection support to knowledge loaders
cybermaggedon Oct 31, 2024
bda5fd8
Fix timeouts
cybermaggedon Oct 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion trustgraph-base/trustgraph/clients/document_rag_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def __init__(
output_schema=DocumentRagResponse,
)

def request(self, query, timeout=500):
def request(self, query, timeout=300):

return self.call(
query=query, timeout=timeout
Expand Down
2 changes: 1 addition & 1 deletion trustgraph-base/trustgraph/clients/triples_query_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def request(
self,
s, p, o,
user="trustgraph", collection="default",
limit=10, timeout=60,
limit=10, timeout=120,
):
return self.call(
s=self.create_value(s),
Expand Down
29 changes: 27 additions & 2 deletions trustgraph-parquet/scripts/load-graph-embeddings
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ Loads Graph embeddings into TrustGraph processing.

import pulsar
from pulsar.schema import JsonSchema
from trustgraph.schema import GraphEmbeddings, Value
from trustgraph.schema import GraphEmbeddings, Value, Metadata
from trustgraph.schema import graph_embeddings_store_queue
import argparse
import os
Expand All @@ -24,6 +24,8 @@ class Loader:
output_queue,
log_level,
file,
user,
collection,
):

self.client = pulsar.Client(
Expand All @@ -38,6 +40,8 @@ class Loader:
)

self.file = file
self.user = user
self.collection = collection

def run(self):

Expand Down Expand Up @@ -66,11 +70,16 @@ class Loader:
n = ent.as_py()

r = GraphEmbeddings(
metadata=Metadata(
metadata=[],
user=self.user,
collection=self.collection,
),
vectors=b,
entity=Value(
value=n,
is_uri=n.startswith("https:")
)
),
)

self.producer.send(r)
Expand All @@ -90,6 +99,8 @@ def main():

default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
default_output_queue = graph_embeddings_store_queue
default_user = 'trustgraph'
default_collection = 'default'

parser.add_argument(
'-p', '--pulsar-host',
Expand All @@ -103,6 +114,18 @@ def main():
help=f'Output queue (default: {default_output_queue})'
)

parser.add_argument(
'-u', '--user',
default=default_user,
help=f'User ID (default: {default_user})'
)

parser.add_argument(
'-c', '--collection',
default=default_collection,
help=f'Collection ID (default: {default_collection})'
)

parser.add_argument(
'-l', '--log-level',
type=LogLevel,
Expand All @@ -127,6 +150,8 @@ def main():
output_queue=args.output_queue,
log_level=args.log_level,
file=args.file,
user=args.user,
collection=args.collection,
)

p.run()
Expand Down
48 changes: 42 additions & 6 deletions trustgraph-parquet/scripts/load-triples
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ Loads Graph embeddings into TrustGraph processing.

import pulsar
from pulsar.schema import JsonSchema
from trustgraph.schema import Triple, Value
from trustgraph.schema import Triples, Triple, Value, Metadata
from trustgraph.schema import triples_store_queue
import argparse
import os
Expand All @@ -24,6 +24,8 @@ class Loader:
output_queue,
log_level,
file,
user,
collection,
):

self.client = pulsar.Client(
Expand All @@ -33,11 +35,13 @@ class Loader:

self.producer = self.client.create_producer(
topic=output_queue,
schema=JsonSchema(Triple),
schema=JsonSchema(Triples),
chunking_enabled=True,
)

self.file = file
self.user = user
self.collection = collection

def run(self):

Expand Down Expand Up @@ -66,10 +70,26 @@ class Loader:

for s, p, o in zip(sc, pc, oc):

r = Triple(
s=Value(value=s.as_py(), is_uri=True),
p=Value(value=p.as_py(), is_uri=True),
o=Value(value=o.as_py(), is_uri=o.as_py().startswith("https:"))
r = Triples(
metadata=Metadata(
metadata=[],
user=self.user,
collection=self.collection,
),
triples=[
Triple(
s=Value(
value=s.as_py(), is_uri=True
),
p=Value(
value=p.as_py(), is_uri=True
),
o=Value(
value=o.as_py(),
is_uri=o.as_py().startswith("https:")
)
)
]
)

self.producer.send(r)
Expand All @@ -89,6 +109,8 @@ def main():

default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
default_output_queue = triples_store_queue
default_user = 'trustgraph'
default_collection = 'default'

parser.add_argument(
'-p', '--pulsar-host',
Expand All @@ -102,6 +124,18 @@ def main():
help=f'Output queue (default: {default_output_queue})'
)

parser.add_argument(
'-u', '--user',
default=default_user,
help=f'User ID (default: {default_user})'
)

parser.add_argument(
'-c', '--collection',
default=default_collection,
help=f'Collection ID (default: {default_collection})'
)

parser.add_argument(
'-l', '--log-level',
type=LogLevel,
Expand All @@ -126,6 +160,8 @@ def main():
output_queue=args.output_queue,
log_level=args.log_level,
file=args.file,
user=args.user,
collection=args.collection,
)

p.run()
Expand Down
Loading