Skip to content

Commit

Permalink
Allow an update processor argument when bulk loading
Browse files Browse the repository at this point in the history
  • Loading branch information
kevinschaper committed Mar 6, 2024
1 parent 5251a43 commit ef27b4e
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 2 deletions.
6 changes: 4 additions & 2 deletions linkml_solr/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,10 @@ def main(verbose: int, quiet: bool):
@click.option('--url', '-u',
default=DEFAULT_SOLR_URL,
help='solr url.')
@click.option('--processor', '-p',
help='Processor argument to pass when bulk loading to Solr')
@click.argument('files', nargs=-1)
def bulkload(files, format, schema, url, core):
def bulkload(files, format, schema, url, core, processor=None):
"""
Convert multiple golr yaml schemas to linkml
"""
Expand All @@ -60,7 +62,7 @@ def bulkload(files, format, schema, url, core):
with open(schema) as stream:
schema_obj = yaml_loader.load(stream, target_class=SchemaDefinition)
for f in files:
bulkload_file(f, format=format, schema=schema_obj, core=core, base_url=url)
bulkload_file(f, format=format, schema=schema_obj, core=core, base_url=url, processor=processor)

@main.command()
@click.option('--schema', '-s',
Expand Down
4 changes: 4 additions & 0 deletions linkml_solr/utils/solr_bulkload.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ def bulkload_file(f,
base_url=None,
core=None,
schema: SchemaDefinition = None,
processor: str = None,
):
"""
Bulkload a file using solr bulkload API
Expand All @@ -21,6 +22,7 @@ def bulkload_file(f,
:param base_url:
:param core:
:param schema:
:param processor: Processor argument to pass when bulk loading to Solr
:return:
"""
mvslots = _get_multivalued_slots(schema)
Expand All @@ -29,6 +31,8 @@ def bulkload_file(f,
internal_separator = '%7C'
parts = [f'f.{s}.split=true&f.{s}.separator={internal_separator}' for s in mvslots]
url = f'{base_url}/{core}/update?{"&".join(parts)}&commit=true&separator={separator}'
if (processor is not None):
url = f'{url}&processor={processor}'
if format == 'csv':
ct = 'application/csv'
elif format == 'json':
Expand Down

0 comments on commit ef27b4e

Please sign in to comment.