diff --git a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh index f6c4f99..feaae38 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh @@ -1,5 +1,18 @@ #!/bin/bash -e QUERY_SET_ID="${1}" +JUDGMENTS_ID="12345" +INDEX="ecommerce" +ID_FIELD="asin" -curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}" | jq +curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}" \ + -H "Content-Type: application/json" \ + --data-binary '{ + "query": { + "match": { + "description": { + "query": "#$query##" + } + } + } + }' diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java index 5bfbdf4..3dfbdea 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java @@ -18,6 +18,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.rest.RestStatus; import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel; import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModelParameters; @@ -66,6 +67,11 @@ public class SearchQualityEvaluationRestHandler extends BaseRestHandler { */ public static final String QUERYSET_RUN_URL = "/_plugins/search_quality_eval/run"; + /** + * The placeholder in the query that gets replaced by the query term when running a query set. + */ + public static final String QUERY_PLACEHOLDER = "#$query##"; + @Override public String getName() { return "Search Quality Evaluation Framework"; @@ -149,16 +155,30 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli } else if(QUERYSET_RUN_URL.equalsIgnoreCase(request.path())) { final String querySetId = request.param("id"); - final String judgmentsId = request.param("judgments"); + final String judgmentsId = request.param("judgments_id"); + final String index = request.param("index"); + final String idField = request.param("id_field", "_id"); - if(querySetId == null || judgmentsId == null) { + if(querySetId == null || judgmentsId == null || index == null) { return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"Missing required parameters.\"}")); } + if(!request.hasContent()) { + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"Missing query in body.\"}")); + } + + // Get the query JSON from the content. + final String query = new String(BytesReference.toBytes(request.content())); + + // Validate the query has a QUERY_PLACEHOLDER. + if(!query.contains(QUERY_PLACEHOLDER)) { + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"Missing query placeholder in query.\"}")); + } + try { final OpenSearchQuerySetRunner openSearchQuerySetRunner = new OpenSearchQuerySetRunner(client); - final QuerySetRunResult querySetRunResult = openSearchQuerySetRunner.run(querySetId, judgmentsId); + final QuerySetRunResult querySetRunResult = openSearchQuerySetRunner.run(querySetId, judgmentsId, index, idField, query); openSearchQuerySetRunner.save(querySetRunResult); } catch (Exception ex) { diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index e3a5df0..3c4c6c9 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -27,6 +27,8 @@ import java.util.List; import java.util.Map; +import static org.opensearch.eval.SearchQualityEvaluationRestHandler.QUERY_PLACEHOLDER; + /** * A {@link QuerySetRunner} for Amazon OpenSearch. */ @@ -45,7 +47,7 @@ public OpenSearchQuerySetRunner(final Client client) { } @Override - public QuerySetRunResult run(final String querySetId, final String judgmentsId) { + public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String index, final String idField, final String query) { // Get the query set. final SearchSourceBuilder getQuerySetSearchSourceBuilder = new SearchSourceBuilder(); @@ -68,27 +70,14 @@ public QuerySetRunResult run(final String querySetId, final String judgmentsId) for(Map queryMap : queries) { // Loop over each query in the map and run each one. - for (final String query : queryMap.keySet()) { - - // TODO: Allow the user to pass these values in. - final String index = "ecommerce"; - final String idField = "asin"; - - // TODO: Allow the user to pass this in. - final String q = "{\n" + - " \"query\": {\n" + - " \"match\": {\n" + - " \"description\": {\n" + - " \"query\": \" + query + \"\n" + - " }\n" + - " }\n" + - " }\n" + - "}"; - - // TODO: What should this query be? + for (final String userQuery : queryMap.keySet()) { + + // Replace the query placeholder with the user query. + final String q = query.replace(QUERY_PLACEHOLDER, userQuery); + + // Build the query from the one that was passed in. final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - //searchSourceBuilder.query(QueryBuilders.wrapperQuery(q)); - searchSourceBuilder.query(QueryBuilders.matchQuery("description", query)); + searchSourceBuilder.query(QueryBuilders.wrapperQuery(q)); searchSourceBuilder.from(0); searchSourceBuilder.size(10); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunner.java index 59686ff..fc60d67 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunner.java @@ -17,10 +17,13 @@ public interface QuerySetRunner { /** * Runs the query set. * @param querySetId The ID of the query set to run. - * @param judgmentsId The ID of the judgments set to use for search metric calcuation. + * @param judgmentsId The ID of the judgments set to use for search metric calculation. + * @param index The name of the index to run the query sets against. + * @param idField The field in the index that is used to uniquely identify a document. + * @param query The query that will be used to run the query set. * @return The query set {@link QuerySetRunResult results} and calculated metrics. */ - QuerySetRunResult run(String querySetId, final String judgmentsId); + QuerySetRunResult run(String querySetId, final String judgmentsId, final String index, final String idField, final String query); /** * Saves the query set results to a persistent store, which may be the search engine itself.