Getting judgments for metrics calculation. Improving error handling and validation.

data/esci/index.sh

@@ -1,3 +1,10 @@
 #!/bin/bash -e
 
+echo "Deleting existing ubi_events and ubi_queries indexes..."
+curl -s -X DELETE "http://localhost:9200/ubi_queries,ubi_events"
+
+echo "Initializing UBI..."
+curl -s -X POST "http://localhost:9200/_plugins/ubi/initialize"
+
+echo "Indexing queries and events..."
 curl -s -X POST "http://localhost:9200/_bulk?pretty" -H "Content-Type: application/x-ndjson" --data-binary @ubi_queries_events.ndjson

opensearch-search-quality-evaluation-plugin/scripts/delete-query-sets.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -e
+
+curl -s -X DELETE "http://localhost:9200/search_quality_eval_query_sets"

opensearch-search-quality-evaluation-plugin/scripts/get-judgments.sh

@@ -1,3 +1,7 @@
 #!/bin/bash -e
 
+JUDGMENT_ID=$1
+
+#curl -s "http://localhost:9200/judgments/_doc/${JUDGMENT_ID}" | jq
+
 curl -s "http://localhost:9200/judgments/_search" | jq

opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh

@@ -1,19 +1,21 @@
 #!/bin/bash -e
 
 QUERY_SET_ID="${1}"
-JUDGMENTS_ID="12345"
+JUDGMENTS_ID="669fc8aa-3fe7-418f-952b-df7354af8f37"
 INDEX="ecommerce"
 ID_FIELD="asin"
 K="10"
 
+curl -s -X DELETE "http://localhost:9200/search_quality_eval_query_sets_run_results"
+
 curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}" \
    -H "Content-Type: application/json" \
     --data-binary '{
-                  "query": {
-                    "match": {
-                      "description": {
-                        "query": "#$query##"
+                    "query": {
+                      "match": {
+                        "description": {
+                          "query": "#$query##"
+                        }
                       }
                     }
-                  }
-                }'
+                  }'

opensearch-search-quality-evaluation-plugin/scripts/walkthrough.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -e
+
+# Example walkthrough end-to-end for the plugin.
+
+# Delete existing UBI indexes and create new ones.
+curl -s -X DELETE "http://localhost:9200/ubi_queries,ubi_events"
+curl -s -X POST "http://localhost:9200/_plugins/ubi/initialize"
+
+# IMPORTANT: Now index data (UBI and ESCI).
+
+# Create judgments.
+curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?click_model=coec&max_rank=20"
+
+# Create a query set.
+curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss&query_set_size=100"
+
+# Run the query set.
+./run-query-set.sh ${QUERY_SET_ID}
+
+# Look at the results.
+curl -s "http://localhost:9200/search_quality_eval_query_sets_run_results/_search" | jq

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java

@@ -79,7 +79,12 @@ public class SearchQualityEvaluationPlugin extends Plugin implements ActionPlugi
     /**
      * The name of the index that stores the query set run results.
      */
-    public static final String QUERY_SETS_RUN_RESULTS = "search_quality_eval_query_sets_run_results";
+    public static final String QUERY_SETS_RUN_RESULTS_INDEX_NAME = "search_quality_eval_query_sets_run_results";
+
+    /**
+     * The name of the index that stores the implicit judgments.
+     */
+    public static final String JUDGMENTS_INDEX_NAME = "judgments";
 
     @Override
     public Collection<Object> createComponents(

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java

@@ -160,7 +160,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
             final String idField = request.param("id_field", "_id");
             final int k = Integer.parseInt(request.param("k", "10"));
 
-            if(querySetId == null || judgmentsId == null || index == null) {
+            if(querySetId == null || querySetId.isEmpty() || judgmentsId == null || judgmentsId.isEmpty() || index == null || index.isEmpty()) {
                 return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"Missing required parameters.\"}"));
             }
 
@@ -187,7 +187,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
                 openSearchQuerySetRunner.save(querySetRunResult);
 
             } catch (Exception ex) {
-                LOGGER.error("Unable to run query set with ID {}: ", querySetId, ex);
+                LOGGER.error("Unable to run query set.", ex);
                 return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, ex.getMessage()));
             }
 

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java

@@ -51,10 +51,9 @@ public class CoecClickModel extends ClickModel {
 
     public static final String CLICK_MODEL_NAME = "coec";
 
-    // OpenSearch indexes.
+    // OpenSearch indexes for COEC data.
     public static final String INDEX_RANK_AGGREGATED_CTR = "rank_aggregated_ctr";
     public static final String INDEX_QUERY_DOC_CTR = "click_through_rates";
-    public static final String INDEX_JUDGMENTS = "judgments";
 
     // UBI event names.
     public static final String EVENT_CLICK = "click";

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java

@@ -36,12 +36,16 @@
 import java.util.Set;
 import java.util.UUID;
 
+import static org.opensearch.eval.SearchQualityEvaluationPlugin.JUDGMENTS_INDEX_NAME;
 import static org.opensearch.eval.SearchQualityEvaluationPlugin.UBI_EVENTS_INDEX_NAME;
 import static org.opensearch.eval.SearchQualityEvaluationPlugin.UBI_QUERIES_INDEX_NAME;
-import static org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel.INDEX_JUDGMENTS;
 import static org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel.INDEX_QUERY_DOC_CTR;
 import static org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel.INDEX_RANK_AGGREGATED_CTR;
 
+/**
+ * Functionality for interacting with OpenSearch.
+ * TODO: Move these functions out of this class.
+ */
 public class OpenSearchHelper {
 
     private static final Logger LOGGER = LogManager.getLogger(OpenSearchHelper.class.getName());
@@ -329,7 +333,7 @@ public String indexJudgments(final Collection<Judgment> judgments) throws Except
         final Map<String, Object> judgmentsSource = new HashMap<>();
         judgmentsSource.put("judgments", j);
 
-        final IndexRequest indexRequest = new IndexRequest(INDEX_JUDGMENTS)
+        final IndexRequest indexRequest = new IndexRequest(JUDGMENTS_INDEX_NAME)
                 .id(judgmentsId)
                 .source(judgmentsSource);
 

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/SearchMetrics.java → opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/SearchMetrics.java

@@ -6,9 +6,10 @@
  * this file be licensed under the Apache-2.0 license or a
  * compatible open source license.
  */
-package org.opensearch.eval.runners;
+package org.opensearch.eval.metrics;
 
 import org.opensearch.eval.judgments.model.Judgment;
+import org.opensearch.eval.runners.QueryResult;
 
 import java.util.HashMap;
 import java.util.List;

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java

@@ -16,8 +16,10 @@
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.client.Client;
 import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.common.util.CollectionUtils;
 import org.opensearch.eval.SearchQualityEvaluationPlugin;
 import org.opensearch.eval.judgments.model.Judgment;
+import org.opensearch.eval.metrics.SearchMetrics;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.builder.SearchSourceBuilder;
@@ -33,29 +35,35 @@
 /**
  * A {@link QuerySetRunner} for Amazon OpenSearch.
  */
-public class OpenSearchQuerySetRunner implements QuerySetRunner {
+public class OpenSearchQuerySetRunner extends QuerySetRunner {
 
     private static final Logger LOGGER = LogManager.getLogger(OpenSearchQuerySetRunner.class);
 
-    final Client client;
-
     /**
      * Creates a new query set runner
      * @param client An OpenSearch {@link Client}.
      */
     public OpenSearchQuerySetRunner(final Client client) {
-        this.client = client;
+        super(client);
     }
 
     @Override
-    public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String index, final String idField, final String query, final int k) {
+    public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String index, final String idField, final String query, final int k) throws Exception {
+
+        // Get the judgments we will use for metric calculation.
+        final List<Judgment> judgments = getJudgments(judgmentsId);
 
-        // TODO: Get the judgments we will use for metric calculation.
-        final List<Judgment> judgments = new ArrayList<>();
+        if(CollectionUtils.isEmpty(judgments)) {
+            // We have to have judgments to continue. The judgmentsId is either wrong or doesn't exist.
+            throw new RuntimeException("Judgments are empty. Check judgment_id and try again.");
+        }
 
         // Get the query set.
         final SearchSourceBuilder getQuerySetSearchSourceBuilder = new SearchSourceBuilder();
         getQuerySetSearchSourceBuilder.query(QueryBuilders.matchQuery("_id", querySetId));
+        getQuerySetSearchSourceBuilder.from(0);
+        // TODO: Need to page through to make sure we get all of the queries.
+        getQuerySetSearchSourceBuilder.size(500);
 
         final SearchRequest getQuerySetSearchRequest = new SearchRequest(SearchQualityEvaluationPlugin.QUERY_SETS_INDEX_NAME);
         getQuerySetSearchRequest.source(getQuerySetSearchSourceBuilder);
@@ -83,7 +91,6 @@ public QuerySetRunResult run(final String querySetId, final String judgmentsId,
                     final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
                     searchSourceBuilder.query(QueryBuilders.wrapperQuery(q));
                     searchSourceBuilder.from(0);
-                    // TODO: If k is > 10, we'll need to page through these.
                     searchSourceBuilder.size(k);
 
                     String[] includeFields = new String[] {idField};
@@ -110,7 +117,7 @@ public void onResponse(final SearchResponse searchResponse) {
 
                             }
 
-                            queryResults.add(new QueryResult(query, orderedDocumentIds, judgments, k));
+                            queryResults.add(new QueryResult(userQuery, orderedDocumentIds, judgments, k));
 
                         }
 
@@ -129,8 +136,8 @@ public void onFailure(Exception ex) {
 
             return new QuerySetRunResult(queryResults, searchMetrics);
 
-        } catch (Exception e) {
-            throw new RuntimeException(e);
+        } catch (Exception ex) {
+            throw new RuntimeException("Unable to run query set.", ex);
         }
 
     }
@@ -146,7 +153,7 @@ public void save(final QuerySetRunResult result) throws Exception {
         results.put("search_metrics", result.getSearchMetrics().getSearchMetricsAsMap());
         results.put("query_results", result.getQueryResultsAsMap());
 
-        final IndexRequest indexRequest = new IndexRequest(SearchQualityEvaluationPlugin.QUERY_SETS_RUN_RESULTS);
+        final IndexRequest indexRequest = new IndexRequest(SearchQualityEvaluationPlugin.QUERY_SETS_RUN_RESULTS_INDEX_NAME);
         indexRequest.source(results);
 
         client.index(indexRequest, new ActionListener<>() {
@@ -160,6 +167,7 @@ public void onFailure(Exception ex) {
                 throw new RuntimeException(ex);
             }
         });
+
     }
 
 }

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QueryResult.java

@@ -9,6 +9,7 @@
 package org.opensearch.eval.runners;
 
 import org.opensearch.eval.judgments.model.Judgment;
+import org.opensearch.eval.metrics.SearchMetrics;
 
 import java.util.List;
 

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java

@@ -8,6 +8,8 @@
  */
 package org.opensearch.eval.runners;
 
+import org.opensearch.eval.metrics.SearchMetrics;
+
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -73,7 +75,6 @@ public Collection<Map<String, Object>> getQueryResultsAsMap() {
 
             qs.add(q);
 
-
         }
 
         return qs;

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunner.java

@@ -8,11 +8,34 @@
  */
 package org.opensearch.eval.runners;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.action.search.SearchRequest;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.client.Client;
+import org.opensearch.eval.SearchQualityEvaluationPlugin;
+import org.opensearch.eval.judgments.model.Judgment;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.search.builder.SearchSourceBuilder;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
 /**
- * Interface for query set runners. Classes that implement this interface
+ * Base class for query set runners. Classes that extend this class
  * should be specific to a search engine. See the {@link OpenSearchQuerySetRunner} for an example.
  */
-public interface QuerySetRunner {
+public abstract class QuerySetRunner {
+
+    private static final Logger LOGGER = LogManager.getLogger(QuerySetRunner.class);
+
+    protected final Client client;
+
+    public QuerySetRunner(final Client client) {
+        this.client = client;
+    }
 
     /**
      * Runs the query set.
@@ -24,12 +47,57 @@ public interface QuerySetRunner {
      * @param k The k used for metrics calculation, i.e. DCG@k.
      * @return The query set {@link QuerySetRunResult results} and calculated metrics.
      */
-    QuerySetRunResult run(String querySetId, final String judgmentsId, final String index, final String idField, final String query, final int k);
+    abstract QuerySetRunResult run(String querySetId, final String judgmentsId, final String index, final String idField, final String query, final int k) throws Exception;
 
     /**
      * Saves the query set results to a persistent store, which may be the search engine itself.
      * @param result The {@link QuerySetRunResult results}.
      */
-    void save(QuerySetRunResult result) throws Exception;
+    abstract void save(QuerySetRunResult result) throws Exception;
+
+    public List<Judgment> getJudgments(final String judgmentsId) throws Exception {
+
+        final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.query(QueryBuilders.matchQuery("_id", judgmentsId));
+        searchSourceBuilder.trackTotalHits(true);
+
+        final SearchRequest getQuerySetSearchRequest = new SearchRequest(SearchQualityEvaluationPlugin.JUDGMENTS_INDEX_NAME);
+        getQuerySetSearchRequest.source(searchSourceBuilder);
+
+        // TODO: Don't use .get()
+        final SearchResponse searchResponse = client.search(getQuerySetSearchRequest).get();
+
+        final List<Judgment> judgments = new ArrayList<>();
+
+        if(searchResponse.getHits().getTotalHits().value == 0) {
+
+            // The judgment_id is probably not valid.
+            // This will return an empty list.
+
+        } else {
+
+            // TODO: Make sure the search gets something back.
+            final Collection<Map<String, Object>> j = (Collection<Map<String, Object>>) searchResponse.getHits().getAt(0).getSourceAsMap().get("judgments");
+
+            for (final Map<String, Object> judgment : j) {
+
+                final String queryId = judgment.get("query_id").toString();
+                final double judgmentValue = Double.parseDouble(judgment.get("judgment").toString());
+                final String query = judgment.get("query").toString();
+                final String document = judgment.get("document").toString();
+
+                final Judgment jobj = new Judgment(queryId, query, document, judgmentValue);
+                LOGGER.info("Judgment: {}", jobj.toJudgmentString());
+
+                judgments.add(jobj);
+
+            }
+
+        }
+
+        return judgments;
+
+    }
+
 
 }