Getting judgments for metrics calculation. Improving error handling and validation.

data/esci/index.sh

@@ -1,3 +1,10 @@
 #!/bin/bash -e
 
+echo "Deleting existing ubi_events and ubi_queries indexes..."
+curl -s -X DELETE "http://localhost:9200/ubi_queries,ubi_events"
+
+echo "Initializing UBI..."
+curl -s -X POST "http://localhost:9200/_plugins/ubi/initialize"
+
+echo "Indexing queries and events..."
 curl -s -X POST "http://localhost:9200/_bulk?pretty" -H "Content-Type: application/x-ndjson" --data-binary @ubi_queries_events.ndjson

opensearch-search-quality-evaluation-plugin/Dockerfile

@@ -1,6 +1,6 @@
 FROM opensearchproject/opensearch:2.18.0
 
-RUN /usr/share/opensearch/bin/opensearch-plugin install --batch https://github.com/opensearch-project/user-behavior-insights/releases/download/2.18.0.1/opensearch-ubi-2.18.0.1.zip
+RUN /usr/share/opensearch/bin/opensearch-plugin install --batch https://github.com/opensearch-project/user-behavior-insights/releases/download/2.18.0.2/opensearch-ubi-2.18.0.2.zip
 
 ADD ./build/distributions/search-quality-evaluation-plugin-2.18.0.0.zip  /tmp/search-quality-evaluation-plugin.zip
 RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/search-quality-evaluation-plugin.zip

opensearch-search-quality-evaluation-plugin/scripts/create-query-set-using-pptss-sampling.sh

@@ -1,8 +1,13 @@
 #!/bin/bash -e
 
-#QUERY_SET=`curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss" | jq .query_set | tr -d '"'`
+curl -s -X DELETE "http://localhost:9200/search_quality_eval_query_sets"
+
 curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss&query_set_size=100"
 
+
+#QUERY_SET=`curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss" | jq .query_set | tr -d '"'`
+#curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss&query_set_size=100"
+
 #echo ${QUERY_SET}
 
 #curl -s -X GET http://localhost:9200/search_quality_eval_query_sets/_doc/${QUERY_SET} | jq

opensearch-search-quality-evaluation-plugin/scripts/delete-query-sets.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -e
+
+curl -s -X DELETE "http://localhost:9200/search_quality_eval_query_sets"

opensearch-search-quality-evaluation-plugin/scripts/get-judgments.sh

@@ -1,3 +1,6 @@
 #!/bin/bash -e
 
+#JUDGMENT_ID=$1
+#curl -s "http://localhost:9200/judgments/_doc/${JUDGMENT_ID}" | jq
+
 curl -s "http://localhost:9200/judgments/_search" | jq

opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh

@@ -1,19 +1,21 @@
 #!/bin/bash -e
 
 QUERY_SET_ID="${1}"
-JUDGMENTS_ID="12345"
+JUDGMENTS_ID="669fc8aa-3fe7-418f-952b-df7354af8f37"
 INDEX="ecommerce"
 ID_FIELD="asin"
 K="10"
 
+curl -s -X DELETE "http://localhost:9200/search_quality_eval_query_sets_run_results"
+
 curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}" \
    -H "Content-Type: application/json" \
     --data-binary '{
-                  "query": {
-                    "match": {
-                      "description": {
-                        "query": "#$query##"
+                    "query": {
+                      "match": {
+                        "title": {
+                          "query": "#$query##"
+                        }
                       }
                     }
-                  }
-                }'
+                  }'

opensearch-search-quality-evaluation-plugin/scripts/walkthrough.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -e
+
+# Example walkthrough end-to-end for the plugin.
+
+# Delete existing UBI indexes and create new ones.
+curl -s -X DELETE "http://localhost:9200/ubi_queries,ubi_events"
+curl -s -X POST "http://localhost:9200/_plugins/ubi/initialize"
+
+# IMPORTANT: Now index data (UBI and ESCI).
+
+# Create judgments.
+curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?click_model=coec&max_rank=20"
+
+# Create a query set.
+curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss&query_set_size=100"
+
+# Run the query set.
+./run-query-set.sh ${QUERY_SET_ID}
+
+# Look at the results.
+curl -s "http://localhost:9200/search_quality_eval_query_sets_run_results/_search" | jq

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java

@@ -79,7 +79,12 @@ public class SearchQualityEvaluationPlugin extends Plugin implements ActionPlugi
     /**
      * The name of the index that stores the query set run results.
      */
-    public static final String QUERY_SETS_RUN_RESULTS = "search_quality_eval_query_sets_run_results";
+    public static final String QUERY_SETS_RUN_RESULTS_INDEX_NAME = "search_quality_eval_query_sets_run_results";
+
+    /**
+     * The name of the index that stores the implicit judgments.
+     */
+    public static final String JUDGMENTS_INDEX_NAME = "judgments";
 
     @Override
     public Collection<Object> createComponents(

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java

@@ -22,7 +22,7 @@
 import org.opensearch.core.rest.RestStatus;
 import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel;
 import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModelParameters;
-import org.opensearch.eval.runners.OpenSearchQuerySetRunner;
+import org.opensearch.eval.runners.OpenSearchAbstractQuerySetRunner;
 import org.opensearch.eval.runners.QuerySetRunResult;
 import org.opensearch.eval.samplers.AllQueriesQuerySampler;
 import org.opensearch.eval.samplers.AllQueriesQuerySamplerParameters;
@@ -160,7 +160,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
             final String idField = request.param("id_field", "_id");
             final int k = Integer.parseInt(request.param("k", "10"));
 
-            if(querySetId == null || judgmentsId == null || index == null) {
+            if(querySetId == null || querySetId.isEmpty() || judgmentsId == null || judgmentsId.isEmpty() || index == null || index.isEmpty()) {
                 return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"Missing required parameters.\"}"));
             }
 
@@ -182,12 +182,12 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
 
             try {
 
-                final OpenSearchQuerySetRunner openSearchQuerySetRunner = new OpenSearchQuerySetRunner(client);
+                final OpenSearchAbstractQuerySetRunner openSearchQuerySetRunner = new OpenSearchAbstractQuerySetRunner(client);
                 final QuerySetRunResult querySetRunResult = openSearchQuerySetRunner.run(querySetId, judgmentsId, index, idField, query, k);
                 openSearchQuerySetRunner.save(querySetRunResult);
 
             } catch (Exception ex) {
-                LOGGER.error("Unable to run query set with ID {}: ", querySetId, ex);
+                LOGGER.error("Unable to run query set.", ex);
                 return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, ex.getMessage()));
             }
 
@@ -234,27 +234,20 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
                             .source(job)
                             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
 
-                    final AtomicBoolean success = new AtomicBoolean(false);
-
                     client.index(indexRequest, new ActionListener<>() {
                         @Override
                         public void onResponse(final IndexResponse indexResponse) {
-                            LOGGER.debug("Job completed successfully: {}", jobId);
-                            success.set(true);
+                            LOGGER.debug("Click model job completed successfully: {}", jobId);
                         }
 
                         @Override
                         public void onFailure(final Exception ex) {
                             LOGGER.error("Unable to run job with ID {}", jobId, ex);
-                            success.set(false);
+                            throw new RuntimeException("Unable to run job", ex);
                         }
                     });
 
-                    if(success.get()) {
-                        return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"judgments_id\": \"" + jobId + "\"}"));
-                    } else {
-                        return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR,"Unable to index judgments."));
-                    }
+                    return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"judgments_id\": \"" + jobId + "\"}"));
 
                 } else {
                     return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"Invalid click model.\"}"));

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java

@@ -51,10 +51,9 @@ public class CoecClickModel extends ClickModel {
 
     public static final String CLICK_MODEL_NAME = "coec";
 
-    // OpenSearch indexes.
+    // OpenSearch indexes for COEC data.
     public static final String INDEX_RANK_AGGREGATED_CTR = "rank_aggregated_ctr";
     public static final String INDEX_QUERY_DOC_CTR = "click_through_rates";
-    public static final String INDEX_JUDGMENTS = "judgments";
 
     // UBI event names.
     public static final String EVENT_CLICK = "click";

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java

@@ -74,6 +74,33 @@ public static void showJudgments(final Collection<Judgment> judgments) {
 
     }
 
+    /**
+     * Find a judgment in a collection of judgments.
+     * @param judgments The collection of {@link Judgment judgments}.
+     * @param query The query to find.
+     * @param documentId The document ID to find.
+     * @return The matching {@link Judgment judgment}.
+     */
+    public static Judgment findJudgment(final Collection<Judgment> judgments, final String query, final String documentId) {
+
+        for(final Judgment judgment : judgments) {
+
+            // LOGGER.info("Comparing {}:{} with {}:{}", judgment.getQuery(), judgment.getDocument(), query, documentId);
+
+            if(judgment.getQuery().equalsIgnoreCase(query) && judgment.getDocument().equalsIgnoreCase(documentId)) {
+                LOGGER.info("Judgment score of {}  for query {} and document {} was found.", judgment.getJudgment(), query, documentId);
+                return judgment;
+            }
+
+        }
+
+        // A judgment for this query and document was not found.
+        LOGGER.warn("A judgment for query {} and document {} was not found.", query, documentId);
+        // TODO: Would this ever happen?
+        return null;
+
+    }
+
     @Override
     public String toString() {
         return "query_id: " + queryId + ", query: " + query + ", document: " + document + ", judgment: " + MathUtils.round(judgment);

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/ubi/query/QueryResponse.java

@@ -17,18 +17,18 @@ public class QueryResponse {
 
     private final String queryId;
     private final String queryResponseId;
-    private final List<String> queryResponseObjectIds;
+    private final List<String> queryResponseHitIds;
 
     /**
      * Creates a query response.
      * @param queryId The ID of the query.
      * @param queryResponseId The ID of the query response.
-     * @param queryResponseObjectIds A list of IDs for the hits in the query.
+     * @param queryResponseHitIds A list of IDs for the hits in the query.
      */
-    public QueryResponse(final String queryId, final String queryResponseId, final List<String> queryResponseObjectIds) {
+    public QueryResponse(final String queryId, final String queryResponseId, final List<String> queryResponseHitIds) {
         this.queryId = queryId;
         this.queryResponseId = queryResponseId;
-        this.queryResponseObjectIds = queryResponseObjectIds;
+        this.queryResponseHitIds = queryResponseHitIds;
     }
 
     /**
@@ -51,8 +51,8 @@ public String getQueryResponseId() {
      * Gets the list of query response hit IDs.
      * @return A list of query response hit IDs.
      */
-    public List<String> getQueryResponseObjectIds() {
-        return queryResponseObjectIds;
+    public List<String> getQueryResponseHitIds() {
+        return queryResponseHitIds;
     }
 
 }

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java

@@ -36,12 +36,16 @@
 import java.util.Set;
 import java.util.UUID;
 
+import static org.opensearch.eval.SearchQualityEvaluationPlugin.JUDGMENTS_INDEX_NAME;
 import static org.opensearch.eval.SearchQualityEvaluationPlugin.UBI_EVENTS_INDEX_NAME;
 import static org.opensearch.eval.SearchQualityEvaluationPlugin.UBI_QUERIES_INDEX_NAME;
-import static org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel.INDEX_JUDGMENTS;
 import static org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel.INDEX_QUERY_DOC_CTR;
 import static org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel.INDEX_RANK_AGGREGATED_CTR;
 
+/**
+ * Functionality for interacting with OpenSearch.
+ * TODO: Move these functions out of this class.
+ */
 public class OpenSearchHelper {
 
     private static final Logger LOGGER = LogManager.getLogger(OpenSearchHelper.class.getName());
@@ -230,9 +234,9 @@ public long getCountOfQueriesForUserQueryHavingResultInRankR(final String userQu
 
         for(final SearchHit searchHit : response.getHits().getHits()) {
 
-            final List<String> queryResponseObjectIds = (List<String>) searchHit.getSourceAsMap().get("query_response_object_ids");
+            final List<String> queryResponseHidsIds = (List<String>) searchHit.getSourceAsMap().get("query_response_hit_ids");
 
-            if(queryResponseObjectIds.get(rank).equals(objectId)) {
+            if(queryResponseHidsIds.get(rank).equals(objectId)) {
                 countOfTimesShownAtRank++;
             }
 
@@ -320,23 +324,22 @@ public String indexJudgments(final Collection<Judgment> judgments) throws Except
 
         final String judgmentsId = UUID.randomUUID().toString();
 
-        final Collection<Map<String, Object>> j = new ArrayList<>();
+        final BulkRequest request = new BulkRequest();
 
-        for (final Judgment judgment : judgments) {
-            j.add(judgment.getJudgmentAsMap());
-        }
+        for(final Judgment judgment : judgments) {
 
-        final Map<String, Object> judgmentsSource = new HashMap<>();
-        judgmentsSource.put("judgments", j);
+            final Map<String, Object> j = judgment.getJudgmentAsMap();
+            j.put("judgments_id", judgmentsId);
 
-        final IndexRequest indexRequest = new IndexRequest(INDEX_JUDGMENTS)
-                .id(judgmentsId)
-                .source(judgmentsSource);
+            final IndexRequest indexRequest = new IndexRequest(JUDGMENTS_INDEX_NAME)
+                    .id(UUID.randomUUID().toString())
+                    .source(j);
 
-        final BulkRequest request = new BulkRequest();
-        request.add(indexRequest);
+            request.add(indexRequest);
+
+        }
 
-        client.bulk(request, new ActionListener<BulkResponse>() {
+        client.bulk(request, new ActionListener<>() {
             @Override
             public void onResponse(BulkResponse bulkItemResponses) {
                 LOGGER.info("Judgments indexed: {}", judgmentsId);

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/DcgSearchMetric.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.eval.metrics;
+
+import java.util.List;
+
+public class DcgSearchMetric extends SearchMetric {
+
+    protected final List<Double> relevanceScores;
+
+    public DcgSearchMetric(final int k, final List<Double> relevanceScores) {
+        super(k);
+        this.relevanceScores = relevanceScores;
+    }
+
+    @Override
+    public String getName() {
+        return "dcg_at_" + k;
+    }
+
+    @Override
+    public double calculate() {
+
+        double dcg = 0.0;
+        for(int i = 0; i < relevanceScores.size(); i++) {
+            double relevance = relevanceScores.get(i);
+            dcg += relevance / Math.log(i + 2); // Add 2 to avoid log(1) = 0
+        }
+        return dcg;
+
+    }
+
+}