From 0e240b86ab9a4ee17d9b5d514ea6637817681ba3 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Tue, 3 Dec 2024 10:34:25 -0500 Subject: [PATCH] Passing judgments into SearchMetrics for calculating metrics. --- .../eval/runners/OpenSearchQuerySetRunner.java | 8 ++++++-- .../java/org/opensearch/eval/runners/QueryResult.java | 7 +++++-- .../org/opensearch/eval/runners/SearchMetrics.java | 11 ++++++++--- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index fc9f958..7679ab1 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -17,6 +17,7 @@ import org.opensearch.client.Client; import org.opensearch.core.action.ActionListener; import org.opensearch.eval.SearchQualityEvaluationPlugin; +import org.opensearch.eval.judgments.model.Judgment; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; @@ -49,6 +50,9 @@ public OpenSearchQuerySetRunner(final Client client) { @Override public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String index, final String idField, final String query, final int k) { + // TODO: Get the judgments we will use for metric calculation. + final List judgments = new ArrayList<>(); + // Get the query set. final SearchSourceBuilder getQuerySetSearchSourceBuilder = new SearchSourceBuilder(); getQuerySetSearchSourceBuilder.query(QueryBuilders.matchQuery("_id", querySetId)); @@ -105,7 +109,7 @@ public void onResponse(final SearchResponse searchResponse) { } - queryResults.add(new QueryResult(query, orderedDocumentIds, k)); + queryResults.add(new QueryResult(query, orderedDocumentIds, judgments, k)); } @@ -120,7 +124,7 @@ public void onFailure(Exception ex) { } // TODO: Calculate the search metrics given the results and the judgments. - final SearchMetrics searchMetrics = new SearchMetrics(queryResults, k); + final SearchMetrics searchMetrics = new SearchMetrics(queryResults, judgments, k); return new QuerySetRunResult(queryResults, searchMetrics); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QueryResult.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QueryResult.java index 6910190..28af2b2 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QueryResult.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QueryResult.java @@ -8,6 +8,8 @@ */ package org.opensearch.eval.runners; +import org.opensearch.eval.judgments.model.Judgment; + import java.util.List; /** @@ -24,12 +26,13 @@ public class QueryResult { * @param query The query used to generate this result. * @param orderedDocumentIds A list of ordered document IDs in the same order as they appeared * in the query. + * @param judgments A list of {@link Judgment judgments} used for metric calculation. * @param k The k used for metrics calculation, i.e. DCG@k. */ - public QueryResult(final String query, final List orderedDocumentIds, final int k) { + public QueryResult(final String query, final List orderedDocumentIds, final List judgments, final int k) { this.query = query; this.orderedDocumentIds = orderedDocumentIds; - this.searchMetrics = new SearchMetrics(query, orderedDocumentIds, k); + this.searchMetrics = new SearchMetrics(query, orderedDocumentIds, judgments, k); } /** diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/SearchMetrics.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/SearchMetrics.java index c04c6fc..ae2edbc 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/SearchMetrics.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/SearchMetrics.java @@ -8,6 +8,8 @@ */ package org.opensearch.eval.runners; +import org.opensearch.eval.judgments.model.Judgment; + import java.util.HashMap; import java.util.List; import java.util.Map; @@ -26,9 +28,10 @@ public class SearchMetrics { /** * Create the search metrics for an entire query set. * @param queryResults A list of {@link QueryResult}. + * @param judgments A list of {@link Judgment judgments} used for metric calculation. * @param k The k used for metrics calculation, i.e. DCG@k. */ - public SearchMetrics(final List queryResults, final int k) { + public SearchMetrics(final List queryResults, final List judgments, final int k) { this.k = k; // TODO: Calculate the metrics for the whole query set. @@ -36,10 +39,12 @@ public SearchMetrics(final List queryResults, final int k) { /** * Create the search metrics for a single query. - + * @param query The user query. + * @param orderedDocumentIds The documents returned for the user query in order. + * @param judgments A list of {@link Judgment judgments} used for metric calculation. * @param k The k used for metrics calculation, i.e. DCG@k. */ - public SearchMetrics(final String query, final List orderedDocumentIds, final int k) { + public SearchMetrics(final String query, final List orderedDocumentIds, final List judgments, final int k) { this.k = k; // TODO: Calculate the metrics for the single query.