From 504e3f95f10cfa272a382ed784f6102fe571e895 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 30 Dec 2024 14:24:29 -0500 Subject: [PATCH] Working on converting to a standalone app. --- .../eval/engine/OpenSearchEngine.java | 36 +++++++++++++++---- .../opensearch/eval/engine/SearchEngine.java | 6 ++-- .../clickmodel/coec/CoecClickModel.java | 2 +- .../eval/model/data/AbstractData.java | 5 +++ .../eval/model/{ => data}/Judgment.java | 4 +-- .../eval/samplers/AbstractQuerySampler.java | 12 ++----- .../eval/samplers/AllQueriesQuerySampler.java | 12 +++---- ...roportionalToSizeAbstractQuerySampler.java | 12 +++---- 8 files changed, 55 insertions(+), 34 deletions(-) create mode 100644 opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/model/data/AbstractData.java rename opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/model/{ => data}/Judgment.java (96%) diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/engine/OpenSearchEngine.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/engine/OpenSearchEngine.java index 32cb2b1..bfe0bec 100644 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/engine/OpenSearchEngine.java +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/engine/OpenSearchEngine.java @@ -15,20 +15,26 @@ import org.opensearch.client.json.jackson.JacksonJsonpMapper; import org.opensearch.client.opensearch.OpenSearchClient; import org.opensearch.client.opensearch._types.Refresh; +import org.opensearch.client.opensearch._types.Time; import org.opensearch.client.opensearch._types.mapping.IntegerNumberProperty; import org.opensearch.client.opensearch._types.mapping.Property; import org.opensearch.client.opensearch._types.mapping.TypeMapping; import org.opensearch.client.opensearch.core.BulkRequest; import org.opensearch.client.opensearch.core.BulkResponse; import org.opensearch.client.opensearch.core.IndexRequest; +import org.opensearch.client.opensearch.core.ScrollRequest; +import org.opensearch.client.opensearch.core.ScrollResponse; +import org.opensearch.client.opensearch.core.SearchResponse; import org.opensearch.client.opensearch.core.bulk.BulkOperation; import org.opensearch.client.opensearch.core.bulk.IndexOperation; +import org.opensearch.client.opensearch.core.search.Hit; import org.opensearch.client.opensearch.indices.CreateIndexRequest; import org.opensearch.client.opensearch.indices.ExistsRequest; import org.opensearch.client.transport.OpenSearchTransport; import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; +import org.opensearch.eval.Constants; import org.opensearch.eval.model.ClickthroughRate; -import org.opensearch.eval.model.Judgment; +import org.opensearch.eval.model.data.Judgment; import org.opensearch.eval.model.ubi.query.UbiQuery; import org.opensearch.eval.utils.TimeUtils; @@ -36,6 +42,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -99,15 +106,30 @@ public boolean deleteIndex(String index) throws IOException { } - @Override - public String indexJudgment(String index, String id, Judgment judgment) throws IOException { + public Collection getJudgments(final String index) throws IOException { + + final Collection judgments = new ArrayList<>(); + + final SearchResponse searchResponse = client.search(s -> s.index(index).size(1000).scroll(Time.of(t -> t.offset(1000))), Judgment.class); + + String scrollId = searchResponse.scrollId(); + List> searchHits = searchResponse.hits().hits(); + + while (searchHits != null && !searchHits.isEmpty()) { + + for (int i = 0; i < searchResponse.hits().hits().size(); i++) { + judgments.add(searchResponse.hits().hits().get(i).source()); + } + + final ScrollRequest scrollRequest = new ScrollRequest.Builder().scrollId(scrollId).build(); + final ScrollResponse scrollResponse = client.scroll(scrollRequest, Judgment.class); + + scrollId = scrollResponse.scrollId(); + searchHits = scrollResponse.hits().hits(); - if(id == null) { - id = UUID.randomUUID().toString(); } - final IndexRequest indexRequest = new IndexRequest.Builder().index(index).id(id).document(judgment).build(); - return client.index(indexRequest).id(); + return judgments; } diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/engine/SearchEngine.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/engine/SearchEngine.java index 66559ec..1996f11 100644 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/engine/SearchEngine.java +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/engine/SearchEngine.java @@ -1,7 +1,7 @@ package org.opensearch.eval.engine; import org.opensearch.eval.model.ClickthroughRate; -import org.opensearch.eval.model.Judgment; +import org.opensearch.eval.model.data.Judgment; import org.opensearch.eval.model.ubi.query.UbiQuery; import java.io.IOException; @@ -20,8 +20,10 @@ public abstract class SearchEngine { public abstract long getCountOfQueriesForUserQueryHavingResultInRankR(final String userQuery, final String objectId, final int rank) throws Exception; public abstract void indexRankAggregatedClickthrough(final Map rankAggregatedClickThrough) throws Exception; public abstract void indexClickthroughRates(final Map> clickthroughRates) throws Exception; - public abstract String indexJudgment(String index, String id, Judgment judgment) throws IOException; + public abstract String indexJudgments(final Collection judgments) throws Exception; public abstract boolean bulkIndex(String index, Map documents) throws IOException; + public abstract Collection getJudgments(final String index) throws IOException; + } diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java index 24a1492..d879947 100644 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java @@ -15,7 +15,7 @@ import org.opensearch.eval.engine.SearchEngine; import org.opensearch.eval.judgments.clickmodel.ClickModel; import org.opensearch.eval.model.ClickthroughRate; -import org.opensearch.eval.model.Judgment; +import org.opensearch.eval.model.data.Judgment; import org.opensearch.eval.model.ubi.event.UbiEvent; import org.opensearch.eval.judgments.queryhash.IncrementalUserQueryHash; import org.opensearch.eval.utils.MathUtils; diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/model/data/AbstractData.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/model/data/AbstractData.java new file mode 100644 index 0000000..f095ec3 --- /dev/null +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/model/data/AbstractData.java @@ -0,0 +1,5 @@ +package org.opensearch.eval.model.data; + +public abstract class AbstractData { + +} diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/model/Judgment.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/model/data/Judgment.java similarity index 96% rename from opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/model/Judgment.java rename to opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/model/data/Judgment.java index 89ec7b0..d8b853b 100644 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/model/Judgment.java +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/model/data/Judgment.java @@ -6,7 +6,7 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ -package org.opensearch.eval.model; +package org.opensearch.eval.model.data; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -18,7 +18,7 @@ /** * A judgment of a search result's quality for a given query. */ -public class Judgment { +public class Judgment extends AbstractData { private static final Logger LOGGER = LogManager.getLogger(Judgment.class.getName()); diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AbstractQuerySampler.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AbstractQuerySampler.java index 4209add..6945b59 100644 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AbstractQuerySampler.java +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AbstractQuerySampler.java @@ -15,6 +15,7 @@ import org.opensearch.client.opensearch.core.IndexRequest; import org.opensearch.client.opensearch.indices.CreateIndexRequest; import org.opensearch.eval.Constants; +import org.opensearch.eval.engine.SearchEngine; import org.opensearch.eval.utils.TimeUtils; import java.util.ArrayList; @@ -45,7 +46,7 @@ public abstract class AbstractQuerySampler { /** * Index the query set. */ - protected String indexQuerySet(final OpenSearchClient client, final String name, final String description, final String sampling, Map queries) throws Exception { + protected String indexQuerySet(final SearchEngine searchEngine, final String name, final String description, final String sampling, Map queries) throws Exception { LOGGER.info("Indexing {} queries for query set {}", queries.size(), name); @@ -73,15 +74,6 @@ protected String indexQuerySet(final OpenSearchClient client, final String name, // TODO: Create a mapping for the query set index. - final IndexData indexData = new IndexData("Document 1", "Text for document 1"); - - final IndexRequest indexRequest = new IndexRequest.Builder().index(Constants.QUERY_SETS_INDEX_NAME) - .id(querySetId) - .document(indexData) - .source(querySet); - - client.index(indexRequest); -// // final IndexRequest indexRequest = new IndexRequest().index(Constants.QUERY_SETS_INDEX_NAME) // .id(querySetId) // .source(querySet) diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AllQueriesQuerySampler.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AllQueriesQuerySampler.java index e9d4401..7e5e615 100644 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AllQueriesQuerySampler.java +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AllQueriesQuerySampler.java @@ -10,8 +10,8 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; -import org.opensearch.client.node.NodeClient; import org.opensearch.eval.Constants; +import org.opensearch.eval.engine.SearchEngine; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; @@ -26,15 +26,15 @@ public class AllQueriesQuerySampler extends AbstractQuerySampler { public static final String NAME = "none"; - private final NodeClient client; + private final SearchEngine searchEngine; private final AllQueriesQuerySamplerParameters parameters; /** * Creates a new sampler. - * @param client The OpenSearch {@link NodeClient client}. + * @param searchEngine The OpenSearch {@link SearchEngine engine}. */ - public AllQueriesQuerySampler(final NodeClient client, final AllQueriesQuerySamplerParameters parameters) { - this.client = client; + public AllQueriesQuerySampler(final SearchEngine searchEngine, final AllQueriesQuerySamplerParameters parameters) { + this.searchEngine = searchEngine; this.parameters = parameters; } @@ -72,7 +72,7 @@ public String sample() throws Exception { } - return indexQuerySet(client, parameters.getName(), parameters.getDescription(), parameters.getSampling(), queries); + return indexQuerySet(searchEngine, parameters.getName(), parameters.getDescription(), parameters.getSampling(), queries); } diff --git a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeAbstractQuerySampler.java b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeAbstractQuerySampler.java index 112494c..82939b7 100644 --- a/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeAbstractQuerySampler.java +++ b/opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeAbstractQuerySampler.java @@ -13,9 +13,9 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchScrollRequest; -import org.opensearch.client.node.NodeClient; import org.opensearch.common.unit.TimeValue; import org.opensearch.eval.Constants; +import org.opensearch.eval.engine.SearchEngine; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.Scroll; import org.opensearch.search.SearchHit; @@ -39,16 +39,16 @@ public class ProbabilityProportionalToSizeAbstractQuerySampler extends AbstractQ private static final Logger LOGGER = LogManager.getLogger(ProbabilityProportionalToSizeAbstractQuerySampler.class); - private final NodeClient client; + private final SearchEngine searchEngine; private final ProbabilityProportionalToSizeParameters parameters; /** * Creates a new PPTSS sampler. - * @param client The OpenSearch {@link NodeClient client}. + * @param searchEngine The OpenSearch {@link SearchEngine engine}. * @param parameters The {@link ProbabilityProportionalToSizeParameters parameters} for the sampling. */ - public ProbabilityProportionalToSizeAbstractQuerySampler(final NodeClient client, final ProbabilityProportionalToSizeParameters parameters) { - this.client = client; + public ProbabilityProportionalToSizeAbstractQuerySampler(final SearchEngine searchEngine, final ProbabilityProportionalToSizeParameters parameters) { + this.searchEngine = searchEngine; this.parameters = parameters; } @@ -165,7 +165,7 @@ public String sample() throws Exception { } - return indexQuerySet(client, parameters.getName(), parameters.getDescription(), parameters.getSampling(), querySet); + return indexQuerySet(searchEngine, parameters.getName(), parameters.getDescription(), parameters.getSampling(), querySet); }