diff --git a/opensearch-search-quality-evaluation-plugin/Dockerfile b/opensearch-search-quality-evaluation-plugin/Dockerfile index 685718d..02f56c8 100644 --- a/opensearch-search-quality-evaluation-plugin/Dockerfile +++ b/opensearch-search-quality-evaluation-plugin/Dockerfile @@ -2,5 +2,5 @@ FROM opensearchproject/opensearch:2.18.0 RUN /usr/share/opensearch/bin/opensearch-plugin install --batch https://github.com/opensearch-project/user-behavior-insights/releases/download/2.18.0.2/opensearch-ubi-2.18.0.2.zip -ADD ./build/distributions/search-quality-evaluation-plugin-2.18.0.0.zip /tmp/search-quality-evaluation-plugin.zip +ADD ./build/distributions/search-quality-evaluation-plugin-0.0.1.zip /tmp/search-quality-evaluation-plugin.zip RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/search-quality-evaluation-plugin.zip diff --git a/opensearch-search-quality-evaluation-plugin/gradle.properties b/opensearch-search-quality-evaluation-plugin/gradle.properties index 4f76f87..2659a68 100644 --- a/opensearch-search-quality-evaluation-plugin/gradle.properties +++ b/opensearch-search-quality-evaluation-plugin/gradle.properties @@ -1,2 +1,2 @@ opensearchVersion = 2.18.0 -evalVersion = 2.18.0.0 +evalVersion = 0.0.1 diff --git a/opensearch-search-quality-evaluation-plugin/scripts/get-click-through-rates.sh b/opensearch-search-quality-evaluation-plugin/scripts/get-click-through-rates.sh new file mode 100755 index 0000000..16377a2 --- /dev/null +++ b/opensearch-search-quality-evaluation-plugin/scripts/get-click-through-rates.sh @@ -0,0 +1,3 @@ +#!/bin/bash -e + +curl -s "http://localhost:9200/click_through_rates/_search" | jq diff --git a/opensearch-search-quality-evaluation-plugin/scripts/get-rank-aggregated-clickthrough.sh b/opensearch-search-quality-evaluation-plugin/scripts/get-rank-aggregated-clickthrough.sh new file mode 100755 index 0000000..639a012 --- /dev/null +++ b/opensearch-search-quality-evaluation-plugin/scripts/get-rank-aggregated-clickthrough.sh @@ -0,0 +1,3 @@ +#!/bin/bash -e + +curl -s "http://localhost:9200/rank_aggregated_ctr/_search" | jq diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java index 6f4245f..381d2ee 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java @@ -46,7 +46,6 @@ import java.util.LinkedList; import java.util.Map; import java.util.Set; -import java.util.UUID; public class CoecClickModel extends ClickModel { @@ -346,16 +345,28 @@ public Map getRankAggregatedClickThrough() throws Exception { for(final Integer x : clickCounts.keySet()) { - if(!(impressionCounts.get(x) == null)) { + if(impressionCounts.get(x) != null) { - // Calculate the CTR by dividing the number of clicks by the number of impressions. - LOGGER.debug("Position = {}, Click Count = {}, Event Count = {}", x, clickCounts.get(x), impressionCounts.get(x)); - rankAggregatedClickThrough.put(x, clickCounts.get(x) / impressionCounts.get(x)); + if(clickCounts.get(x) != null) { + + // Calculate the CTR by dividing the number of clicks by the number of impressions. + LOGGER.debug("Position = {}, Impression Counts = {}, Click Count = {}", x, impressionCounts.get(x), clickCounts.get(x)); + rankAggregatedClickThrough.put(x, clickCounts.get(x) / impressionCounts.get(x)); + + } else { + + // This document has impressions but no clicks so it's CTR is zero. + LOGGER.debug("Position = {}, Impression Counts = {}, Click Count = {}", x, clickCounts.get(x), clickCounts.get(x)); + rankAggregatedClickThrough.put(x, 0.0); + + } } else { + // This will happen in the case where a document has a "click" event but not an "impression." This // likely should not happen, but we will protect against an NPE anyway by setting the CTR to zero. rankAggregatedClickThrough.put(x, (double) 0); + } } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java index 4b8cf4c..8832c72 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/opensearch/OpenSearchHelper.java @@ -267,12 +267,12 @@ public void indexClickthroughRates(final Map> clic final BulkRequest request = new BulkRequest(); - for(final String queryId : clickthroughRates.keySet()) { + for(final String userQuery : clickthroughRates.keySet()) { - for(final ClickthroughRate clickthroughRate : clickthroughRates.get(queryId)) { + for(final ClickthroughRate clickthroughRate : clickthroughRates.get(userQuery)) { final Map jsonMap = new HashMap<>(); - jsonMap.put("query_id", queryId); + jsonMap.put("user_query", userQuery); jsonMap.put("clicks", clickthroughRate.getClicks()); jsonMap.put("events", clickthroughRate.getEvents()); jsonMap.put("ctr", clickthroughRate.getClickthroughRate());