From d5f8dad0c8815932f86cb4d6d86621655d9d7362 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Mon, 20 Jan 2025 08:32:25 +0100 Subject: [PATCH] SQ various fixes + Unit Tests --- .../analytics/common/InQueryCteFilter.java | 26 +- .../AbstractJdbcEventAnalyticsManager.java | 5 +- .../data/JdbcEnrollmentAnalyticsManager.java | 163 ++-- .../event/data/JdbcEventAnalyticsManager.java | 2 +- ...efaultProgramIndicatorSubqueryBuilder.java | 4 +- .../dhis/analytics/util/sql/Condition.java | 3 +- .../EnrollmentAnalyticsManagerCteTest.java | 747 ++++++++++++++++++ 7 files changed, 897 insertions(+), 53 deletions(-) create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsManagerCteTest.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/InQueryCteFilter.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/InQueryCteFilter.java index 2fbbb97e7bd..3c7ca2bbcf1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/InQueryCteFilter.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/InQueryCteFilter.java @@ -30,7 +30,10 @@ import static org.hisp.dhis.analytics.QueryKey.NV; import java.util.List; +import java.util.Objects; import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.hisp.dhis.common.QueryFilter; /** Mimics the logic of @{@link org.hisp.dhis.common.InQueryFilter} to be used in CTEs */ @@ -59,10 +62,15 @@ public String getSqlFilter(int offset) { StringBuilder condition = new StringBuilder(); String alias = cteDefinition.getAlias(offset); if (hasNonMissingValue(filterItems)) { - // TODO GIUSEPPE! - + condition + .append("%s.%s in".formatted(alias, field)) + .append( + streamOfNonMissingValues(filterItems) + .filter(Objects::nonNull) + .map(this::quoteIfNecessary) + .collect(Collectors.joining(",", " (", ")"))); if (hasMissingValue(filterItems)) { - + throw new UnsupportedOperationException("Not implemented yet"); // TODO GIUSEPPE! } } else { @@ -124,4 +132,16 @@ private boolean anyMatch(List filterItems, Predicate predi) { private boolean hasMissingValue(List filterItems) { return anyMatch(filterItems, this::isMissingItem); } + + private Stream streamOfNonMissingValues(List filterItems) { + return filterItems.stream().filter(this::isNotMissingItem); + } + + private String quoteIfNecessary(String item) { + return isText ? quote(item) : item; + } + + protected String quote(String filterItem) { + return "'" + filterItem + "'"; + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index fdcbe004bcd..285975475b2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -1440,7 +1440,7 @@ protected List getSelectColumnsWithCTE(EventQueryParams params, CteConte } } else if (queryItem.hasProgramStage()) { // Handle program stage items with CTE - columns.add(getColumnWithCte(queryItem, "", cteContext)); + columns.add(getColumnWithCte(queryItem, cteContext)); } else { // Handle other types as before ColumnAndAlias columnAndAlias = getColumnAndAlias(queryItem, false, ""); @@ -1462,7 +1462,8 @@ protected boolean useExperimentalAnalyticsQueryEngine() { */ protected abstract String getSelectClause(EventQueryParams params); - protected abstract String getColumnWithCte(QueryItem item, String suffix, CteContext cteContext); + /** Returns the column name associated with the CTE */ + protected abstract String getColumnWithCte(QueryItem item, CteContext cteContext); /** * Generates the SQL for the from-clause. Generally this means which analytics table to get data diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index 8561618a548..a695084990b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -160,13 +160,12 @@ public void getEnrollments(EventQueryParams params, Grid grid, int maxLimit) { ? buildAggregatedEnrollmentQueryWithCte(grid.getHeaders(), params) : getAggregatedEnrollmentsSql(grid.getHeaders(), params); } else { - // getAggregatedEnrollmentsSql sql = useExperimentalAnalyticsQueryEngine() ? buildEnrollmentQueryWithCte(params) : getAggregatedEnrollmentsSql(params, maxLimit); } - + System.out.println(sql); if (params.analyzeOnly()) { withExceptionHandling( () -> executionPlanStore.addExecutionPlan(params.getExplainOrderId(), sql)); @@ -519,56 +518,127 @@ private String addFiltersToWhereClause(EventQueryParams params) { return getQueryItemsAndFiltersWhereClause(params, new SqlHelper()); } - private String addCteFiltersToWhereClause(EventQueryParams params, CteContext cteContext) { - StringBuilder cteWhereClause = new StringBuilder(); - Set processedItems = new HashSet<>(); // Track processed items + /** + * Builds a WHERE clause by combining CTE filters and non-CTE filters for event queries. + * + * @param params The event query parameters containing items and filters + * @param cteContext The CTE context containing CTE definitions + * @return A Condition representing the combined WHERE clause + * @throws IllegalArgumentException if params or cteContext is null + */ + private Condition addCteFiltersToWhereClause(EventQueryParams params, CteContext cteContext) { + if (params == null || cteContext == null) { + throw new IllegalArgumentException("Query parameters and CTE context cannot be null"); + } + + Set processedItems = new HashSet<>(); + + // Build CTE conditions + Condition cteConditions = buildCteConditions(params, cteContext, processedItems); + + // Get non-CTE conditions + String nonCteWhereClause = + getQueryItemsAndFiltersWhereClause(params, processedItems, new SqlHelper()) + .replace("where", ""); + + // Combine conditions + if (!nonCteWhereClause.isEmpty()) { + return cteConditions != null + ? Condition.and(cteConditions, Condition.raw(nonCteWhereClause)) + : Condition.raw(nonCteWhereClause); + } + + return cteConditions; + } + + /** + * Builds conditions for CTE filters. + * + * @param params The event query parameters + * @param cteContext The CTE context + * @param processedItems Set to track processed items + * @return Combined condition for CTE filters + */ + private Condition buildCteConditions( + EventQueryParams params, CteContext cteContext, Set processedItems) { + + List conditions = new ArrayList<>(); - // Get all filters from the query items and item filters List filters = Stream.concat(params.getItems().stream(), params.getItemFilters().stream()) .filter(QueryItem::hasFilter) .toList(); - // Iterate over each filter and apply the correct condition + for (QueryItem item : filters) { String cteName = CteUtils.computeKey(item); if (cteContext.containsCte(cteName)) { - processedItems.add(item); // Mark item as processed - CteDefinition cteDef = cteContext.getDefinitionByItemUid(cteName); - for (QueryFilter filter : item.getFilters()) { - if (IN.equals(filter.getOperator())) { - InQueryCteFilter inQueryCteFilter = - new InQueryCteFilter("value", filter.getFilter(), item.isText(), cteDef); - cteWhereClause - .append(" and ") - .append( - inQueryCteFilter.getSqlFilter( - computeRowNumberOffset(item.getProgramStageOffset()))); - } else { - String value = getSqlFilterValue(filter, item); - - cteWhereClause - .append(" and ") - .append(cteDef.getAlias()) - .append(".value ") - .append("NULL".equals(value) ? "is" : filter.getSqlOperator()) - .append(" ") - .append(value); - } - } + processedItems.add(item); + conditions.addAll(buildItemConditions(item, cteContext.getDefinitionByItemUid(cteName))); } } - // Add filters for items that are not part of the CTE - String nonCteWhereClause = - getQueryItemsAndFiltersWhereClause(params, processedItems, new SqlHelper()) - .replace("where", ""); - if (nonCteWhereClause.isEmpty()) return cteWhereClause.toString(); - String currentWhereClause = cteWhereClause.toString().toLowerCase().trim(); - cteWhereClause.append( - currentWhereClause.endsWith("and") ? nonCteWhereClause : " and " + nonCteWhereClause); + return conditions.isEmpty() ? null : Condition.and(conditions.toArray(new Condition[0])); + } + + /** + * Builds conditions for a single query item. + * + * @param item The query item + * @param cteDef The CTE definition + * @return List of conditions for the item + */ + private List buildItemConditions(QueryItem item, CteDefinition cteDef) { + return item.getFilters().stream() + .map(filter -> buildFilterCondition(filter, item, cteDef)) + .collect(Collectors.toList()); + } + + /** + * Builds a condition for a single filter. + * + * @param filter The query filter + * @param item The query item + * @param cteDef The CTE definition + * @return Condition for the filter + */ + private Condition buildFilterCondition(QueryFilter filter, QueryItem item, CteDefinition cteDef) { + return IN.equals(filter.getOperator()) + ? buildInFilterCondition(filter, item, cteDef) + : buildStandardFilterCondition(filter, item, cteDef); + } + + /** + * Builds a condition for an IN filter. + * + * @param filter The IN query filter + * @param item The query item + * @param cteDef The CTE definition + * @return Condition for the IN filter + */ + private Condition buildInFilterCondition( + QueryFilter filter, QueryItem item, CteDefinition cteDef) { + InQueryCteFilter inQueryCteFilter = + new InQueryCteFilter("value", filter.getFilter(), item.isText(), cteDef); + + return Condition.raw( + inQueryCteFilter.getSqlFilter(computeRowNumberOffset(item.getProgramStageOffset()))); + } + + /** + * Builds a condition for a standard (non-IN) filter. + * + * @param filter The query filter + * @param item The query item + * @param cteDef The CTE definition + * @return Condition for the standard filter + */ + private Condition buildStandardFilterCondition( + QueryFilter filter, QueryItem item, CteDefinition cteDef) { + String value = getSqlFilterValue(filter, item); + String operator = "NULL".equals(value) ? "is" : filter.getSqlOperator(); - return cteWhereClause.toString(); + return Condition.raw(String.format("%s.value %s %s", cteDef.getAlias(), operator, value)); } private String getSqlFilterValue(QueryFilter filter, QueryItem item) { @@ -729,12 +799,19 @@ protected ColumnAndAlias getCoordinateColumn(QueryItem item, String suffix) { } @Override - protected String getColumnWithCte(QueryItem item, String suffix, CteContext cteContext) { + protected String getColumnWithCte(QueryItem item, CteContext cteContext) { List columns = new ArrayList<>(); + // Get the CTE definition for the item CteDefinition cteDef = cteContext.getDefinitionByItemUid(computeKey(item)); + if (cteDef == null) { + throw new IllegalArgumentException("CTE definition not found for item: " + item); + } int programStageOffset = computeRowNumberOffset(item.getProgramStageOffset()); - String alias = getAlias(item).orElse(null); + // calculate the alias for the column + // if the item is not a repeatable stage, the alias is the program stage + item name + String alias = + getAlias(item).orElse("%s.%s".formatted(item.getProgramStage().getUid(), item.getItemId())); columns.add("%s.value as %s".formatted(cteDef.getAlias(programStageOffset), quote(alias))); if (cteDef.isRowContext()) { // Add additional status and exists columns for row context @@ -1164,8 +1241,6 @@ private String buildMainCteSql( * * @param cteContext the {@link CteContext} to which the new CTE definition(s) will be added * @param item the {@link QueryItem} containing program-stage details - * @param params the {@link EventQueryParams}, used for checking row-context eligibility, offsets, - * etc. * @param eventTableName the event table name * @param colName the quoted column name for the item */ @@ -1503,7 +1578,7 @@ private void addFromClause(SelectBuilder sb, EventQueryParams params) { */ private void addWhereClause(SelectBuilder sb, EventQueryParams params, CteContext cteContext) { Condition baseConditions = Condition.raw(getWhereClause(params)); - Condition cteConditions = Condition.raw(addCteFiltersToWhereClause(params, cteContext)); + Condition cteConditions = addCteFiltersToWhereClause(params, cteContext); sb.where(Condition.and(baseConditions, cteConditions)); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java index 1a2b084a560..ea41bfbbe90 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java @@ -396,7 +396,7 @@ private String getCoordinateSelectExpression(EventQueryParams params) { } @Override - protected String getColumnWithCte(QueryItem item, String suffix, CteContext cteContext) { + protected String getColumnWithCte(QueryItem item, CteContext cteContext) { // TODO: Implement return ""; } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java index bfd2cb06050..855d44939f6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java @@ -116,7 +116,7 @@ public void contributeCte( earliestStartDate, latestDate) // FIXME this is a bit of an hack - .replace("subax\\.", ""); + .replace("subax.", ""); filter = "where " + piResolvedSqlFilter; } @@ -128,7 +128,7 @@ public void contributeCte( earliestStartDate, latestDate) // FIXME this is a bit of an hack - .replace("subax\\.", ""); + .replace("subax.", ""); String cteSql = "select enrollment, %s(%s) as value from %s %s group by enrollment" diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/sql/Condition.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/sql/Condition.java index 9293d0644f2..95b5721d18b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/sql/Condition.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/sql/Condition.java @@ -31,6 +31,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; /** @@ -145,7 +146,7 @@ static Condition raw(String sql) { * @return a new And condition containing all provided conditions */ static Condition and(Condition... conditions) { - return new And(Arrays.asList(conditions)); + return new And(Arrays.asList(conditions).stream().filter(Objects::nonNull).toList()); } /** diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsManagerCteTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsManagerCteTest.java new file mode 100644 index 00000000000..f88cc9dadf1 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EnrollmentAnalyticsManagerCteTest.java @@ -0,0 +1,747 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.event.data; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hisp.dhis.analytics.DataType.NUMERIC; +import static org.hisp.dhis.analytics.QueryKey.NV; +import static org.hisp.dhis.common.DimensionalObject.OPTION_SEP; +import static org.hisp.dhis.common.QueryOperator.EQ; +import static org.hisp.dhis.common.QueryOperator.IN; +import static org.hisp.dhis.common.QueryOperator.NEQ; +import static org.hisp.dhis.test.TestBase.createProgram; +import static org.hisp.dhis.test.TestBase.createProgramIndicator; +import static org.hisp.dhis.test.TestBase.getDate; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.function.Consumer; +import org.hisp.dhis.analytics.TimeField; +import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; +import org.hisp.dhis.analytics.event.EventQueryParams; +import org.hisp.dhis.analytics.event.data.programindicator.DefaultProgramIndicatorSubqueryBuilder; +import org.hisp.dhis.common.Grid; +import org.hisp.dhis.common.QueryOperator; +import org.hisp.dhis.common.ValueType; +import org.hisp.dhis.db.sql.PostgreSqlBuilder; +import org.hisp.dhis.db.sql.SqlBuilder; +import org.hisp.dhis.program.Program; +import org.hisp.dhis.program.ProgramIndicator; +import org.hisp.dhis.program.ProgramIndicatorService; +import org.hisp.dhis.relationship.RelationshipConstraint; +import org.hisp.dhis.relationship.RelationshipEntity; +import org.hisp.dhis.relationship.RelationshipType; +import org.hisp.dhis.setting.SystemSettings; +import org.hisp.dhis.setting.SystemSettingsService; +import org.hisp.dhis.system.grid.ListGrid; +import org.hisp.dhis.test.random.BeanRandomizer; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.springframework.jdbc.BadSqlGrammarException; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.support.rowset.SqlRowSet; + +/** + * @author Luciano Fiandesio + */ +@MockitoSettings(strictness = Strictness.LENIENT) +@ExtendWith(MockitoExtension.class) +class EnrollmentAnalyticsManagerCteTest extends EventAnalyticsTest { + private JdbcEnrollmentAnalyticsManager subject; + + @Mock private JdbcTemplate jdbcTemplate; + + @Mock private ExecutionPlanStore executionPlanStore; + + @Mock private SqlRowSet rowSet; + + @Mock private ProgramIndicatorService programIndicatorService; + + @Spy private SqlBuilder sqlBuilder = new PostgreSqlBuilder(); + + @Mock private SystemSettingsService systemSettingsService; + + @Spy + private EnrollmentTimeFieldSqlRenderer enrollmentTimeFieldSqlRenderer = + new EnrollmentTimeFieldSqlRenderer(sqlBuilder); + + @Spy private SystemSettings systemSettings; + + @Captor private ArgumentCaptor sql; + + private String DEFAULT_COLUMNS = + "enrollment,trackedentity,enrollmentdate,occurreddate,storedby," + + "createdbydisplayname" + + "," + + "lastupdatedbydisplayname" + + ",lastupdated,ST_AsGeoJSON(enrollmentgeometry),longitude,latitude,ouname,ounamehierarchy,oucode,enrollmentstatus"; + + private final BeanRandomizer rnd = BeanRandomizer.create(); + + @BeforeEach + public void setUp() { + when(jdbcTemplate.queryForRowSet(anyString())).thenReturn(this.rowSet); + when(systemSettingsService.getCurrentSettings()).thenReturn(systemSettings); + when(systemSettings.getUseExperimentalAnalyticsQueryEngine()).thenReturn(true); + DefaultProgramIndicatorSubqueryBuilder programIndicatorSubqueryBuilder = + new DefaultProgramIndicatorSubqueryBuilder(programIndicatorService, systemSettingsService); + + subject = + new JdbcEnrollmentAnalyticsManager( + jdbcTemplate, + programIndicatorService, + programIndicatorSubqueryBuilder, + enrollmentTimeFieldSqlRenderer, + executionPlanStore, + systemSettingsService, + sqlBuilder); + } + + @Test + void verifyWithProgramAndStartEndDate() { + EventQueryParams params = + new EventQueryParams.Builder(createRequestParams()) + .withStartDate(getDate(2017, 1, 1)) + .withEndDate(getDate(2017, 12, 31)) + .build(); + + Grid grid = new ListGrid(); + int unlimited = 0; + + subject.getEnrollments(params, grid, unlimited); + + verify(jdbcTemplate).queryForRowSet(sql.capture()); + + String expected = + "ax.\"quarterly\",ax.\"ou\" from " + + getTable(programA.getUid()) + + " as ax where (((enrollmentdate >= '2017-01-01' and enrollmentdate < '2018-01-01'))) and (ax.\"uidlevel1\" = 'ouabcdefghA' ) "; + + assertSql(sql.getValue(), expected); + assertTrue(grid.hasLastDataRow()); + } + + @Test + void verifyWithLastUpdatedTimeField() { + EventQueryParams params = + new EventQueryParams.Builder(createRequestParams()) + .withStartDate(getDate(2017, 1, 1)) + .withEndDate(getDate(2017, 12, 31)) + .withTimeField(TimeField.LAST_UPDATED.name()) + .build(); + + subject.getEnrollments(params, new ListGrid(), 10000); + + verify(jdbcTemplate).queryForRowSet(sql.capture()); + + String expected = + "ax.\"quarterly\",ax.\"ou\" from " + + getTable(programA.getUid()) + + " as ax where (((lastupdated >= '2017-01-01' and lastupdated < '2018-01-01'))) and (ax.\"uidlevel1\" = 'ouabcdefghA' ) limit 10001"; + + assertSql(sql.getValue(), expected); + } + + @Test + void verifyWithRepeatableProgramStageAndNumericDataElement() { + verifyWithRepeatableProgramStageAndDataElement(ValueType.NUMBER); + } + + @Test + void verifyWithRepeatableProgramStageAndTextDataElement() { + verifyWithRepeatableProgramStageAndDataElement(ValueType.TEXT); + } + + @Test + void verifyWithProgramStageAndTextDataElement() { + verifyWithProgramStageAndDataElement(ValueType.TEXT); + } + + @Test + void verifyWithProgramStageAndNumericDataElement() { + verifyWithProgramStageAndDataElement(ValueType.NUMBER); + } + + private void verifyWithProgramStageAndDataElement(ValueType valueType) { + EventQueryParams params = createRequestParams(this.programStage, valueType); + + subject.getEnrollments(params, new ListGrid(), 100); + + verify(jdbcTemplate).queryForRowSet(sql.capture()); + + String subSelect = + "(select \"fWIAEtYVEGk\" from analytics_event_" + + programA.getUid() + + " where analytics_event_" + + programA.getUid() + + ".eventstatus != 'SCHEDULE' and analytics_event_" + + programA.getUid() + + ".enrollment = ax.enrollment and \"fWIAEtYVEGk\" is not null and ps = '" + + programStage.getUid() + + "' order by occurreddate desc, created desc limit 1 )"; + + if (valueType == ValueType.NUMBER) { + subSelect = subSelect + " as \"fWIAEtYVEGk\""; + } + String expected = + "ax.\"quarterly\",ax.\"ou\"," + + subSelect + + " from " + + getTable(programA.getUid()) + + " as ax where (ax.\"quarterly\" in ('2000Q1') ) and (ax.\"uidlevel1\" = 'ouabcdefghA' ) " + + "and ps = '" + + programStage.getUid() + + "' limit 101"; + + assertSql(sql.getValue(), expected); + } + + private void verifyWithRepeatableProgramStageAndDataElement(ValueType valueType) { + EventQueryParams params = createRequestParams(repeatableProgramStage, valueType); + + subject.getEnrollments(params, new ListGrid(), 100); + + verify(jdbcTemplate).queryForRowSet(sql.capture()); + + String programUid = repeatableProgramStage.getProgram().getUid(); + + String programStageUid = repeatableProgramStage.getUid(); + + String dataElementUid = dataElementA.getUid(); + + String expected = + "select enrollment,trackedentity,enrollmentdate,occurreddate,storedby,createdbydisplayname,lastupdatedbydisplayname,lastupdated,ST_AsGeoJSON(enrollmentgeometry),longitude,latitude," + + "ouname,ounamehierarchy,oucode,enrollmentstatus,ax.\"quarterly\",ax.\"ou\"," + + "(select \"" + + dataElementUid + + "\" from analytics_event_" + + programUid + + " where analytics_event_" + + programUid + + ".eventstatus != 'SCHEDULE' and analytics_event_" + + programUid + + ".enrollment = ax.enrollment and ps = '" + + repeatableProgramStage.getUid() + + "' order by occurreddate desc, created desc offset 1 limit 1 ) " + + "as \"" + + programStageUid + + "[-1]." + + dataElementUid + + "\", exists ((select \"" + + dataElementUid + + "\" " + + "from analytics_event_" + + programUid + + " where analytics_event_" + + programUid + + ".eventstatus != 'SCHEDULE' and analytics_event_" + + programUid + + ".enrollment = ax.enrollment and ps = '" + + programStageUid + + "' order by occurreddate desc, created desc offset 1 limit 1 )) " + + "as \"" + + programStageUid + + "[-1]." + + dataElementUid + + ".exists\"" + + ",(select eventstatus " + + "from analytics_event_" + + programUid + + " where analytics_event_" + + programUid + + ".eventstatus != 'SCHEDULE' and analytics_event_" + + programUid + + ".enrollment = ax.enrollment and ps = '" + + programStageUid + + "' order by occurreddate desc, created desc offset 1 limit 1 ) " + + "as \"" + + programStageUid + + "[-1]." + + dataElementUid + + ".status\" " + + "from analytics_enrollment_" + + programUid + + " as ax where (ax.\"quarterly\" in ('2000Q1') ) and (ax.\"uidlevel1\" = 'ouabcdefghA' ) " + + "and ps = '" + + programStageUid + + "' limit 101"; + + assertEquals(expected, sql.getValue()); + } + + @Test + void verifyWithProgramStageAndTextualDataElementAndFilter() { + EventQueryParams params = createRequestParamsWithFilter(programStage, ValueType.TEXT); + + subject.getEnrollments(params, new ListGrid(), 10000); + + verify(jdbcTemplate).queryForRowSet(sql.capture()); + + String subSelect = + "(select \"fWIAEtYVEGk\" from analytics_event_" + + programA.getUid() + + " where analytics_event_" + + programA.getUid() + + ".eventstatus != 'SCHEDULE' and analytics_event_" + + programA.getUid() + + ".enrollment = ax.enrollment and \"fWIAEtYVEGk\" is not null and ps = '" + + programStage.getUid() + + "' order by occurreddate desc, created desc limit 1 )"; + + String expected = + "ax.\"quarterly\",ax.\"ou\"," + + subSelect + + " from " + + getTable(programA.getUid()) + + " as ax where (ax.\"quarterly\" in ('2000Q1') ) and (ax.\"uidlevel1\" = 'ouabcdefghA' ) " + + "and ps = '" + + programStage.getUid() + + "' and " + + subSelect + + " > '10' limit 10001"; + + assertSql(sql.getValue(), expected); + } + + @Test + void verifyGetEventsWithProgramStatusParam() { + mockEmptyRowSet(); + + EventQueryParams params = createRequestParamsWithStatuses(); + + subject.getEnrollments(params, new ListGrid(), 10000); + + verify(jdbcTemplate).queryForRowSet(sql.capture()); + + String expected = + "ax.\"quarterly\",ax.\"ou\" from " + + getTable(programA.getUid()) + + " as ax where (ax.\"quarterly\" in ('2000Q1') ) and (ax.\"uidlevel1\" = 'ouabcdefghA' )" + + " and enrollmentstatus in ('ACTIVE','COMPLETED') limit 10001"; + + assertSql(sql.getValue(), expected); + } + + @Test + void testBadGrammarExceptionNonMultipleQueries() { + // Given + mockEmptyRowSet(); + EventQueryParams params = createRequestParamsWithStatuses(); + when(jdbcTemplate.queryForRowSet(anyString())).thenThrow(BadSqlGrammarException.class); + + // Then + assertThrows( + BadSqlGrammarException.class, () -> subject.getEnrollments(params, new ListGrid(), 10000)); + } + + @Test + void testBadGrammarExceptionWithMultipleQueries() { + // Given + mockEmptyRowSet(); + EventQueryParams params = createRequestParamsWithMultipleQueries(); + SQLException sqlException = new SQLException("Some exception", "HY000"); + BadSqlGrammarException badSqlGrammarException = + new BadSqlGrammarException("task", "select * from nothing", sqlException); + when(jdbcTemplate.queryForRowSet(anyString())).thenThrow(badSqlGrammarException); + + // Then + assertDoesNotThrow(() -> subject.getEnrollments(params, new ListGrid(), 10000)); + } + + @Test + void verifyWithProgramStageAndNumericDataElementAndFilter2() { + EventQueryParams params = createRequestParamsWithFilter(programStage, ValueType.NUMBER); + + subject.getEnrollments(params, new ListGrid(), 10000); + + verify(jdbcTemplate).queryForRowSet(sql.capture()); + + String subSelect = + "(select \"fWIAEtYVEGk\" from analytics_event_" + + programA.getUid() + + " where analytics_event_" + + programA.getUid() + + ".eventstatus != 'SCHEDULE' and analytics_event_" + + programA.getUid() + + ".enrollment = ax.enrollment and \"fWIAEtYVEGk\" is not null and ps = '" + + programStage.getUid() + + "' order by occurreddate desc, created desc limit 1 )"; + + String expected = + "ax.\"quarterly\",ax.\"ou\"," + + subSelect + + " as \"fWIAEtYVEGk\"" + + " from " + + getTable(programA.getUid()) + + " as ax where (ax.\"quarterly\" in ('2000Q1') ) and (ax.\"uidlevel1\" = 'ouabcdefghA' ) " + + "and ps = '" + + programStage.getUid() + + "' and " + + subSelect + + " > '10' limit 10001"; + + assertSql(sql.getValue(), expected); + } + + @Test + void verifyGetEnrollmentsWithMissingValueEqFilter() { + String subSelect = + "(select \"fWIAEtYVEGk\" from analytics_event_" + + programA.getUid() + + " where analytics_event_" + + programA.getUid() + + ".eventstatus != 'SCHEDULE' and analytics_event_" + + programA.getUid() + + ".enrollment = ax.enrollment and \"fWIAEtYVEGk\" is not null and ps = '" + + programStage.getUid() + + "' order by occurreddate desc, created desc limit 1 )"; + + String expected = subSelect + " is null"; + + testIt( + EQ, + NV, + Collections.singleton((capturedSql) -> assertThat(capturedSql, containsString(expected)))); + } + + @Test + void verifyGetEnrollmentsWithMissingValueNeqFilter() { + String subSelect = + "(select \"fWIAEtYVEGk\" from analytics_event_" + + programA.getUid() + + " where analytics_event_" + + programA.getUid() + + ".eventstatus != 'SCHEDULE' and analytics_event_" + + programA.getUid() + + ".enrollment = ax.enrollment and \"fWIAEtYVEGk\" is not null and ps = '" + + programStage.getUid() + + "' order by occurreddate desc, created desc limit 1 )"; + + String expected = subSelect + " is not null"; + testIt( + NEQ, + NV, + Collections.singleton((capturedSql) -> assertThat(capturedSql, containsString(expected)))); + } + + @Test + void verifyGetEnrollmentsWithMissingValueAndNumericValuesInFilter() { + String subSelect = + "(select \"fWIAEtYVEGk\" from analytics_event_" + + programA.getUid() + + " where analytics_event_" + + programA.getUid() + + ".eventstatus != 'SCHEDULE' and analytics_event_" + + programA.getUid() + + ".enrollment = ax.enrollment and \"fWIAEtYVEGk\" is not null and ps = '" + + programStage.getUid() + + "' order by occurreddate desc, created desc limit 1 )"; + + String numericValues = String.join(OPTION_SEP, "10", "11", "12"); + String expected = + "(" + + subSelect + + " in (" + + String.join(",", numericValues.split(OPTION_SEP)) + + ") or (" + + subSelect + + " is null and exists(" + + subSelect + + ")))"; + testIt( + IN, + numericValues + OPTION_SEP + NV, + Collections.singleton((capturedSql) -> assertThat(capturedSql, containsString(expected)))); + } + + @Test + void verifyGetEnrollmentsWithoutMissingValueAndNumericValuesInFilter() { + String cte = + noEof( + """ + ( select enrollment, + "fWIAEtYVEGk" as value, + row_number() over ( + partition by enrollment order by occurreddate desc, created desc ) as rn + from analytics_event_%s + where eventstatus != 'SCHEDULE' and ps = '%s' ) + """) + .formatted(programA.getUid(), programStage.getUid()); + + String numericValues = String.join(OPTION_SEP, "10", "11", "12"); + String inClause = " in (" + String.join(",", numericValues.split(OPTION_SEP)) + ")"; + + Collection> assertions = + Arrays.asList( + (sql) -> assertThat(sql, containsString(cte)), + (sql) -> { + // check that the IN clause is in the root query where condition + var whereToOrderBy = + sql.toLowerCase() + .substring( + sql.toLowerCase().lastIndexOf("where"), + sql.toLowerCase().lastIndexOf("limit")); + assertThat(whereToOrderBy, containsString(inClause)); + }, + (sql) -> assertThat(sql, containsString(inClause))); + + testIt(IN, numericValues, assertions); + } + + @Test + void verifyGetEnrollmentsWithOnlyMissingValueInFilter() { + String subSelect = + "(select \"fWIAEtYVEGk\" from analytics_event_" + + programA.getUid() + + " where analytics_event_" + + programA.getUid() + + ".eventstatus != 'SCHEDULE' and analytics_event_" + + programA.getUid() + + ".enrollment = ax.enrollment and \"fWIAEtYVEGk\" is not null and ps = '" + + programStage.getUid() + + "' order by occurreddate desc, created desc limit 1 )"; + + String expected = subSelect + " is null"; + String unexpected = "(" + subSelect + " in ("; + testIt( + IN, + NV, + List.of( + (capturedSql) -> assertThat(capturedSql, containsString(expected)), + (capturedSql) -> assertThat(capturedSql, not(containsString(unexpected))))); + } + + private void testIt( + QueryOperator operator, String filter, Collection> assertions) { + subject.getEnrollments( + createRequestParamsWithFilter(programStage, ValueType.INTEGER, operator, filter), + new ListGrid(), + 10000); + verify(jdbcTemplate).queryForRowSet(sql.capture()); + + assertions.forEach(consumer -> consumer.accept(sql.getValue())); + } + + @Test + void verifyWithProgramIndicatorAndRelationshipTypeBothSidesTrackedEntity() { + Date startDate = getDate(2015, 1, 1); + Date endDate = getDate(2017, 4, 8); + + String piSubquery = "distinct event"; + + ProgramIndicator programIndicatorA = createProgramIndicator('A', programA, "", ""); + + RelationshipType relationshipTypeA = createRelationshipType(); + + EventQueryParams.Builder params = + new EventQueryParams.Builder(createRequestParams(programIndicatorA, relationshipTypeA)) + .withStartDate(startDate) + .withEndDate(endDate); + + when(programIndicatorService.getAnalyticsSql( + "", NUMERIC, programIndicatorA, getDate(2000, 1, 1), getDate(2017, 4, 8), "subax")) + .thenReturn(piSubquery); + + subject.getEnrollments(params.build(), new ListGrid(), 100); + + verify(jdbcTemplate).queryForRowSet(sql.capture()); + + String expected = + "ax.\"quarterly\",ax.\"ou\",(SELECT avg (" + + piSubquery + + ") FROM analytics_event_" + + programA.getUid().toLowerCase() + + " as subax WHERE " + + "subax.trackedentity in (select te.uid from trackedentity te " + + "LEFT JOIN relationshipitem ri on te.trackedentityid = ri.trackedentityid " + + "LEFT JOIN relationship r on r.from_relationshipitemid = ri.relationshipitemid " + + "LEFT JOIN relationshipitem ri2 on r.to_relationshipitemid = ri2.relationshipitemid " + + "LEFT JOIN relationshiptype rty on rty.relationshiptypeid = r.relationshiptypeid " + + "LEFT JOIN trackedentity te on te.trackedentityid = ri2.trackedentityid " + + "WHERE rty.relationshiptypeid = " + + relationshipTypeA.getId() + + " AND te.uid = ax.trackedentity )) as \"" + + programIndicatorA.getUid() + + "\" " + + "from analytics_enrollment_" + + programA.getUid() + + " as ax where (((enrollmentdate >= '2015-01-01' and enrollmentdate < '2017-04-09'))) and (ax.\"uidlevel1\" = 'ouabcdefghA' ) limit 101"; + + assertSql(sql.getValue(), expected); + } + + @Test + void verifyWithProgramIndicatorAndRelationshipTypeDifferentConstraint() { + Date startDate = getDate(2015, 1, 1); + Date endDate = getDate(2017, 4, 8); + + String piSubquery = "distinct event"; + + ProgramIndicator programIndicatorA = createProgramIndicator('A', programA, "", ""); + + RelationshipType relationshipTypeA = + createRelationshipType(RelationshipEntity.PROGRAM_INSTANCE); + + EventQueryParams.Builder params = + new EventQueryParams.Builder(createRequestParams(programIndicatorA, relationshipTypeA)) + .withStartDate(startDate) + .withEndDate(endDate); + + when(programIndicatorService.getAnalyticsSql( + "", NUMERIC, programIndicatorA, getDate(2000, 1, 1), getDate(2017, 4, 8), "subax")) + .thenReturn(piSubquery); + + subject.getEnrollments(params.build(), new ListGrid(), 100); + + verify(jdbcTemplate).queryForRowSet(sql.capture()); + + String expected = + "ax.\"quarterly\",ax.\"ou\",(SELECT avg (" + + piSubquery + + ") FROM analytics_event_" + + programA.getUid().toLowerCase() + + " as subax WHERE " + + " subax.trackedentity in (select te.uid from trackedentity te LEFT JOIN relationshipitem ri on te.trackedentityid = ri.trackedentityid " + + "LEFT JOIN relationship r on r.from_relationshipitemid = ri.relationshipitemid " + + "LEFT JOIN relationshipitem ri2 on r.to_relationshipitemid = ri2.relationshipitemid " + + "LEFT JOIN relationshiptype rty on rty.relationshiptypeid = r.relationshiptypeid " + + "LEFT JOIN enrollment en on en.enrollmentid = ri2.enrollmentid WHERE rty.relationshiptypeid " + + "= " + + relationshipTypeA.getId() + + " AND en.uid = ax.enrollment ))" + + " as \"" + + programIndicatorA.getUid() + + "\" " + + "from analytics_enrollment_" + + programA.getUid() + + " as ax where (((enrollmentdate >= '2015-01-01' and enrollmentdate < '2017-04-09'))) and (ax.\"uidlevel1\" = 'ouabcdefghA' ) limit 101"; + + assertSql(sql.getValue(), expected); + } + + @Override + String getTableName() { + return "analytics_enrollment"; + } + + private RelationshipType createRelationshipType(RelationshipEntity toConstraint) { + RelationshipType relationshipTypeA = rnd.nextObject(RelationshipType.class); + + RelationshipConstraint from = new RelationshipConstraint(); + from.setRelationshipEntity(RelationshipEntity.TRACKED_ENTITY_INSTANCE); + + RelationshipConstraint to = new RelationshipConstraint(); + to.setRelationshipEntity(toConstraint); + + relationshipTypeA.setFromConstraint(from); + relationshipTypeA.setToConstraint(to); + return relationshipTypeA; + } + + private RelationshipType createRelationshipType() { + return createRelationshipType(RelationshipEntity.TRACKED_ENTITY_INSTANCE); + } + + private void assertSql(String actual, String expected) { + assertThat(actual, is("select " + DEFAULT_COLUMNS + "," + expected)); + } + + @Test + void verifyWithProgramIndicatorAndRelationshipTypeBothSidesTrackedEntity2() { + Date startDate = getDate(2015, 1, 1); + Date endDate = getDate(2017, 4, 8); + Program programB = createProgram('B'); + String piSubquery = "distinct event"; + + ProgramIndicator programIndicatorA = createProgramIndicator('A', programB, "", ""); + + RelationshipType relationshipTypeA = createRelationshipType(); + + EventQueryParams.Builder params = + new EventQueryParams.Builder(createRequestParams(programIndicatorA, relationshipTypeA)) + .withStartDate(startDate) + .withEndDate(endDate); + + when(programIndicatorService.getAnalyticsSql( + "", NUMERIC, programIndicatorA, getDate(2000, 1, 1), getDate(2017, 4, 8), "subax")) + .thenReturn(piSubquery); + + subject.getEnrollments(params.build(), new ListGrid(), 100); + + verify(jdbcTemplate).queryForRowSet(sql.capture()); + + String expected = + "ax.\"quarterly\",ax.\"ou\",(SELECT avg (" + + piSubquery + + ") FROM analytics_event_" + + programB.getUid().toLowerCase() + + " as subax WHERE " + + "subax.trackedentity in (select te.uid from trackedentity te " + + "LEFT JOIN relationshipitem ri on te.trackedentityid = ri.trackedentityid " + + "LEFT JOIN relationship r on r.from_relationshipitemid = ri.relationshipitemid " + + "LEFT JOIN relationshipitem ri2 on r.to_relationshipitemid = ri2.relationshipitemid " + + "LEFT JOIN relationshiptype rty on rty.relationshiptypeid = r.relationshiptypeid " + + "LEFT JOIN trackedentity te on te.trackedentityid = ri2.trackedentityid " + + "WHERE rty.relationshiptypeid = " + + relationshipTypeA.getId() + + " AND te.uid = ax.trackedentity )) as \"" + + programIndicatorA.getUid() + + "\" " + + "from analytics_enrollment_" + + programA.getUid() + + " as ax where (((enrollmentdate >= '2015-01-01' and enrollmentdate < '2017-04-09'))) and (ax.\"uidlevel1\" = 'ouabcdefghA' ) limit 101"; + + assertSql(sql.getValue(), expected); + } + + private String noEof(String sql) { + return sql.replaceAll("\\s+", " ").trim(); + } +}