diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java new file mode 100644 index 000000000000..c452baa01fc3 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.common; + +import lombok.experimental.UtilityClass; +import org.hisp.dhis.common.QueryItem; + +@UtilityClass +public class CTEUtils { + + public static String computeKey(QueryItem queryItem) { + + if (queryItem.hasProgramStage()) { + return "%s_%s".formatted(queryItem.getProgramStage().getUid(), queryItem.getItemId()); + } else if (queryItem.isProgramIndicator()) { + return queryItem.getItemId(); + } + + // TODO continue with the rest of the method + return ""; + } + + public static String getIdentifier(QueryItem queryItem) { + String stage = queryItem.hasProgramStage() ? queryItem.getProgramStage().getUid() : "default"; + return stage + "." + queryItem.getItemId(); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CteContext.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CteContext.java new file mode 100644 index 000000000000..72de799e1610 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CteContext.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.common; + +import static org.hisp.dhis.analytics.common.CTEUtils.computeKey; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import org.hisp.dhis.common.QueryItem; +import org.hisp.dhis.program.ProgramIndicator; +import org.hisp.dhis.program.ProgramStage; + +public class CteContext { + private final Map cteDefinitions = new LinkedHashMap<>(); + + public CteDefinition getDefinitionByItemUid(String itemUid) { + return cteDefinitions.get(itemUid); + } + + /** + * Adds a CTE definition to the context. + * + * @param programStage The program stage + * @param item The query item + * @param cteDefinition The CTE definition (the SQL query) + * @param offset The calculated offset + * @param isRowContext Whether the CTE is a row context + */ + public void addCte( + ProgramStage programStage, + QueryItem item, + String cteDefinition, + int offset, + boolean isRowContext) { + String key = computeKey(item); + if (cteDefinitions.containsKey(key)) { + cteDefinitions.get(key).getOffsets().add(offset); + } else { + var cteDef = + new CteDefinition( + programStage.getUid(), item.getItemId(), cteDefinition, offset, isRowContext); + cteDefinitions.put(key, cteDef); + } + } + + public void addExistsCte(ProgramStage programStage, QueryItem item, String cteDefinition) { + var cteDef = + new CteDefinition(programStage.getUid(), item.getItemId(), cteDefinition, -999, false) + .setExists(true); + cteDefinitions.put(programStage.getUid(), cteDef); + } + + /** + * Adds a CTE definition to the context. + * + * @param programIndicator The program indicator + * @param cteDefinition The CTE definition (the SQL query) + * @param functionRequiresCoalesce Whether the function requires to be "wrapped" in coalesce to + * avoid null values (e.g. avg, sum) + */ + public void addProgramIndicatorCte( + ProgramIndicator programIndicator, String cteDefinition, boolean functionRequiresCoalesce) { + cteDefinitions.put( + programIndicator.getUid(), + new CteDefinition(programIndicator.getUid(), cteDefinition, functionRequiresCoalesce)); + } + + public void addCteFilter(QueryItem item, String ctedefinition) { + String key = computeKey(item); + if (!cteDefinitions.containsKey(key)) { + ProgramStage programStage = item.getProgramStage(); + cteDefinitions.put( + key, + new CteDefinition( + item.getItemId(), + programStage == null ? null : programStage.getUid(), + ctedefinition, + true)); + } + } + + public String getCteDefinition() { + if (cteDefinitions.isEmpty()) { + return ""; + } + + StringBuilder sb = new StringBuilder("with "); + boolean first = true; + for (Map.Entry entry : cteDefinitions.entrySet()) { + if (!first) { + sb.append(", "); + } + CteDefinition cteDef = entry.getValue(); + sb.append(cteDef.asCteName(entry.getKey())) + .append(" AS (") + .append(entry.getValue().getCteDefinition()) + .append(")"); + first = false; + } + return sb.toString(); + } + + // Rename to item uid + public Set getCteNames() { + return cteDefinitions.keySet(); + } + + public boolean containsCte(String cteName) { + return cteDefinitions.containsKey(cteName); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CteDefinition.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CteDefinition.java new file mode 100644 index 000000000000..bd4419315a91 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CteDefinition.java @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2004-2025, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.common; + +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; +import org.apache.commons.text.RandomStringGenerator; + +public class CteDefinition { + + // Query item id + @Getter private String itemId; + // The program stage uid + private final String programStageUid; + // The program indicator uid + private String programIndicatorUid; + // The CTE definition (the SQL query) + @Getter private final String cteDefinition; + // The calculated offset + @Getter private final List offsets = new ArrayList<>(); + // The alias of the CTE + private final String alias; + // Whether the CTE is a row context (TODO this need a better explanation) + @Getter private boolean rowContext; + // Whether the CTE is a program indicator + @Getter private boolean programIndicator = false; + // Whether the CTE is a filter + @Getter private boolean filter = false; + // Whether the CTE is a exists, used for checking if the enrollment exists + private boolean isExists = false; + + @Getter private boolean requiresCoalesce = false; + + private static final String PS_PREFIX = "ps"; + private static final String PI_PREFIX = "pi"; + + public CteDefinition setExists(boolean exists) { + this.isExists = exists; + return this; + } + + public String getAlias() { + if (offsets.isEmpty()) { + return alias; + } + return computeAlias(offsets.get(0)); + } + + public String getAlias(int offset) { + return computeAlias(offset); + } + + private String computeAlias(int offset) { + return alias + "_" + offset; + } + + public CteDefinition( + String programStageUid, String queryItemId, String cteDefinition, int offset) { + this.programStageUid = programStageUid; + this.itemId = queryItemId; + this.cteDefinition = cteDefinition; + this.offsets.add(offset); + // one alias per offset + this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build().generate(5); + this.rowContext = false; + } + + public CteDefinition( + String programStageUid, + String queryItemId, + String cteDefinition, + int offset, + boolean isRowContext) { + this(programStageUid, queryItemId, cteDefinition, offset); + this.rowContext = isRowContext; + } + + public CteDefinition(String programIndicatorUid, String cteDefinition, boolean requiresCoalesce) { + this.cteDefinition = cteDefinition; + this.programIndicatorUid = programIndicatorUid; + this.programStageUid = null; + // ignore offset + this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build().generate(5); + this.rowContext = false; + this.programIndicator = true; + this.requiresCoalesce = requiresCoalesce; + } + + public CteDefinition( + String queryItemId, String programStageUid, String cteDefinition, boolean isFilter) { + this.itemId = queryItemId; + this.cteDefinition = cteDefinition; + this.programIndicatorUid = null; + this.programStageUid = programStageUid; + // ignore offset + this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build().generate(5); + this.rowContext = false; + this.programIndicator = false; + this.filter = isFilter; + } + + /** + * @param uid the uid of an dimension item or ProgramIndicator + * @return the name of the CTE + */ + public String asCteName(String uid) { + if (isExists) { + return uid.toLowerCase(); + } + if (programIndicator) { + return "%s_%s".formatted(PI_PREFIX, programIndicatorUid.toLowerCase()); + } + if (filter) { + return uid.toLowerCase(); + } + + return "%s_%s_%s".formatted(PS_PREFIX, programStageUid.toLowerCase(), uid.toLowerCase()); + } + + public boolean isProgramStage() { + return !filter && !programIndicator && !isExists; + } + + public boolean isExists() { + return isExists; + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/InQueryCteFilter.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/InQueryCteFilter.java new file mode 100644 index 000000000000..2fbbb97e7bd6 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/InQueryCteFilter.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2004-2025, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.common; + +import static org.hisp.dhis.analytics.QueryKey.NV; + +import java.util.List; +import java.util.function.Predicate; +import org.hisp.dhis.common.QueryFilter; + +/** Mimics the logic of @{@link org.hisp.dhis.common.InQueryFilter} to be used in CTEs */ +public class InQueryCteFilter { + + private final String filter; + + private final CteDefinition cteDefinition; + + private final String field; + + private final boolean isText; + + public InQueryCteFilter( + String field, String encodedFilter, boolean isText, CteDefinition cteDefinition) { + this.filter = encodedFilter; + this.field = field; + this.isText = isText; + this.cteDefinition = cteDefinition; + } + + public String getSqlFilter(int offset) { + + List filterItems = QueryFilter.getFilterItems(this.filter); + + StringBuilder condition = new StringBuilder(); + String alias = cteDefinition.getAlias(offset); + if (hasNonMissingValue(filterItems)) { + // TODO GIUSEPPE! + + if (hasMissingValue(filterItems)) { + + // TODO GIUSEPPE! + } + } else { + if (hasMissingValue(filterItems)) { + condition.append("%s.enrollment is not null".formatted(alias)); + condition.append(" and "); + condition.append("%s.%s is null".formatted(alias, field)); + } + } + + return condition.toString(); + } + + /** + * Checks if the filter items contain any non-missing values (values that are not {@link + * org.hisp.dhis.analytics.QueryKey#NV}). Non-missing values represent actual values that should + * be included in the SQL IN clause. This method is used to determine if the generated SQL + * condition needs to include an IN clause. + * + * @param filterItems the list of filter items to check for non-missing values + * @return true if any item in the list is not equal to {@link + * org.hisp.dhis.analytics.QueryKey#NV}, indicating at least one actual value that should be + * included in the SQL IN clause; false if all values are missing + */ + private boolean hasNonMissingValue(List filterItems) { + return anyMatch(filterItems, this::isNotMissingItem); + } + + private boolean isNotMissingItem(String filterItem) { + return !isMissingItem(filterItem); + } + + private boolean isMissingItem(String filterItem) { + return NV.equals(filterItem); + } + + /** + * Checks if any item in the list matches the given predicate. + * + * @param filterItems the list of items to check + * @param predi the predicate to test against + * @return true if any item matches the predicate, false otherwise + */ + private boolean anyMatch(List filterItems, Predicate predi) { + return filterItems.stream().anyMatch(predi); + } + + /** + * Checks if the filter items contain any missing values represented by the special marker {@link + * org.hisp.dhis.analytics.QueryKey#NV}. Missing values indicate that the corresponding database + * field should be treated as NULL in the SQL query. This method is used to determine if the + * generated SQL condition needs to include an IS NULL clause. + * + * @param filterItems the list of filter items to check for missing values + * @return true if any item in the list equals{@link org.hisp.dhis.analytics.QueryKey#NV}, + * indicating a missing value that should be treated as NULL in the SQL query; false otherwise + * @see org.hisp.dhis.analytics.QueryKey#NV + */ + private boolean hasMissingValue(List filterItems) { + return anyMatch(filterItems, this::isMissingItem); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/ProgramIndicatorSubqueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/ProgramIndicatorSubqueryBuilder.java index 411757dc52a6..4da3ca38da3b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/ProgramIndicatorSubqueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/ProgramIndicatorSubqueryBuilder.java @@ -101,4 +101,19 @@ String getAggregateClauseForProgramIndicator( AnalyticsType outerSqlEntity, Date earliestStartDate, Date latestDate); + + void contributeCTE( + ProgramIndicator programIndicator, + AnalyticsType outerSqlEntity, + Date earliestStartDate, + Date latestDate, + CteContext cteContext); + + void contributeCTE( + ProgramIndicator programIndicator, + RelationshipType relationshipType, + AnalyticsType outerSqlEntity, + Date earliestStartDate, + Date latestDate, + CteContext cteContext); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 47952b35a9d4..4bd189c34202 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -75,6 +75,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.UUID; import java.util.stream.Collector; import java.util.stream.Collectors; @@ -91,9 +92,12 @@ import org.hisp.dhis.analytics.EventOutputType; import org.hisp.dhis.analytics.SortOrder; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; +import org.hisp.dhis.analytics.common.CteContext; +import org.hisp.dhis.analytics.common.CteDefinition; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; import org.hisp.dhis.analytics.event.EventQueryParams; import org.hisp.dhis.analytics.util.AnalyticsUtils; +import org.hisp.dhis.analytics.util.sql.SqlConditionJoiner; import org.hisp.dhis.common.DimensionType; import org.hisp.dhis.common.DimensionalItemObject; import org.hisp.dhis.common.DimensionalObject; @@ -169,7 +173,7 @@ public abstract class AbstractJdbcEventAnalyticsManager { * @param params the {@link EventQueryParams}. * @param maxLimit the configurable max limit of records. */ - private String getPagingClause(EventQueryParams params, int maxLimit) { + protected String getPagingClause(EventQueryParams params, int maxLimit) { String sql = ""; if (params.isPaging()) { @@ -191,7 +195,7 @@ private String getPagingClause(EventQueryParams params, int maxLimit) { * * @param params the {@link EventQueryParams}. */ - private String getSortClause(EventQueryParams params) { + protected String getSortClause(EventQueryParams params) { String sql = ""; if (params.isSorting()) { @@ -413,7 +417,7 @@ private void addItemSelectColumns( * @param queryItem * @return true when eligible for row context */ - private boolean rowContextAllowedAndNeeded(EventQueryParams params, QueryItem queryItem) { + protected boolean rowContextAllowedAndNeeded(EventQueryParams params, QueryItem queryItem) { return params.getEndpointItem() == ENROLLMENT && params.isRowContext() && queryItem.hasProgramStage() @@ -944,7 +948,9 @@ protected String getAggregatedEnrollmentsSql(List headers, EventQuer sql += getFromClause(params); - sql += getWhereClause(params); + String whereClause = getWhereClause(params); + String filterWhereClause = getQueryItemsAndFiltersWhereClause(params, new SqlHelper()); + sql += SqlConditionJoiner.joinSqlConditions(whereClause, filterWhereClause); String headerColumns = getHeaderColumns(headers, sql).stream().collect(joining(",")); String orgColumns = getOrgUnitLevelColumns(params).stream().collect(joining(",")); @@ -1079,13 +1085,18 @@ private void addGridDoubleTypeValue( } } + protected String getQueryItemsAndFiltersWhereClause(EventQueryParams params, SqlHelper helper) { + return getQueryItemsAndFiltersWhereClause(params, Set.of(), helper); + } + /** * Returns a SQL where clause string for query items and query item filters. * * @param params the {@link EventQueryParams}. * @param helper the {@link SqlHelper}. */ - protected String getQueryItemsAndFiltersWhereClause(EventQueryParams params, SqlHelper helper) { + protected String getQueryItemsAndFiltersWhereClause( + EventQueryParams params, Set exclude, SqlHelper helper) { if (params.isEnhancedCondition()) { return getItemsSqlForEnhancedConditions(params, helper); } @@ -1096,6 +1107,7 @@ protected String getQueryItemsAndFiltersWhereClause(EventQueryParams params, Sql Map> itemsByRepeatableFlag = Stream.concat(params.getItems().stream(), params.getItemFilters().stream()) .filter(QueryItem::hasFilter) + .filter(queryItem -> !exclude.contains(queryItem)) .collect( groupingBy( queryItem -> @@ -1113,13 +1125,13 @@ protected String getQueryItemsAndFiltersWhereClause(EventQueryParams params, Sql List orConditions = repeatableConditionsByIdentifier.values().stream() .map(sameGroup -> joinSql(sameGroup, OR_JOINER)) - .collect(toList()); + .toList(); // Non-repeatable conditions List andConditions = asSqlCollection(itemsByRepeatableFlag.get(false), params) .map(IdentifiableSql::getSql) - .collect(toList()); + .toList(); if (orConditions.isEmpty() && andConditions.isEmpty()) { return StringUtils.EMPTY; @@ -1173,7 +1185,7 @@ private String joinSql(Stream conditions, Collector sqlConditionByGroup = Stream.concat(params.getItems().stream(), params.getItemFilters().stream()) .filter(QueryItem::hasFilter) @@ -1237,7 +1249,7 @@ private String getIdentifier(QueryItem queryItem) { @Getter @Builder - private static class IdentifiableSql { + public static class IdentifiableSql { private final String identifier; private final String sql; @@ -1250,7 +1262,7 @@ private static class IdentifiableSql { * @param filter the {@link QueryFilter}. * @param params the {@link EventQueryParams}. */ - private String toSql(QueryItem item, QueryFilter filter, EventQueryParams params) { + protected String toSql(QueryItem item, QueryFilter filter, EventQueryParams params) { String field = item.hasAggregationType() ? getSelectSql(filter, item, params) @@ -1393,6 +1405,49 @@ protected String getCoalesce(List fields, String defaultColumnName) { return args.isEmpty() ? defaultColumnName : sql; } + protected List getSelectColumnsWithCTE(EventQueryParams params, CteContext cteContext) { + List columns = new ArrayList<>(); + + // Mirror the logic of addDimensionSelectColumns + addDimensionSelectColumns(columns, params, false); + + // Mirror the logic of addItemSelectColumns but with CTE references + for (QueryItem queryItem : params.getItems()) { + if (queryItem.isProgramIndicator()) { + // For program indicators, use CTE reference + String piUid = queryItem.getItem().getUid(); + CteDefinition cteDef = cteContext.getDefinitionByItemUid(piUid); + // COALESCE(fbyta.value, 0) as CH6wamtY9kK + String col = + cteDef.isRequiresCoalesce() + ? "coalesce(%s.value, 0) as %s".formatted(cteDef.getAlias(), piUid) + : "%s.value as %s".formatted(cteDef.getAlias(), piUid); + columns.add(col); + } else if (ValueType.COORDINATE == queryItem.getValueType()) { + // Handle coordinates + columns.add(getCoordinateColumn(queryItem).asSql()); + } else if (ValueType.ORGANISATION_UNIT == queryItem.getValueType()) { + // Handle org units + if (params.getCoordinateFields().stream() + .anyMatch(f -> queryItem.getItem().getUid().equals(f))) { + columns.add(getCoordinateColumn(queryItem, OU_GEOMETRY_COL_SUFFIX).asSql()); + } else { + columns.add(getOrgUnitQueryItemColumnAndAlias(params, queryItem).asSql()); + } + } else if (queryItem.hasProgramStage()) { + // Handle program stage items with CTE + columns.add(getColumnWithCte(queryItem, "", cteContext)); + } else { + // Handle other types as before + ColumnAndAlias columnAndAlias = getColumnAndAlias(queryItem, false, ""); + columns.add(columnAndAlias.asSql()); + } + } + // remove duplicates + var ded = columns.stream().distinct().toList(); + return ded; + } + /** * Returns a select SQL clause for the given query. * @@ -1400,6 +1455,8 @@ protected String getCoalesce(List fields, String defaultColumnName) { */ protected abstract String getSelectClause(EventQueryParams params); + protected abstract String getColumnWithCte(QueryItem item, String suffix, CteContext cteContext); + /** * Generates the SQL for the from-clause. Generally this means which analytics table to get data * from. diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index da2129f70c2f..2f95581513e6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -27,30 +27,42 @@ */ package org.hisp.dhis.analytics.event.data; +import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.joining; import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.hisp.dhis.analytics.AnalyticsConstants.ANALYTICS_TBL_ALIAS; import static org.hisp.dhis.analytics.DataType.BOOLEAN; +import static org.hisp.dhis.analytics.common.CTEUtils.computeKey; import static org.hisp.dhis.analytics.event.data.OrgUnitTableJoiner.joinOrgUnitTables; import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; import static org.hisp.dhis.common.DataDimensionType.ATTRIBUTE; import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; +import static org.hisp.dhis.common.QueryOperator.IN; import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString; import static org.hisp.dhis.commons.util.TextUtils.removeLastOr; import static org.hisp.dhis.util.DateUtils.toMediumDate; import com.google.common.collect.Sets; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; +import org.hisp.dhis.analytics.common.CTEUtils; +import org.hisp.dhis.analytics.common.CteContext; +import org.hisp.dhis.analytics.common.CteDefinition; +import org.hisp.dhis.analytics.common.InQueryCteFilter; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; import org.hisp.dhis.analytics.event.EnrollmentAnalyticsManager; import org.hisp.dhis.analytics.event.EventQueryParams; @@ -64,7 +76,10 @@ import org.hisp.dhis.common.FallbackCoordinateFieldType; import org.hisp.dhis.common.Grid; import org.hisp.dhis.common.OrganisationUnitSelectionMode; +import org.hisp.dhis.common.QueryFilter; import org.hisp.dhis.common.QueryItem; +import org.hisp.dhis.common.QueryOperator; +import org.hisp.dhis.common.RequestTypeAware; import org.hisp.dhis.common.ValueStatus; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.commons.collection.ListUtils; @@ -74,6 +89,7 @@ import org.hisp.dhis.event.EventStatus; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.program.AnalyticsType; +import org.hisp.dhis.program.ProgramIndicator; import org.hisp.dhis.program.ProgramIndicatorService; import org.hisp.dhis.system.util.ListBuilder; import org.locationtech.jts.util.Assert; @@ -125,10 +141,14 @@ public JdbcEnrollmentAnalyticsManager( @Override public void getEnrollments(EventQueryParams params, Grid grid, int maxLimit) { - String sql = - params.isAggregatedEnrollments() - ? getAggregatedEnrollmentsSql(grid.getHeaders(), params) - : getAggregatedEnrollmentsSql(params, maxLimit); + String sql; + if (params.isAggregatedEnrollments()) { + sql = getAggregatedEnrollmentsSql(grid.getHeaders(), params); + } else { + sql = buildEnrollmentQueryWithCte(params); + } + + System.out.println("SQL: " + sql); // FIXME: Remove debug line if (params.analyzeOnly()) { withExceptionHandling( @@ -274,6 +294,7 @@ public long getEnrollmentCount(EventQueryParams params) { sql += getFromClause(params); sql += getWhereClause(params); + sql += addFiltersToWhereClause(params); long count = 0; @@ -289,7 +310,7 @@ public long getEnrollmentCount(EventQueryParams params) { withExceptionHandling( () -> jdbcTemplate.queryForObject(finalSqlValue, Long.class), params.isMultipleQueries()) - .orElse(0l); + .orElse(0L); } return count; @@ -408,12 +429,6 @@ protected String getWhereClause(EventQueryParams params) { sql += "and ps = '" + params.getProgramStage().getUid() + "' "; } - // --------------------------------------------------------------------- - // Query items and filters - // --------------------------------------------------------------------- - - sql += getQueryItemsAndFiltersWhereClause(params, hlp); - // --------------------------------------------------------------------- // Filter expression // --------------------------------------------------------------------- @@ -476,6 +491,130 @@ protected String getWhereClause(EventQueryParams params) { return sql; } + private String addFiltersToWhereClause(EventQueryParams params) { + return getQueryItemsAndFiltersWhereClause(params, new SqlHelper()); + } + + private String addCteFiltersToWhereClause(EventQueryParams params, CteContext cteContext) { + StringBuilder cteWhereClause = new StringBuilder(); + Set processedItems = new HashSet<>(); // Track processed items + + // Get all filters from the query items and item filters + List filters = + Stream.concat(params.getItems().stream(), params.getItemFilters().stream()) + .filter(QueryItem::hasFilter) + .toList(); + // Iterate over each filter and apply the correct condition + for (QueryItem item : filters) { + String cteName = CTEUtils.computeKey(item); + + if (cteContext.containsCte(cteName)) { + processedItems.add(item); // Mark item as processed + CteDefinition cteDef = cteContext.getDefinitionByItemUid(cteName); + for (QueryFilter filter : item.getFilters()) { + if (IN.equals(filter.getOperator())) { + InQueryCteFilter inQueryCteFilter = + new InQueryCteFilter("value", filter.getFilter(), item.isText(), cteDef); + cteWhereClause + .append(" and ") + .append( + inQueryCteFilter.getSqlFilter( + computeRowNumberOffset(item.getProgramStageOffset()))); + } else { + String value = getSqlFilterValue(filter, item); + + cteWhereClause + .append(" and ") + .append(cteDef.getAlias()) + .append(".value ") + .append("NULL".equals(value) ? "is" : filter.getSqlOperator()) + .append(" ") + .append(value); + } + } + } + } + // Add filters for items that are not part of the CTE + String nonCteWhereClause = + getQueryItemsAndFiltersWhereClause(params, processedItems, new SqlHelper()) + .replace("where", ""); + if (nonCteWhereClause.isEmpty()) return cteWhereClause.toString(); + + String currentWhereClause = cteWhereClause.toString().toLowerCase().trim(); + cteWhereClause.append( + currentWhereClause.endsWith("and") ? nonCteWhereClause : " and " + nonCteWhereClause); + + return cteWhereClause.toString(); + } + + private String getSqlFilterValue(QueryFilter filter, QueryItem item) { + if ("NV".equals(filter.getFilter())) { + return "NULL"; // Special case for 'null' filters + } + + // Handle IN operator: wrap the value(s) in parentheses + if (filter.getOperator() == QueryOperator.IN) { + String[] values = filter.getFilter().split(","); // Support multiple values + String quotedValues = + Arrays.stream(values) + .map(value -> item.isNumeric() ? value : sqlBuilder.singleQuote(value)) + .collect(Collectors.joining(", ")); + return "(" + quotedValues + ")"; + } + + // Handle text and numeric values + return item.isNumeric() + ? filter.getSqlBindFilter() + : sqlBuilder.singleQuote(filter.getSqlBindFilter()); + } + + private String buildFilterCteSql(List queryItems, EventQueryParams params) { + return queryItems.stream() + .map( + item -> { + // Determine the correct table: event table or enrollment table + String tableName = + item.hasProgramStage() + ? "analytics_event_" + + item.getProgram() + .getUid() + .toLowerCase() // Event table for program stage + : params.getTableName(); // Enrollment table + + String columnName = quote(item.getItemName()); // Raw column name without alias + String programStageCondition = + item.hasProgramStage() + ? "AND ps = '" + item.getProgramStage().getUid() + "'" + : ""; // Add program stage filter if available + + return """ + select + enrollment, + %s as value + from + (select + enrollment, + %s, + row_number() over ( + partition by enrollment + order by + occurreddate desc, + created desc + ) as rn + from + %s + where + eventstatus != 'SCHEDULE' + %s + ) ranked + where + rn = 1 + """ + .formatted(columnName, columnName, tableName, programStageCondition); + }) + .collect(Collectors.joining("\nUNION ALL\n")); + } + @Override protected String getSelectClause(EventQueryParams params) { List selectCols = @@ -565,6 +704,29 @@ protected ColumnAndAlias getCoordinateColumn(QueryItem item, String suffix) { return ColumnAndAlias.EMPTY; } + protected String getColumnWithCte(QueryItem item, String suffix, CteContext cteContext) { + List columns = new ArrayList<>(); + String colName = item.getItemName(); + + CteDefinition cteDef = cteContext.getDefinitionByItemUid(computeKey(item)); + int programStageOffset = computeRowNumberOffset(item.getProgramStageOffset()); + String alias = getAlias(item).orElse(null); + columns.add("%s.value as %s".formatted(cteDef.getAlias(programStageOffset), quote(alias))); + if (cteDef.isRowContext()) { + // Add additional status and exists columns for row context + columns.add( + "COALESCE(%s.rn = %s, false) as %s" + .formatted( + cteDef.getAlias(programStageOffset), + programStageOffset + 1, + quote(alias + ".exists"))); + columns.add( + "%s.eventstatus as %s" + .formatted(cteDef.getAlias(programStageOffset), quote(alias + ".status"))); + } + return String.join(",\n", columns); + } + /** * Creates a column "selector" for the given item name. The suffix will be appended as part of the * item name. The column selection is based on events analytics tables. @@ -767,4 +929,380 @@ private String createOrderType(int offset) { return ORDER_BY_EXECUTION_DATE.replace(DIRECTION_PLACEHOLDER, "asc"); } } + + // New methods // + + private void handleProgramIndicatorCte( + QueryItem item, CteContext cteContext, EventQueryParams params) { + ProgramIndicator pi = (ProgramIndicator) item.getItem(); + if (item.hasRelationshipType()) { + programIndicatorSubqueryBuilder.contributeCTE( + pi, + item.getRelationshipType(), + getAnalyticsType(), + params.getEarliestStartDate(), + params.getLatestEndDate(), + cteContext); + } else { + programIndicatorSubqueryBuilder.contributeCTE( + pi, + getAnalyticsType(), + params.getEarliestStartDate(), + params.getLatestEndDate(), + cteContext); + } + } + + /** + * Builds the CTE definitions for the given {@link EventQueryParams}. + * + *

For each {@link QueryItem} in {@code params}, this method: + * + *

    + *
  • Identifies if the item is a {@link ProgramIndicator} and delegates to {@link + * #handleProgramIndicatorCte(QueryItem, CteContext, EventQueryParams)}. + *
  • Identifies if the item has a {@link org.hisp.dhis.program.ProgramStage} and generates the + * appropriate CTE SQL, including any row-context details if the stage is repeatable. + *
  • Adds each resulting CTE (and optional "exists" CTE) to the provided {@link CteContext}. + *
+ * + * @param params the {@link EventQueryParams} describing what data is being queried + * @return a {@link CteContext} instance containing all relevant CTE definitions + */ + private CteContext getCteDefinitions(EventQueryParams params) { + CteContext cteContext = new CteContext(); + + for (QueryItem item : params.getItems()) { + if (item.isProgramIndicator()) { + // Handle any program indicator CTE logic. + handleProgramIndicatorCte(item, cteContext, params); + } else if (item.hasProgramStage()) { + // Build CTE for program-stage-based items (including repeatable logic). + buildProgramStageCte(cteContext, item, params); + } + } + + return cteContext; + } + + /** + * Builds and registers a CTE definition for the given {@link QueryItem} (which must have a {@link + * org.hisp.dhis.program.ProgramStage}). This covers both repeatable and non-repeatable program + * stages, optionally adding row-context CTEs if needed. + * + * @param cteContext the {@link CteContext} to which the new CTE definition(s) will be added + * @param item the {@link QueryItem} containing program-stage details + * @param params the {@link EventQueryParams}, used for checking row-context eligibility, offsets, + * etc. + */ + private void buildProgramStageCte( + CteContext cteContext, QueryItem item, EventQueryParams params) { + // The event table name, e.g. "analytics_event_XYZ". + String eventTableName = ANALYTICS_EVENT + item.getProgram().getUid(); + + // Quoted column name for the item (e.g. "ax"."my_column"). + String colName = quote(item.getItemName()); + + // Determine if row context is needed (repeatable stage + rowContextAllowed). + boolean hasRowContext = rowContextAllowedAndNeeded(params, item); + + // Build the main CTE SQL. + // If hasRowContext == true, we'll also include the eventstatus column. + String cteSql = + """ + select + enrollment, + %s as value,%s + row_number() over ( + partition by enrollment + order by occurreddate desc, created desc + ) as rn + from %s + where eventstatus != 'SCHEDULE' + and ps = '%s' + """ + .formatted( + colName, + hasRowContext ? " eventstatus," : "", + eventTableName, + item.getProgramStage().getUid()); + + // Register this CTE in the context. + // The createOffset2(...) method calculates the row offset based on + // item.getProgramStageOffset(). + cteContext.addCte( + item.getProgramStage(), + item, + cteSql, + computeRowNumberOffset(item.getProgramStageOffset()), + hasRowContext); + + // If row context is needed, we add an extra "exists" CTE for event checks. + if (hasRowContext) { + String existCte = + """ + select distinct + enrollment + from + %s + where + eventstatus != 'SCHEDULE' + and ps = '%s' + """ + .formatted(eventTableName, item.getProgramStage().getUid()); + + cteContext.addExistsCte(item.getProgramStage(), item, existCte); + } + } + + private void appendCteJoins(StringBuilder sql, CteContext cteContext) { + for (String itemUid : cteContext.getCteNames()) { + CteDefinition cteDef = cteContext.getDefinitionByItemUid(itemUid); + + // Handle Program Stage CTE (potentially with multiple offsets) + if (cteDef.isProgramStage()) { + for (Integer offset : cteDef.getOffsets()) { + String alias = cteDef.getAlias(offset); + // Using a text block with .formatted() for clarity: + String join = + """ + LEFT JOIN %s %s + ON %s.enrollment = ax.enrollment + AND %s.rn = %d + """ + .formatted( + cteDef.asCteName(itemUid), // e.g. ps_ABC123_xyz + alias, // random alias + alias, // alias for table + alias, // alias repeated + offset + 1 // offset index + ); + sql.append(join); + } + } + + // Handle 'Exists' type CTE + if (cteDef.isExists()) { + String join = + String.format( + """ + LEFT JOIN %s ee ON ee.enrollment = ax.enrollment + """, + cteDef.asCteName(itemUid)); + sql.append(join); + } + + // Handle Program Indicator CTE + if (cteDef.isProgramIndicator()) { + String alias = cteDef.getAlias(); + String join = + """ + LEFT JOIN %s %s + ON %s.enrollment = ax.enrollment + """ + .formatted(cteDef.asCteName(itemUid), alias, alias); + sql.append(join); + } + + // Handle Filter CTE + if (cteDef.isFilter()) { + String alias = cteDef.getAlias(); + String join = + """ + LEFT JOIN %s %s + ON %s.enrollment = ax.enrollment + """ + .formatted(cteDef.asCteName(itemUid), alias, alias); + sql.append(join); + } + } + } + + /** + * Computes a zero-based offset for use with the SQL row_number() function in CTEs that + * partition and order events by date (e.g., most recent first). + * + *

In this context, an {@code offset} of 0 typically means “the most recent event” (row_number + * = 1), a positive offset means “the Nth future event after the most recent” (for example, offset + * = 1 means row_number = 2), and a negative offset means “the Nth older event before the most + * recent”. + * + *

Internally, this method transforms the supplied {@code offset} into a + * zero-based index, suitable for comparing against the row_number output. For + * instance: + * + *

    + *
  • If {@code offset == 0}, returns {@code 0}. + *
  • If {@code offset > 0}, returns {@code offset - 1} (i.e., offset 1 becomes 0-based 0). + *
  • If {@code offset < 0}, returns the absolute value ({@code -offset}). + *
+ * + * @param offset an integer specifying how many positions away from the most recent event + * (row_number = 1) you want to select. A positive offset selects a future row_number, a + * negative offset selects a past row_number, and zero selects the most recent. + * @return an integer representing the zero-based offset to use in a {@code row_number} comparison + */ + private int computeRowNumberOffset(int offset) { + if (offset == 0) { + return 0; + } + + if (offset < 0) { + return (-1 * offset); + } else { + return (offset - 1); + } + } + + private void generateFilterCTEs(EventQueryParams params, CteContext cteContext) { + // Combine items and item filters + List queryItems = + Stream.concat(params.getItems().stream(), params.getItemFilters().stream()) + .filter(QueryItem::hasFilter) + .toList(); + + // Group query items by repeatable and non-repeatable stages + Map> itemsByRepeatableFlag = + queryItems.stream() + .collect( + groupingBy( + queryItem -> + queryItem.hasRepeatableStageParams() + && params.getEndpointItem() + == RequestTypeAware.EndpointItem.ENROLLMENT)); + + // Process repeatable stage filters + itemsByRepeatableFlag.getOrDefault(true, List.of()).stream() + .collect(groupingBy(CTEUtils::getIdentifier)) + .forEach( + (identifier, items) -> { + String cteSql = buildFilterCteSql(items, params); + // TODO is this correct? items.get(0) + cteContext.addCteFilter(items.get(0), cteSql); + }); + + // Process non-repeatable stage filters + itemsByRepeatableFlag + .getOrDefault(false, List.of()) + .forEach( + queryItem -> { + if (queryItem.hasProgram() && queryItem.hasProgramStage()) { + String cteSql = buildFilterCteSql(List.of(queryItem), params); + cteContext.addCteFilter(queryItem, cteSql); + } + }); + } + + private String buildEnrollmentQueryWithCte(EventQueryParams params) { + // LUCIANO // + + // 1. Create the CTE context (collect all CTE definitions for program indicators, program + // stages, etc.) + CteContext cteContext = getCteDefinitions(params); + + // 2. Generate any additional CTE filters that might be needed + generateFilterCTEs(params, cteContext); + + // 3. Build up the final SQL using dedicated sub-steps + StringBuilder sql = new StringBuilder(); + + // 3.1: Append the WITH clause if needed + appendCteClause(sql, cteContext); + + // 3.2: Append the SELECT clause, including columns from the CTE context + appendSelectClause(sql, params, cteContext); + + // 3.3: Append the FROM clause (the main enrollment analytics table) + appendFromClause(sql, params); + + // 3.4: Append LEFT JOINs for each relevant CTE definition + appendCteJoins(sql, cteContext); + + // 3.5: Collect and append WHERE conditions (including filters from CTE) + appendWhereClause(sql, params, cteContext); + + // 3.6: Append ORDER BY and paging + appendSortingAndPaging(sql, params); + + return sql.toString(); + } + + /** + * Appends the WITH clause using the CTE definitions from cteContext. If there are no CTE + * definitions, nothing is appended. + */ + private void appendCteClause(StringBuilder sql, CteContext cteContext) { + String cteDefinitions = cteContext.getCteDefinition(); + if (!cteDefinitions.isEmpty()) { + sql.append(cteDefinitions).append("\n"); + } + } + + /** + * Appends the SELECT clause, including both the standard enrollment columns (or aggregated + * columns) and columns derived from the CTE definitions. + */ + private void appendSelectClause( + StringBuilder sql, EventQueryParams params, CteContext cteContext) { + // Get the standard columns, and prepend the alias "ax." to each column name + // (except for columns that are part of a function, e.g. "count(abc)"). + List aliasedSelectCols = + getStandardColumns().stream() + .map(c -> (!c.contains("(") || !c.contains(")")) ? "ax." + c : c) + .toList(); + + List selectCols = + ListUtils.distinctUnion( + params.isAggregatedEnrollments() ? List.of("enrollment") : aliasedSelectCols, + getSelectColumnsWithCTE(params, cteContext)); + + // Join the list of select columns with commas and append + sql.append("select ").append(String.join(",\n", selectCols)).append("\n"); + } + + /** Appends the FROM clause, i.e. the main table name and alias. */ + private void appendFromClause(StringBuilder sql, EventQueryParams params) { + sql.append("from ").append(params.getTableName()).append(" AS ax"); + } + + /** + * Collects the WHERE conditions from both the base enrollment table and the CTE-based filters, + * then appends them to the SQL. + */ + private void appendWhereClause( + StringBuilder sql, EventQueryParams params, CteContext cteContext) { + List conditions = new ArrayList<>(); + + String baseWhereClause = getWhereClause(params).trim(); + String cteFilters = addCteFiltersToWhereClause(params, cteContext).trim(); + + if (!baseWhereClause.isEmpty()) { + // Remove leading "WHERE" if present + conditions.add(baseWhereClause.replaceFirst("(?i)^WHERE\\s+", "")); + } + if (!cteFilters.isEmpty()) { + conditions.add(cteFilters.replaceFirst("(?i)^AND\\s+", "")); + } + + if (!conditions.isEmpty()) { + sql.append(" WHERE ").append(String.join(" AND ", conditions)); + } + } + + /** Appends the ORDER BY clause if sorting is specified, plus LIMIT/OFFSET for paging. */ + private void appendSortingAndPaging(StringBuilder sql, EventQueryParams params) { + // Add ORDER BY if needed + if (params.isSorting()) { + sql.append(" ").append(getSortClause(params)); + } + // Add paging (LIMIT/OFFSET) + sql.append(" ").append(getPagingClause(params, 5000)); + } + + protected String getSortClause(EventQueryParams params) { + if (params.isSorting()) { + return super.getSortClause(params); + } + return ""; + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java index 23da08181fd1..d28f27b029b3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java @@ -62,6 +62,7 @@ import org.hisp.dhis.analytics.Rectangle; import org.hisp.dhis.analytics.TimeField; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; +import org.hisp.dhis.analytics.common.CteContext; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; import org.hisp.dhis.analytics.event.EventAnalyticsManager; import org.hisp.dhis.analytics.event.EventQueryParams; @@ -391,6 +392,12 @@ private String getCoordinateSelectExpression(EventQueryParams params) { return String.format("ST_AsGeoJSON(%s, 6) as geometry", field); } + @Override + protected String getColumnWithCte(QueryItem item, String suffix, CteContext cteContext) { + // TODO: Implement + return ""; + } + /** * Returns a from SQL clause for the given analytics table partition. If the query has a * non-default time field specified, a join with the {@code date period structure} resource table diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java index 13b83e647c82..d484edbf385e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/programindicator/DefaultProgramIndicatorSubqueryBuilder.java @@ -27,7 +27,6 @@ */ package org.hisp.dhis.analytics.event.data.programindicator; -import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.hisp.dhis.analytics.DataType.BOOLEAN; import static org.hisp.dhis.analytics.DataType.NUMERIC; @@ -38,6 +37,7 @@ import org.hisp.dhis.analytics.AggregationType; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.DataType; +import org.hisp.dhis.analytics.common.CteContext; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; import org.hisp.dhis.analytics.table.model.AnalyticsTable; import org.hisp.dhis.commons.util.TextUtils; @@ -77,6 +77,87 @@ public String getAggregateClauseForProgramIndicator( programIndicator, relationshipType, outerSqlEntity, earliestStartDate, latestDate); } + @Override + public void contributeCTE( + ProgramIndicator programIndicator, + AnalyticsType outerSqlEntity, + Date earliestStartDate, + Date latestDate, + CteContext cteContext) { + contributeCTE( + programIndicator, null, outerSqlEntity, earliestStartDate, latestDate, cteContext); + } + + @Override + public void contributeCTE( + ProgramIndicator programIndicator, + RelationshipType relationshipType, + AnalyticsType outerSqlEntity, + Date earliestStartDate, + Date latestDate, + CteContext cteContext) { + + // Define aggregation function + String function = + TextUtils.emptyIfEqual( + programIndicator.getAggregationTypeFallback().getValue(), + AggregationType.CUSTOM.getValue()); + + String filter = ""; + if (programIndicator.hasFilter()) { + + String piResolvedSqlFilter = + getProgramIndicatorSql( + programIndicator.getFilter(), + NUMERIC, + programIndicator, + earliestStartDate, + latestDate) + // FIXME this is a bit of an hack + .replace("subax.\"ps\"", "ps") + .replace("subax.enrollment", "enrollment"); + + filter = "where " + piResolvedSqlFilter; + } + + String piResolvedSql = + getProgramIndicatorSql( + programIndicator.getExpression(), + NUMERIC, + programIndicator, + earliestStartDate, + latestDate) + // FIXME this is a bit of an hack + .replace("subax.\"ps\"", "ps") + .replace("subax.enrollment", "enrollment"); + + String cteSql = + "select enrollment, %s(%s) as value from %s %s group by enrollment" + .formatted(function, piResolvedSql, getTableName(programIndicator), filter); + + // Register the CTE and its column mapping + cteContext.addProgramIndicatorCte(programIndicator, cteSql, requireCoalesce(function)); + } + + /** + * Determine if the aggregation function requires a COALESCE function to handle NULL values. + * + * @param function the aggregation function + * @return true if the function requires a COALESCE function, false otherwise + */ + private boolean requireCoalesce(String function) { + return switch (function.toLowerCase()) { + // removed "avg" from list because it seems that it does not require COALESCE + // even though it is an aggregation function + case "count", "sum", "min", "max" -> true; + default -> false; + }; + } + + private String getTableName(ProgramIndicator programIndicator) { + return "analytics_event_" + programIndicator.getProgram().getUid().toLowerCase(); + } + /** * Generate a subquery based on the result of a Program Indicator and an (optional) Relationship * Type @@ -169,16 +250,18 @@ private String getWhere( RelationshipTypeJoinGenerator.generate( SUBQUERY_TABLE_ALIAS, relationshipType, programIndicator.getAnalyticsType()); } else { - if (AnalyticsType.ENROLLMENT == outerSqlEntity) { - condition = "enrollment = ax.enrollment"; - } else { - if (AnalyticsType.EVENT == programIndicator.getAnalyticsType()) { - condition = "event = ax.event"; - } + // Remove the reference to the outer query's enrollment + // We'll handle the join in the main query + if (AnalyticsType.ENROLLMENT == programIndicator.getAnalyticsType()) { + // No condition needed, we'll join on enrollment in the main query + condition = ""; + } else if (AnalyticsType.EVENT == programIndicator.getAnalyticsType()) { + // Handle event type if needed + condition = ""; } } - return isNotBlank(condition) ? " WHERE " + condition : condition; + return !condition.isEmpty() ? " WHERE " + condition : ""; } /** diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/sql/SqlConditionJoiner.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/sql/SqlConditionJoiner.java new file mode 100644 index 000000000000..af51d0cb1173 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/sql/SqlConditionJoiner.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2004-2025, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.util.sql; + +public class SqlConditionJoiner { + + public static String joinSqlConditions(String... conditions) { + if (conditions == null || conditions.length == 0) { + return ""; + } + + StringBuilder result = new StringBuilder("where "); + boolean firstCondition = true; + + for (String condition : conditions) { + if (condition == null || condition.trim().isEmpty()) { + continue; + } + + // Remove leading "where" or " where" and trim + String cleanedCondition = condition.trim(); + if (cleanedCondition.toLowerCase().startsWith("where")) { + cleanedCondition = cleanedCondition.substring(5).trim(); + } + + if (!cleanedCondition.isEmpty()) { + if (!firstCondition) { + result.append(" and "); + } + result.append(cleanedCondition); + firstCondition = false; + } + } + + return result.toString(); + } +}