diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java index 308a3970254..566a6e29b63 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEContext.java @@ -27,188 +27,109 @@ */ package org.hisp.dhis.analytics.common; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Set; -import lombok.Getter; -import org.apache.commons.text.RandomStringGenerator; import org.hisp.dhis.common.QueryItem; import org.hisp.dhis.program.ProgramIndicator; import org.hisp.dhis.program.ProgramStage; -public class CTEContext { - private final Map cteDefinitions = new LinkedHashMap<>(); - - public CteDefinitionWithOffset getDefinitionByItemUid(String itemUid) { - return cteDefinitions.get(itemUid); - } - - public void addCTE(ProgramStage programStage, QueryItem item, String cteDefinition, int offset) { - cteDefinitions.put( - item.getItem().getUid(), - new CteDefinitionWithOffset(programStage.getUid(), item.getItemId(), cteDefinition, offset)); - } - - public void addCTE( - ProgramStage programStage, - QueryItem item, - String cteDefinition, - int offset, - boolean isRowContext) { - cteDefinitions.put( - item.getItem().getUid(), - new CteDefinitionWithOffset(programStage.getUid(), item.getItemId(), cteDefinition, offset, isRowContext)); - } - - public void addExistsCTE( - ProgramStage programStage, - QueryItem item, - String cteDefinition) { - var cteDef = new CteDefinitionWithOffset(programStage.getUid(), item.getItemId(), cteDefinition, -999, false) - .setExists(true); - cteDefinitions.put( - programStage.getUid(), - cteDef); - } - - /** - * Adds a CTE definition to the context. - * - * @param programIndicator The program indicator - * @param cteDefinition The CTE definition (the SQL query) - */ - public void addProgramIndicatorCTE(ProgramIndicator programIndicator, String cteDefinition) { - cteDefinitions.put( - programIndicator.getUid(), - new CteDefinitionWithOffset(programIndicator.getUid(), cteDefinition)); - } +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; - public void addCTEFilter(String name, String ctedefinition) { - cteDefinitions.put(name, new CteDefinitionWithOffset(name, ctedefinition, true)); - } +import static org.hisp.dhis.analytics.common.CTEUtils.computeKey; - public String getCTEDefinition() { - if (cteDefinitions.isEmpty()) { - return ""; - } +public class CTEContext { + private final Map cteDefinitions = new LinkedHashMap<>(); - StringBuilder sb = new StringBuilder("WITH "); - boolean first = true; - for (Map.Entry entry : cteDefinitions.entrySet()) { - if (!first) { - sb.append(", "); - } - CteDefinitionWithOffset cteDef = entry.getValue(); - sb.append(cteDef.asCteName(entry.getKey())) - .append(" AS (") - .append(entry.getValue().cteDefinition) - .append(")"); - first = false; + public CteDefinition getDefinitionByItemUid(String itemUid) { + return cteDefinitions.get(itemUid); } - return sb.toString(); - } - // Rename to item uid - public Set getCTENames() { - return cteDefinitions.keySet(); - } - - public boolean containsCteFilter(String cteFilterName) { - return cteDefinitions.containsKey(cteFilterName); - } - - @Getter - public static class CteDefinitionWithOffset { - // Query item id - private String itemId; - // The program stage uid - private final String programStageUid; - // The program indicator uid - private String programIndicatorUid; - // The CTE definition (the SQL query) - private final String cteDefinition; - // The calculated offset - private final int offset; - // The alias of the CTE - private final String alias; - // Whether the CTE is a row context (TODO this need a better explanation) - private boolean isRowContext; - // Whether the CTE is a program indicator - private boolean isProgramIndicator = false; - // Whether the CTE is a filter - private boolean isFilter = false; - // Whether the CTE is a exists, used for checking if the enrollment exists - private boolean isExists = false; - - private static final String PS_PREFIX = "ps"; - private static final String PI_PREFIX = "pi"; - - public CteDefinitionWithOffset setExists(boolean exists) { - this.isExists = exists; - return this; + /** + * Adds a CTE definition to the context. + * + * @param programStage The program stage + * @param item The query item + * @param cteDefinition The CTE definition (the SQL query) + * @param offset The calculated offset + * @param isRowContext Whether the CTE is a row context + */ + public void addCTE( + ProgramStage programStage, + QueryItem item, + String cteDefinition, + int offset, + boolean isRowContext) { + String key = computeKey(item); + if (cteDefinitions.containsKey(key)) { + cteDefinitions.get(key).getOffsets().add(offset); + } else { + var cteDef = new CteDefinition(programStage.getUid(), item.getItemId(), cteDefinition, offset, isRowContext); + cteDefinitions.put(key, cteDef); + } } - public CteDefinitionWithOffset(String programStageUid, String queryItemId, String cteDefinition, int offset) { - this.programStageUid = programStageUid; - this.itemId = queryItemId; - this.cteDefinition = cteDefinition; - this.offset = offset; - this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build().generate(5); - this.isRowContext = false; + public void addExistsCTE( + ProgramStage programStage, + QueryItem item, + String cteDefinition) { + var cteDef = new CteDefinition(programStage.getUid(), item.getItemId(), cteDefinition, -999, false) + .setExists(true); + cteDefinitions.put( + programStage.getUid(), + cteDef); } - public CteDefinitionWithOffset( - String programStageUid, String queryItemId, String cteDefinition, int offset, boolean isRowContext) { - this(programStageUid, queryItemId, cteDefinition, offset); - this.isRowContext = isRowContext; + /** + * Adds a CTE definition to the context. + * + * @param programIndicator The program indicator + * @param cteDefinition The CTE definition (the SQL query) + */ + public void addProgramIndicatorCTE(ProgramIndicator programIndicator, String cteDefinition) { + cteDefinitions.put( + programIndicator.getUid(), + new CteDefinition(programIndicator.getUid(), cteDefinition)); } - public CteDefinitionWithOffset(String programIndicatorUid, String cteDefinition) { - this.cteDefinition = cteDefinition; - this.programIndicatorUid = programIndicatorUid; - this.programStageUid = null; - this.offset = -999; - this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build().generate(5); - this.isRowContext = false; - this.isProgramIndicator = true; + public void addCTEFilter(QueryItem item, String ctedefinition) { + String key = computeKey(item); + if (!cteDefinitions.containsKey(key)) { + ProgramStage programStage = item.getProgramStage(); + cteDefinitions.put(key, new CteDefinition(item.getItemId(), + programStage == null ? null : programStage.getUid(), + ctedefinition, + true)); + } } - public CteDefinitionWithOffset(String cteFilterName, String cteDefinition, boolean isFilter) { - this.cteDefinition = cteDefinition; - this.programIndicatorUid = null; - this.programStageUid = null; - this.offset = -999; - this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build().generate(5); - this.isRowContext = false; - this.isProgramIndicator = false; - this.isFilter = isFilter; + public String getCTEDefinition() { + if (cteDefinitions.isEmpty()) { + return ""; + } + + StringBuilder sb = new StringBuilder("WITH "); + boolean first = true; + for (Map.Entry entry : cteDefinitions.entrySet()) { + if (!first) { + sb.append(", "); + } + CteDefinition cteDef = entry.getValue(); + sb.append(cteDef.asCteName(entry.getKey())) + .append(" AS (") + .append(entry.getValue().getCteDefinition()) + .append(")"); + first = false; + } + return sb.toString(); } - /** - * @param uid the uid of an dimension item or ProgramIndicator - * @return the name of the CTE - */ - public String asCteName(String uid) { - if (isExists) { - return uid.toLowerCase(); - } - if (isProgramIndicator) { - return "%s_%s".formatted(PI_PREFIX, programIndicatorUid.toLowerCase()); - } - if (isFilter) { - return uid.toLowerCase(); - } - - return "%s_%s_%s".formatted(PS_PREFIX, programStageUid.toLowerCase(), uid.toLowerCase()); + // Rename to item uid + public Set getCTENames() { + return cteDefinitions.keySet(); } - public boolean isProgramStage() { - return !isFilter && !isProgramIndicator && !isExists; + public boolean containsCte(String cteName) { + return cteDefinitions.containsKey(cteName); } - public boolean isExists() { - return isExists; - } - } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java index 43511d86568..504f5556561 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CTEUtils.java @@ -27,16 +27,21 @@ */ package org.hisp.dhis.analytics.common; +import lombok.experimental.UtilityClass; import org.hisp.dhis.common.QueryItem; +@UtilityClass public class CTEUtils { - public static String createFilterName(QueryItem queryItem) { - return "filter_" + getIdentifier(queryItem).replace('.', '_').toLowerCase(); - } + public static String computeKey(QueryItem queryItem) { - public static String createFilterNameByIdentifier(String identifier) { - return "filter_" + identifier.replace('.', '_').toLowerCase(); + if (queryItem.hasProgramStage()) { + return "%s_%s".formatted( + queryItem.getProgramStage().getUid(), + queryItem.getItemId()); + } + // TODO continue with the rest of the method + return ""; } public static String getIdentifier(QueryItem queryItem) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CteDefinition.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CteDefinition.java new file mode 100644 index 00000000000..4b64d48e3c2 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/CteDefinition.java @@ -0,0 +1,125 @@ +package org.hisp.dhis.analytics.common; + +import lombok.Getter; +import org.apache.commons.text.RandomStringGenerator; + +import java.util.ArrayList; +import java.util.List; + +public class CteDefinition { + + // Query item id + @Getter + private String itemId; + // The program stage uid + private final String programStageUid; + // The program indicator uid + private String programIndicatorUid; + // The CTE definition (the SQL query) + @Getter + private final String cteDefinition; + // The calculated offset + @Getter + private final List offsets = new ArrayList<>(); + // The alias of the CTE + private final String alias; + // Whether the CTE is a row context (TODO this need a better explanation) + @Getter + private boolean rowContext; + // Whether the CTE is a program indicator + @Getter + private boolean programIndicator = false; + // Whether the CTE is a filter + @Getter + private boolean filter = false; + // Whether the CTE is a exists, used for checking if the enrollment exists + private boolean isExists = false; + + private static final String PS_PREFIX = "ps"; + private static final String PI_PREFIX = "pi"; + + public CteDefinition setExists(boolean exists) { + this.isExists = exists; + return this; + } + + public String getAlias() { + if (offsets.size() <= 1) { + return alias; + } + return computeAlias(offsets.get(0)); + } + + public String getAlias(int offset) { + return computeAlias(offset); + } + + private String computeAlias(int offset) { + return alias + "_" + offset; + } + + public CteDefinition(String programStageUid, String queryItemId, String cteDefinition, int offset) { + this.programStageUid = programStageUid; + this.itemId = queryItemId; + this.cteDefinition = cteDefinition; + this.offsets.add(offset); + // one alias per offset + this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build().generate(5); + this.rowContext = false; + } + + public CteDefinition( + String programStageUid, String queryItemId, String cteDefinition, int offset, boolean isRowContext) { + this(programStageUid, queryItemId, cteDefinition, offset); + this.rowContext = isRowContext; + } + + public CteDefinition(String programIndicatorUid, String cteDefinition) { + this.cteDefinition = cteDefinition; + this.programIndicatorUid = programIndicatorUid; + this.programStageUid = null; + // ignore offset + this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build().generate(5); + this.rowContext = false; + this.programIndicator = true; + } + + public CteDefinition(String queryItemId, String programStageUid, String cteDefinition, boolean isFilter) { + this.itemId = queryItemId; + this.cteDefinition = cteDefinition; + this.programIndicatorUid = null; + this.programStageUid = programStageUid; + // ignore offset + this.alias = new RandomStringGenerator.Builder().withinRange('a', 'z').build().generate(5); + this.rowContext = false; + this.programIndicator = false; + this.filter = isFilter; + } + + /** + * @param uid the uid of an dimension item or ProgramIndicator + * @return the name of the CTE + */ + public String asCteName(String uid) { + if (isExists) { + return uid.toLowerCase(); + } + if (programIndicator) { + return "%s_%s".formatted(PI_PREFIX, programIndicatorUid.toLowerCase()); + } + if (filter) { + return uid.toLowerCase(); + } + + return "%s_%s_%s".formatted(PS_PREFIX, programStageUid.toLowerCase(), uid.toLowerCase()); + } + + public boolean isProgramStage() { + return !filter && !programIndicator && !isExists; + } + + public boolean isExists() { + return isExists; + } + +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/InQueryCteFilter.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/InQueryCteFilter.java new file mode 100644 index 00000000000..85d98bbde80 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/InQueryCteFilter.java @@ -0,0 +1,102 @@ +package org.hisp.dhis.analytics.common; + +import org.hisp.dhis.common.QueryFilter; + +import java.util.List; +import java.util.function.Predicate; + +import static org.hisp.dhis.analytics.QueryKey.NV; +import static org.hisp.dhis.common.QueryOperator.IN; + +/** + * Mimics the logic of @{@link org.hisp.dhis.common.InQueryFilter} to be used in CTEs + */ +public class InQueryCteFilter { + + private final String filter; + + private final CteDefinition cteDefinition; + + private final String field; + + private final boolean isText; + + public InQueryCteFilter(String field, String encodedFilter, boolean isText, CteDefinition cteDefinition) { + this.filter = encodedFilter; + this.field = field; + this.isText = isText; + this.cteDefinition = cteDefinition; + } + + + public String getSqlFilter(int offset) { + + List filterItems = QueryFilter.getFilterItems(this.filter); + + StringBuilder condition = new StringBuilder(); + String alias = cteDefinition.getAlias(offset); + if (hasNonMissingValue(filterItems)) { + // TODO GIUSEPPE! + + if (hasMissingValue(filterItems)) { + + // TODO GIUSEPPE! + } + } else { + if (hasMissingValue(filterItems)) { + condition.append("%s.enrollment is not null".formatted(alias)); + condition.append(" and "); + condition.append("%s.%s is null".formatted(alias, field)); + } + } + + return condition.toString(); + + } + + /** + * Checks if the filter items contain any non-missing values (values that are not {@link org.hisp.dhis.analytics.QueryKey#NV}). + * Non-missing values represent actual values that should be included in the SQL IN clause. + * This method is used to determine if the generated SQL condition needs to include an IN clause. + * + * @param filterItems the list of filter items to check for non-missing values + * @return true if any item in the list is not equal to {@link org.hisp.dhis.analytics.QueryKey#NV}, indicating at least one + * actual value that should be included in the SQL IN clause; false if all values are missing + */ + private boolean hasNonMissingValue(List filterItems) { + return anyMatch(filterItems, this::isNotMissingItem); + } + + private boolean isNotMissingItem(String filterItem) { + return !isMissingItem(filterItem); + } + + private boolean isMissingItem(String filterItem) { + return NV.equals(filterItem); + } + + /** + * Checks if any item in the list matches the given predicate. + * + * @param filterItems the list of items to check + * @param predi the predicate to test against + * @return true if any item matches the predicate, false otherwise + */ + private boolean anyMatch(List filterItems, Predicate predi) { + return filterItems.stream().anyMatch(predi); + } + + /** + * Checks if the filter items contain any missing values represented by the special marker {@link org.hisp.dhis.analytics.QueryKey#NV}. + * Missing values indicate that the corresponding database field should be treated as NULL in the SQL query. + * This method is used to determine if the generated SQL condition needs to include an IS NULL clause. + * + * @param filterItems the list of filter items to check for missing values + * @return true if any item in the list equals{@link org.hisp.dhis.analytics.QueryKey#NV}, indicating a missing value that should + * be treated as NULL in the SQL query; false otherwise + * @see org.hisp.dhis.analytics.QueryKey#NV + */ + private boolean hasMissingValue(List filterItems) { + return anyMatch(filterItems, this::isMissingItem); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java deleted file mode 100644 index 3562ae1e032..00000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/RowContextUtils.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2004-2024, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.analytics.common; - -import org.hisp.dhis.analytics.common.CTEContext.CteDefinitionWithOffset; -import org.hisp.dhis.analytics.event.EventQueryParams; -import org.hisp.dhis.common.QueryItem; -import org.hisp.dhis.db.sql.SqlBuilder; - -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -public class RowContextUtils { - - /** - * Get where clauses for row context items - * @param cteContext CTE context - * @param params Event query parameters - * @param sqlBuilder SQL builder - * @return List of where clauses - */ - public static List getRowContextWhereClauses(CTEContext cteContext, EventQueryParams params, SqlBuilder sqlBuilder) { - List whereClauses = new ArrayList<>(); - Set ctxNames = cteContext.getCTENames(); - - List filters = getItemsWithFilters(params); - - for (String ctxName : ctxNames) { - CteDefinitionWithOffset cteDef = cteContext.getDefinitionByItemUid(ctxName); - // only add where clause for row context items with filters - if (cteDef.isRowContext() && filters.contains(cteDef.getItemId())) { - whereClauses.add("%s.%s IS NULL".formatted(cteDef.getAlias(), - sqlBuilder.quote(cteDef.getItemId()))); - } - // only add where clause for "exists" row context items with filters - if (cteDef.isExists() && filters.contains(cteDef.getItemId())) { - whereClauses.add("ee.enrollment IS NOT NULL"); - } - } - - return whereClauses; - } - - private static List getItemsWithFilters(EventQueryParams params) { - return params.getItems().stream() - .filter(QueryItem::hasFilter) - .map(QueryItem::getItemId) - .toList(); - } -} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 6264ea17cfd..a2506da6149 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -92,7 +92,7 @@ import org.hisp.dhis.analytics.SortOrder; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; import org.hisp.dhis.analytics.common.CTEContext; -import org.hisp.dhis.analytics.common.CTEContext.CteDefinitionWithOffset; +import org.hisp.dhis.analytics.common.CteDefinition; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; import org.hisp.dhis.analytics.event.EventQueryParams; import org.hisp.dhis.analytics.util.AnalyticsUtils; @@ -1404,8 +1404,7 @@ protected List getSelectColumnsWithCTE(EventQueryParams params, CTEConte if (queryItem.isProgramIndicator()) { // For program indicators, use CTE reference String piUid = queryItem.getItem().getUid(); - CteDefinitionWithOffset cteDef = cteContext.getDefinitionByItemUid(piUid); - // ugaee.value as "CH6wamtY9kK", + CteDefinition cteDef = cteContext.getDefinitionByItemUid(piUid); String col = "%s.value as %s".formatted(cteDef.getAlias(), piUid); columns.add(col); } else if (ValueType.COORDINATE == queryItem.getValueType()) { @@ -1428,8 +1427,9 @@ protected List getSelectColumnsWithCTE(EventQueryParams params, CTEConte columns.add(columnAndAlias.asSql()); } } - - return columns; + // remove duplicates + var ded = columns.stream().distinct().toList(); + return ded; } /** diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java index af922b26d55..2b4cdb2ee15 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEnrollmentAnalyticsManager.java @@ -27,40 +27,15 @@ */ package org.hisp.dhis.analytics.event.data; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.joining; -import static org.apache.commons.lang3.StringUtils.EMPTY; -import static org.hisp.dhis.analytics.AnalyticsConstants.ANALYTICS_TBL_ALIAS; -import static org.hisp.dhis.analytics.DataType.BOOLEAN; -import static org.hisp.dhis.analytics.common.CTEUtils.createFilterNameByIdentifier; -import static org.hisp.dhis.analytics.event.data.OrgUnitTableJoiner.joinOrgUnitTables; -import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; -import static org.hisp.dhis.common.DataDimensionType.ATTRIBUTE; -import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; -import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; -import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; -import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString; -import static org.hisp.dhis.commons.util.TextUtils.removeLastOr; -import static org.hisp.dhis.util.DateUtils.toMediumDate; - import com.google.common.collect.Sets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; import org.hisp.dhis.analytics.common.CTEContext; -import org.hisp.dhis.analytics.common.CTEContext.CteDefinitionWithOffset; import org.hisp.dhis.analytics.common.CTEUtils; +import org.hisp.dhis.analytics.common.CteDefinition; +import org.hisp.dhis.analytics.common.InQueryCteFilter; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; -import org.hisp.dhis.analytics.common.RowContextUtils; import org.hisp.dhis.analytics.event.EnrollmentAnalyticsManager; import org.hisp.dhis.analytics.event.EventQueryParams; import org.hisp.dhis.analytics.table.AbstractJdbcTableManager; @@ -95,6 +70,33 @@ import org.springframework.jdbc.support.rowset.SqlRowSet; import org.springframework.stereotype.Service; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.joining; +import static org.apache.commons.lang3.StringUtils.EMPTY; +import static org.hisp.dhis.analytics.AnalyticsConstants.ANALYTICS_TBL_ALIAS; +import static org.hisp.dhis.analytics.DataType.BOOLEAN; +import static org.hisp.dhis.analytics.common.CTEUtils.computeKey; +import static org.hisp.dhis.analytics.event.data.OrgUnitTableJoiner.joinOrgUnitTables; +import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; +import static org.hisp.dhis.common.DataDimensionType.ATTRIBUTE; +import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; +import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; +import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; +import static org.hisp.dhis.common.QueryOperator.IN; +import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString; +import static org.hisp.dhis.commons.util.TextUtils.removeLastOr; +import static org.hisp.dhis.util.DateUtils.toMediumDate; + /** * @author Markus Bekken */ @@ -505,51 +507,46 @@ private String addFiltersToWhereClause(EventQueryParams params) { return getQueryItemsAndFiltersWhereClause(params, new SqlHelper()); } - private String addRowContextFilters(EventQueryParams params, CTEContext cteContext) { - - List rowContextColumns = RowContextUtils.getRowContextWhereClauses(cteContext, params, sqlBuilder); - if (!rowContextColumns.isEmpty()) { - return String.join(" AND ", rowContextColumns); - } - return EMPTY; - } - private String addCteFiltersToWhereClause(EventQueryParams params, CTEContext cteContext) { StringBuilder whereClause = new StringBuilder(); - // Iterate over each filter and apply the correct condition - for (QueryItem item : - Stream.concat(params.getItems().stream(), params.getItemFilters().stream()) + // Get all filters from the query items and item filters + List filters = Stream.concat(params.getItems().stream(), params.getItemFilters().stream()) .filter(QueryItem::hasFilter) - .toList()) { + .toList(); + + // Iterate over each filter and apply the correct condition + for (QueryItem item : filters) { - String cteName = CTEUtils.createFilterName(item); + String cteName = CTEUtils.computeKey(item); - if (cteContext.containsCteFilter(cteName)) { - CteDefinitionWithOffset cteDef = cteContext.getDefinitionByItemUid(cteName); + if (cteContext.containsCte(cteName)) { + CteDefinition cteDef = cteContext.getDefinitionByItemUid(cteName); for (QueryFilter filter : item.getFilters()) { - if ("NV".equals(filter.getFilter())) { // Handle null filters explicitly - whereClause.append(" AND ").append(cteDef.getAlias()).append(".value IS NULL"); + if (IN.equals(filter.getOperator())) { + InQueryCteFilter inQueryCteFilter + = new InQueryCteFilter(sqlBuilder.quote(item.getItemName()), filter.getFilter(), item.isText(), cteDef); + whereClause.append(" and ").append(inQueryCteFilter.getSqlFilter(createOffset2(item.getProgramStageOffset()))); } else { - String operator = getSqlOperator(filter); - String value = getSqlFilterValue(filter, item); - whereClause - .append(" AND ") - .append(cteDef.getAlias()) - .append(".value ") - .append(operator) - .append(" ") - .append(value); + String operator = getSqlOperator(filter); + String value = getSqlFilterValue(filter, item); + whereClause + .append(" AND ") + .append(cteDef.getAlias()) + .append(".value ") + .append("NULL".equals(value) ? "IS" : operator) + .append(" ") + .append(value); } } } else { // If the filter is not part of the CTE, apply it directly to the enrollment table // using the standard where clause method - String filters = getQueryItemsAndFiltersWhereClause(params, new SqlHelper()); - if (StringUtils.isNotBlank(filters) && filters.trim().startsWith("where")) { + String whereConditionFromFilter = getQueryItemsAndFiltersWhereClause(params, new SqlHelper()); + if (StringUtils.isNotBlank(whereConditionFromFilter) && whereConditionFromFilter.trim().startsWith("where")) { // remove the 'where' keyword - filters = filters.trim().substring(5); - whereClause.append("and ").append(filters); + whereConditionFromFilter = whereConditionFromFilter.trim().substring(5); + whereClause.append("and ").append(whereConditionFromFilter); } } } @@ -610,13 +607,42 @@ private String buildFilterCteSql(List queryItems, EventQueryParams pa : ""; // Add program stage filter if available // Generate the CTE SQL - return String.format( - """ - SELECT DISTINCT ON (enrollment) enrollment, %s AS value - FROM %s - WHERE eventstatus != 'SCHEDULE' %s - ORDER BY enrollment, occurreddate DESC, created DESC""", - columnName, tableName, programStageCondition); +// var old = String.format( +// """ +// SELECT DISTINCT ON (enrollment) enrollment, %s AS value +// FROM %s +// WHERE eventstatus != 'SCHEDULE' %s +// ORDER BY enrollment, occurreddate DESC, created DESC""", +// columnName, tableName, programStageCondition); + + return """ + select + enrollment, + %s as value + from + (select + enrollment, + %s, + row_number() over ( + partition by enrollment + order by + occurreddate desc, + created desc + ) as rn + from + %s + where + eventstatus != 'SCHEDULE' + %s + ) ranked + where + rn = 1 + """.formatted( + columnName, + columnName, + tableName, + programStageCondition); + }) .collect(Collectors.joining("\nUNION ALL\n")); } @@ -714,34 +740,16 @@ protected String getColumnWithCte(QueryItem item, String suffix, CTEContext cteC List columns = new ArrayList<>(); String colName = item.getItemName(); - CteDefinitionWithOffset cteDef = cteContext.getDefinitionByItemUid(item.getItem().getUid()); - + CteDefinition cteDef = cteContext.getDefinitionByItemUid(computeKey(item)); + int programStageOffset = createOffset2(item.getProgramStageOffset()); String alias = getAlias(item).orElse(null); - - // ed."lJTx9EZ1dk1" as "EPEcjy3FWmI[-1].lJTx9EZ1dk1" - - columns.add( - """ - %s.%s as %s - """ - .formatted(cteDef.getAlias(), quote(colName), quote(alias))); + columns.add("%s.%s as %s" .formatted(cteDef.getAlias(programStageOffset), quote(colName), quote(alias))); if (cteDef.isRowContext()) { // Add additional status and exists columns for row context - // (ed."lJTx9EZ1dk1" IS NOT NULL) as "EPEcjy3FWmI[-1].lJTx9EZ1dk1.exists", - // ed.eventstatus as "EPEcjy3FWmI[-1].lJTx9EZ1dk1.status" - columns.add( - """ - (%s.%s IS NOT NULL) as %s - """ - .formatted(cteDef.getAlias(), quote(colName), quote(alias + ".exists"))); - - columns.add( - """ - %s.eventstatus as %s - """ - .formatted(cteDef.getAlias(), quote(alias + ".status"))); + columns.add("(%s.%s IS NOT NULL) as %s" .formatted(cteDef.getAlias(programStageOffset), quote(colName), quote(alias + ".exists"))); + columns.add("%s.eventstatus as %s".formatted(cteDef.getAlias(programStageOffset), quote(alias + ".status"))); } - return String.join(", ", columns); + return String.join(",\n", columns); } /** @@ -945,15 +953,7 @@ private CTEContext getCteDefinitions(EventQueryParams params) { if (item.isProgramIndicator()) { handleProgramIndicatorCte(item, cteContext, params); } else if (item.hasProgramStage()) { - // TODO what is this condition would be good to give it a name - if (item.getProgramStage().getRepeatable() - && item.hasRepeatableStageParams() - && !item.getRepeatableStageParams().simpleStageValueExpected()) { - - // TODO: Implement repeatable stage items - log.warn("Repeatable stage items are not yet supported"); - // TODO what is this condition - would be good to give it a name - } else if (item.getProgramStage().getRepeatable() && item.hasRepeatableStageParams()) { + if (item.getProgramStage().getRepeatable() && item.hasRepeatableStageParams()) { String colName = quote(item.getItemName()); boolean hasEventStatusColumn = rowContextAllowedAndNeeded(params, item); @@ -1021,7 +1021,7 @@ select distinct on (enrollment) enrollment, %s as value LIMIT_1); cteContext.addCTE( - item.getProgramStage(), item, cteSql, createOffset2(item.getProgramStageOffset())); + item.getProgramStage(), item, cteSql, createOffset2(item.getProgramStageOffset()), false); } } } @@ -1062,9 +1062,9 @@ private void generateFilterCTEs(EventQueryParams params, CTEContext cteContext) .collect(groupingBy(CTEUtils::getIdentifier)) .forEach( (identifier, items) -> { - String cteName = createFilterNameByIdentifier(identifier); String cteSql = buildFilterCteSql(items, params); - cteContext.addCTEFilter(cteName, cteSql); + // TODO is this correct? items.get(0) + cteContext.addCTEFilter(items.get(0), cteSql); }); // Process non-repeatable stage filters @@ -1073,9 +1073,8 @@ private void generateFilterCTEs(EventQueryParams params, CTEContext cteContext) .forEach( queryItem -> { if (queryItem.hasProgram() && queryItem.hasProgramStage()) { - String cteName = CTEUtils.createFilterName(queryItem); String cteSql = buildFilterCteSql(List.of(queryItem), params); - cteContext.addCTEFilter(cteName, cteSql); + cteContext.addCTEFilter(queryItem, cteSql); } }); } @@ -1101,33 +1100,42 @@ private String buildEnrollmentQueryWithCte(EventQueryParams params) { ListUtils.distinctUnion( params.isAggregatedEnrollments() ? List.of("enrollment") : COLUMNS, getSelectColumnsWithCTE(params, cteContext)); - sql.append("SELECT ").append(String.join(",\n", selectCols)); + sql.append("select ").append(String.join(",\n", selectCols)); // 4. From clause - sql.append("\nFROM ").append(params.getTableName()).append(" AS ax"); + sql.append("\nfrom ").append(params.getTableName()).append(" AS ax"); // 5. Add joins for each CTE + final String LEFT_JOIN = " left join "; for (String itemUid : cteContext.getCTENames()) { - CteDefinitionWithOffset cteDef = cteContext.getDefinitionByItemUid(itemUid); - if (!cteDef.isExists()) { - String join = - """ - LEFT JOIN %s %s - ON - %s.enrollment = ax.enrollment - """.formatted(cteDef.asCteName(itemUid), cteDef.getAlias(), cteDef.getAlias()); - sql.append("\n").append(join); - } else { - sql.append(""" - \nLEFT JOIN %s ee - ON ee.enrollment = ax.enrollment - """.formatted(cteDef.asCteName(itemUid))); - } + CteDefinition cteDef = cteContext.getDefinitionByItemUid(itemUid); if (cteDef.isProgramStage()) { - // equivalent to original OFFSET 1 LIMIT 1 but more efficient - // TODO use constant instead of hardcoded 'rn' column name - String offset = " and %s.rn = %s".formatted(cteDef.getAlias(), cteDef.getOffset() + 1); - sql.append(offset); + List offsets = cteDef.getOffsets(); + for (Integer offset : offsets) { + String alias = cteDef.getAlias(offset); + String join = + "%s %s ON %s.enrollment = ax.enrollment and %s.rn = %s".formatted(cteDef.asCteName(itemUid), + alias, + alias, + alias, + offset + 1); + sql.append(LEFT_JOIN).append(join); + } + } + if (cteDef.isExists()) { + sql.append(LEFT_JOIN).append("%s ee on ee.enrollment = ax.enrollment".formatted(cteDef.asCteName(itemUid))); + } + if (cteDef.isProgramIndicator()) { + sql.append(LEFT_JOIN).append("%s %s on %s.enrollment = ax.enrollment".formatted( + cteDef.asCteName(itemUid), + cteDef.getAlias(), + cteDef.getAlias())); + } + if (cteDef.isFilter()) { + sql.append(LEFT_JOIN).append("%s %s on %s.enrollment = ax.enrollment".formatted( + cteDef.asCteName(itemUid), + cteDef.getAlias(), + cteDef.getAlias())); } } @@ -1152,7 +1160,6 @@ private List collectWhereConditions(EventQueryParams params, CTEContext String baseWhereClause = getWhereClause(params).trim(); String cteFilters = addCteFiltersToWhereClause(params, cteContext).trim(); - String rowContextFilters = addRowContextFilters(params, cteContext).trim(); // Add non-empty conditions if (!baseWhereClause.isEmpty()) { @@ -1162,9 +1169,7 @@ private List collectWhereConditions(EventQueryParams params, CTEContext if (!cteFilters.isEmpty()) { conditions.add(cteFilters.replaceFirst("(?i)^AND\\s+", "")); } - if (!rowContextFilters.isEmpty()) { - conditions.add(rowContextFilters); - } + return conditions; }