From a335311081d28282610c85c7c0070675e0e0e228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 15:01:43 +0100 Subject: [PATCH 01/35] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 11 +++++------ .../analytics/table/AbstractJdbcTableManager.java | 15 +++++++-------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index e72d58f66143..27a8c0b2411b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -32,7 +32,6 @@ import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; - import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -260,11 +259,11 @@ protected String selectForInsert( TrackedEntityAttribute attribute, String fromType, String dataClause) { return replaceQualify( """ - (select ${fromType} from ${trackedentityattributevalue} \ - where trackedentityid=en.trackedentityid \ - and trackedentityattributeid=${attributeId}\ - ${dataClause})\ - ${closingParentheses} as ${attributeUid}""", + (select ${fromType} from ${trackedentityattributevalue} \ + where trackedentityid=en.trackedentityid \ + and trackedentityattributeid=${attributeId}\ + ${dataClause})\ + ${closingParentheses} as ${attributeUid}""", Map.of( "fromType", fromType, "dataClause", dataClause, diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index d9d27b66892d..ef0c5789ff5c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -36,7 +36,6 @@ import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.db.model.constraint.Nullable.NOT_NULL; import static org.hisp.dhis.util.DateUtils.toLongDate; - import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -45,8 +44,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHook; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableManager; @@ -90,6 +87,8 @@ import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.util.Assert; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; /** * @author Lars Helge Overland @@ -746,11 +745,11 @@ protected AnalyticsTableColumn getPartitionColumn() { // is part of the previous PR (https://github.com/dhis2/dhis2-core/pull/19131/files) .selectExpression( """ - CASE - WHEN ev.status = 'SCHEDULE' THEN YEAR(ev.scheduleddate) - ELSE YEAR(ev.occurreddate) - END - """) + CASE + WHEN ev.status = 'SCHEDULE' THEN YEAR(ev.scheduleddate) + ELSE YEAR(ev.occurreddate) + END + """) .skipIndex(Skip.SKIP) .build(); } From 6d20921d1cadbf1b6f1fb84a043f112ae57c4799 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 15:03:20 +0100 Subject: [PATCH 02/35] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 1 + .../analytics/table/AbstractJdbcTableManager.java | 5 +++-- .../table/JdbcEventAnalyticsTableManager.java | 14 +++++++------- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 27a8c0b2411b..f1244e61ecd4 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -32,6 +32,7 @@ import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; + import java.util.ArrayList; import java.util.List; import java.util.Map; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index ef0c5789ff5c..2a4b27ff8c8b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -36,6 +36,7 @@ import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.db.model.constraint.Nullable.NOT_NULL; import static org.hisp.dhis.util.DateUtils.toLongDate; + import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -44,6 +45,8 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.AnalyticsTableHook; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableManager; @@ -87,8 +90,6 @@ import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.util.Assert; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; /** * @author Lars Helge Overland diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 0837bc24ca0b..19fa4f13a404 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -657,7 +657,12 @@ private List getColumnFromOrgUnitDataElement( * @return A SQL select expression for the data element */ private String selectForInsert(DataElement dataElement, String fromType, String dataClause) { - Map replacements = + String sqlTemplate = + dataElement.getValueType().isOrganisationUnit() + ? "(select ${fromType} ${dataClause})${closingParentheses} as ${dataElementUid}" + : "(select ${fromType} from ${event} where eventid=ev.eventid ${dataClause})${closingParentheses} as ${dataElementUid}"; + + Map variables = Map.of( "fromType", fromType, @@ -668,12 +673,7 @@ private String selectForInsert(DataElement dataElement, String fromType, String "dataElementUid", quote(dataElement.getUid())); - String sqlTemplate = - dataElement.getValueType().isOrganisationUnit() - ? "(select ${fromType} ${dataClause})${closingParentheses} as ${dataElementUid}" - : "(select ${fromType} from ${event} where eventid=ev.eventid ${dataClause})${closingParentheses} as ${dataElementUid}"; - - return replaceQualify(sqlTemplate, replacements); + return replaceQualify(sqlTemplate, variables); } private List getColumnFromDataElementWithLegendSet( From 38c1df8f2d9d680b506d8c0ca717a1b34c7cd7c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 15:57:31 +0100 Subject: [PATCH 03/35] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 12 ++++++------ .../table/JdbcEventAnalyticsTableManager.java | 6 +++--- .../JdbcTrackedEntityAnalyticsTableManager.java | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index f1244e61ecd4..7569957175ff 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -108,12 +108,12 @@ protected Skip skipIndex(ValueType valueType, boolean hasOptionSet) { return skipIndex ? Skip.SKIP : Skip.INCLUDE; } - protected String getSelectClause(ValueType valueType, String columnName) { - return getSelectClauseInternal(valueType, columnName, false); + protected String getSelectExpression(ValueType valueType, String columnName) { + return getSelectExpressionInternal(valueType, columnName, false); } - protected String getSelectClauseForTea(ValueType valueType, String columnName) { - return getSelectClauseInternal(valueType, columnName, true); + protected String getExpressionForTea(ValueType valueType, String columnName) { + return getSelectExpressionInternal(valueType, columnName, true); } /** @@ -126,7 +126,7 @@ protected String getSelectClauseForTea(ValueType valueType, String columnName) { * true, organization unit selections will include an additional subquery wrapper * @return A SQL select expression appropriate for the given value type and context */ - private String getSelectClauseInternal( + private String getSelectExpressionInternal( ValueType valueType, String columnName, boolean isTeaContext) { String doubleType = sqlBuilder.dataTypeDouble(); @@ -201,7 +201,7 @@ protected List getTrackedEntityAttributeColumns(Program pr attribute.isNumericType() ? getNumericClause() : attribute.isDateType() ? getDateClause() : ""; - String select = getSelectClauseForTea(attribute.getValueType(), "value"); + String select = getExpressionForTea(attribute.getValueType(), "value"); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); String sql = diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 19fa4f13a404..80299d4a1cea 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -489,7 +489,7 @@ private List getColumnFromTrackedEntityAttribute( DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); String dataClause = attribute.isNumericType() ? numericClause : attribute.isDateType() ? dateClause : ""; - String select = getSelectClauseForTea(attribute.getValueType(), "value"); + String select = getExpressionForTea(attribute.getValueType(), "value"); String sql = selectForInsert(attribute, select, dataClause); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); @@ -512,7 +512,7 @@ private List getColumnFromTrackedEntityAttribute( private List getColumnFromTrackedEntityAttributeWithLegendSet( TrackedEntityAttribute attribute, String numericClause) { - String selectClause = getSelectClause(attribute.getValueType(), "value"); + String selectClause = getSelectExpression(attribute.getValueType(), "value"); String query = """ \s(select l.uid from ${maplegend} l \ @@ -553,7 +553,7 @@ private List getColumnFromDataElement( String dataClause = getDataClause(dataElement.getUid(), dataElement.getValueType()); String columnName = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); - String select = getSelectClause(dataElement.getValueType(), columnName); + String select = getSelectExpression(dataElement.getValueType(), columnName); String sql = selectForInsert(dataElement, select, dataClause); Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index e23833d08d29..a4e1fc4b8997 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -255,7 +255,7 @@ private Stream getAllTrackedEntityAttributes( /** * Returns the select clause, potentially with a cast statement, based on the given value type. * (this method is an adapted version of {@link - * JdbcEventAnalyticsTableManager#getSelectClause(ValueType, String)}) + * JdbcEventAnalyticsTableManager#getSelectExpression(ValueType, String)}) * * @param valueType the value type to represent as database column type. */ From 49b58ea1c3212f4dce4004517c41d9d1474c6b0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 15:58:01 +0100 Subject: [PATCH 04/35] fix: Update code --- .../dhis/analytics/table/AbstractEventJdbcTableManager.java | 4 ++-- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 7569957175ff..2632d4602a05 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -112,7 +112,7 @@ protected String getSelectExpression(ValueType valueType, String columnName) { return getSelectExpressionInternal(valueType, columnName, false); } - protected String getExpressionForTea(ValueType valueType, String columnName) { + protected String getSelectExpressionForTea(ValueType valueType, String columnName) { return getSelectExpressionInternal(valueType, columnName, true); } @@ -201,7 +201,7 @@ protected List getTrackedEntityAttributeColumns(Program pr attribute.isNumericType() ? getNumericClause() : attribute.isDateType() ? getDateClause() : ""; - String select = getExpressionForTea(attribute.getValueType(), "value"); + String select = getSelectExpressionForTea(attribute.getValueType(), "value"); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); String sql = diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 80299d4a1cea..6a89ac2300b1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -489,7 +489,7 @@ private List getColumnFromTrackedEntityAttribute( DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); String dataClause = attribute.isNumericType() ? numericClause : attribute.isDateType() ? dateClause : ""; - String select = getExpressionForTea(attribute.getValueType(), "value"); + String select = getSelectExpressionForTea(attribute.getValueType(), "value"); String sql = selectForInsert(attribute, select, dataClause); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); From 3b8c3ff7c30d057cca794aa96adee30e03b1c4fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 15:59:10 +0100 Subject: [PATCH 05/35] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 2632d4602a05..311adb191ab8 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -117,12 +117,13 @@ protected String getSelectExpressionForTea(ValueType valueType, String columnNam } /** - * Returns the select clause, potentially with a cast statement, based on the given value type. - * This internal method handles both Data Value and Tracked Entity Attribute (TEA) select clauses. + * Returns the select expression, potentially with a cast statement, based on the given value + * type. This internal method handles both Data Value and Tracked Entity Attribute (TEA) select + * expressions. * - * @param valueType The value type to represent as database column type - * @param columnName The name of the column to be selected - * @param isTeaContext Whether the selection is in the context of a Tracked Entity Attribute. When + * @param valueType the value type to represent as database column type + * @param columnName the name of the column to be selected + * @param isTeaContext whether the selection is in the context of a Tracked Entity Attribute. When * true, organization unit selections will include an additional subquery wrapper * @return A SQL select expression appropriate for the given value type and context */ From 3fec6c89299d7696f41a380a08354a0df79fa4d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:00:13 +0100 Subject: [PATCH 06/35] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 6a89ac2300b1..d4ab760e0faa 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -553,8 +553,8 @@ private List getColumnFromDataElement( String dataClause = getDataClause(dataElement.getUid(), dataElement.getValueType()); String columnName = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); - String select = getSelectExpression(dataElement.getValueType(), columnName); - String sql = selectForInsert(dataElement, select, dataClause); + String selectExpression = getSelectExpression(dataElement.getValueType(), columnName); + String sql = getSelectForInsert(dataElement, selectExpression, dataClause); Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); if (dataElement.getValueType().isOrganisationUnit()) { @@ -570,7 +570,7 @@ private List getColumnFromDataElement( .build()); return withLegendSet - ? getColumnFromDataElementWithLegendSet(dataElement, select, dataClause) + ? getColumnFromDataElementWithLegendSet(dataElement, selectExpression, dataClause) : columns; } @@ -621,7 +621,7 @@ private List getColumnFromOrgUnitDataElement( if (isSpatialSupport()) { String fromType = "ou.geometry " + fromClause; - String geoSql = selectForInsert(dataElement, fromType, dataClause); + String geoSql = getSelectForInsert(dataElement, fromType, dataClause); columns.add( AnalyticsTableColumn.builder() @@ -634,7 +634,7 @@ private List getColumnFromOrgUnitDataElement( } String fromTypeSql = "ou.name " + fromClause; - String ouNameSql = selectForInsert(dataElement, fromTypeSql, dataClause); + String ouNameSql = getSelectForInsert(dataElement, fromTypeSql, dataClause); columns.add( AnalyticsTableColumn.builder() @@ -656,7 +656,7 @@ private List getColumnFromOrgUnitDataElement( * @param dataClause The data type related clause * @return A SQL select expression for the data element */ - private String selectForInsert(DataElement dataElement, String fromType, String dataClause) { + private String getSelectForInsert(DataElement dataElement, String fromType, String dataClause) { String sqlTemplate = dataElement.getValueType().isOrganisationUnit() ? "(select ${fromType} ${dataClause})${closingParentheses} as ${dataElementUid}" From 87f0a72de21362a5ad4f6aad473b36004a9ea22e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:03:39 +0100 Subject: [PATCH 07/35] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index d4ab760e0faa..acc7e725a0d0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -651,10 +651,10 @@ private List getColumnFromOrgUnitDataElement( /** * Creates a select statement for data element insertion. * - * @param dataElement The data element to create the select statement for - * @param fromType The SQL snippet for the "from" part of the query - * @param dataClause The data type related clause - * @return A SQL select expression for the data element + * @param dataElement the data element to create the select statement for. + * @param fromType the SQL snippet for the "from" part of the query. + * @param dataClause the data type related clause. + * @return A SQL select expression for the data element. */ private String getSelectForInsert(DataElement dataElement, String fromType, String dataClause) { String sqlTemplate = @@ -764,8 +764,8 @@ private List getDataYears( * Retrieve years for partition tables. Year will become a partition key. The default return value * is the list with the recent year. * - * @param dataYears list of years coming from inner join of event and enrollment tables - * @return list of partition key values + * @param dataYears the list of years coming from inner join of event and enrollment tables. + * @return list of partition key values. */ private List getYearsForPartitionTable(List dataYears) { return ListUtils.mutableCopy(!dataYears.isEmpty() ? dataYears : List.of(Year.now().getValue())); From 0dcaacf599e4ff6374d47814c874ec885e049e18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:05:40 +0100 Subject: [PATCH 08/35] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index acc7e725a0d0..6706ba80d5e9 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -28,6 +28,7 @@ package org.hisp.dhis.analytics.table; import static java.util.stream.Collectors.toList; +import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.hisp.dhis.analytics.table.model.Skip.SKIP; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getClosingParentheses; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; @@ -717,7 +718,7 @@ private String getDataClause(String uid, ValueType valueType) { return " and " + sqlBuilder.regexpMatch(jsonValue, "'" + regex + "'"); } - return ""; + return EMPTY; } private List getDataYears( @@ -734,7 +735,7 @@ private List getDataYears( eventDateExpression, "fromDate", toMediumDate(params.getFromDate()))) - : ""; + : EMPTY; String sql = replaceQualify( """ From a3979733ff39a911f122b0ee36408a281f0c4a5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:09:44 +0100 Subject: [PATCH 09/35] fix: Update code --- .../dhis/analytics/table/AbstractEventJdbcTableManager.java | 4 ++-- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 311adb191ab8..9ed57b532275 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -254,8 +254,8 @@ protected List getTrackedEntityAttributeColumns(Program pr * * @param attribute the {@link TrackedEntityAttribute}. * @param fromType the sql snippet related to "from" part - * @param dataClause the data type related clause like "NUMERIC" - * @return + * @param dataClause the data type related clause like "NUMERIC". + * @return a select statement. */ protected String selectForInsert( TrackedEntityAttribute attribute, String fromType, String dataClause) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 6706ba80d5e9..95058e2d3793 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -706,7 +706,7 @@ private List getColumnFromDataElementWithLegendSet( .selectExpression(sql) .build(); }) - .collect(toList()); + .toList(); } private String getDataClause(String uid, ValueType valueType) { From 74abf2314bc10aa9d63ca7ae62e65bff0f9f528b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:25:17 +0100 Subject: [PATCH 10/35] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 95058e2d3793..f51d49dc3f54 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -551,7 +551,7 @@ private List getColumnFromDataElement( List columns = new ArrayList<>(); DataType dataType = getColumnType(dataElement.getValueType(), isSpatialSupport()); - String dataClause = getDataClause(dataElement.getUid(), dataElement.getValueType()); + String dataClause = getDataValueExpression(dataElement.getUid(), dataElement.getValueType()); String columnName = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); String selectExpression = getSelectExpression(dataElement.getValueType(), columnName); @@ -709,7 +709,14 @@ private List getColumnFromDataElementWithLegendSet( .toList(); } - private String getDataClause(String uid, ValueType valueType) { + /** + * Returns an expression for extracting a data value. + * + * @param uid the identifier. + * @param valueType the {@link ValueType}. + * @return an expression for extracting a data value. + */ + private String getDataValueExpression(String uid, ValueType valueType) { if (valueType.isNumeric() || valueType.isDate()) { String regex = valueType.isNumeric() ? NUMERIC_LENIENT_REGEXP : DATE_REGEXP; @@ -721,11 +728,20 @@ private String getDataClause(String uid, ValueType valueType) { return EMPTY; } + /** + * Returns a list of years for which data exist. + * + * @param params the {@link AnalyticsTableUpdateParams}. + * @param program the {@link Program}. + * @param firstDataYear the first year to include. + * @param lastDataYear the last data year to include. + * @return a list of years for which data exist. + */ private List getDataYears( AnalyticsTableUpdateParams params, Program program, Integer firstDataYear, - Integer latestDataYear) { + Integer lastDataYear) { String fromDateClause = params.getFromDate() != null ? replace( @@ -756,7 +772,7 @@ private List getDataYears( "programId", String.valueOf(program.getId()), "fromDateClause", fromDateClause, "firstDataYear", String.valueOf(firstDataYear), - "latestDataYear", String.valueOf(latestDataYear))); + "latestDataYear", String.valueOf(lastDataYear))); return jdbcTemplate.queryForList(sql, Integer.class); } From a78d712e06abe6fc7988fed04d7564deadb24830 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:26:09 +0100 Subject: [PATCH 11/35] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index f51d49dc3f54..05eb60c8cec7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -660,8 +660,8 @@ private List getColumnFromOrgUnitDataElement( private String getSelectForInsert(DataElement dataElement, String fromType, String dataClause) { String sqlTemplate = dataElement.getValueType().isOrganisationUnit() - ? "(select ${fromType} ${dataClause})${closingParentheses} as ${dataElementUid}" - : "(select ${fromType} from ${event} where eventid=ev.eventid ${dataClause})${closingParentheses} as ${dataElementUid}"; + ? "(select ${fromType} ${dataClause})${closingParentheses} as ${uid}" + : "(select ${fromType} from ${event} where eventid=ev.eventid ${dataClause})${closingParentheses} as ${uid}"; Map variables = Map.of( @@ -671,7 +671,7 @@ private String getSelectForInsert(DataElement dataElement, String fromType, Stri dataClause, "closingParentheses", getClosingParentheses(fromType), - "dataElementUid", + "uid", quote(dataElement.getUid())); return replaceQualify(sqlTemplate, variables); From 4d6ce7815b74348a1dee498f6ca53063539a2618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:33:02 +0100 Subject: [PATCH 12/35] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManager.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 05eb60c8cec7..6e4a1f7af92b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -546,6 +546,14 @@ private List getColumnFromTrackedEntityAttributeWithLegend .collect(toList()); } + /** + * Returns a column for the given data element. If the value type of the data element is {@link + * ValueType#ORGANISATION_UNIT}, an extra column will be included. + * + * @param dataElement the {@link DataElement}. + * @param withLegendSet indicates + * @return + */ private List getColumnFromDataElement( DataElement dataElement, boolean withLegendSet) { List columns = new ArrayList<>(); From 7adfda5ceb245fcd38bfcfb318a82f5341eb4f0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:36:24 +0100 Subject: [PATCH 13/35] fix: Update code --- .../main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java index c4ee634d9821..8498baaaeac3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java @@ -219,7 +219,7 @@ public String coalesce(String expression, String defaultValue) { @Override public String jsonExtract(String column, String property) { - return "JSONExtractString(" + column + ", '" + property + "')"; + return String.format("JSONExtractString(%s, '%s')", column, property); } @Override From 11f688f998919904bf4f8d1a32ccc639d585c50b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:37:59 +0100 Subject: [PATCH 14/35] fix: Update code --- .../src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java index b428875659cc..f7c9be3bd772 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java @@ -222,7 +222,7 @@ public String coalesce(String expression, String defaultValue) { @Override public String jsonExtract(String column, String property) { - return "json_unquote(json_extract(" + column + ", '$." + property + "'))"; + return String.format("json_unquote(json_extract(%s, '$.%s'))", column, property); } @Override From db7a54b377b0f6d60932fabd7b291987c9785b1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:41:47 +0100 Subject: [PATCH 15/35] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 6e4a1f7af92b..1ff5b7589c66 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -488,14 +488,14 @@ private List getColumnFromTrackedEntityAttribute( List columns = new ArrayList<>(); DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); - String dataClause = + String dataExpression = attribute.isNumericType() ? numericClause : attribute.isDateType() ? dateClause : ""; - String select = getSelectExpressionForTea(attribute.getValueType(), "value"); - String sql = selectForInsert(attribute, select, dataClause); + String selectExpression = getSelectExpressionForTea(attribute.getValueType(), "value"); + String sql = selectForInsert(attribute, selectExpression, dataExpression); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); if (attribute.getValueType().isOrganisationUnit()) { - columns.addAll(getColumnsFromOrgUnitTrackedEntityAttribute(attribute, dataClause)); + columns.addAll(getColumnsFromOrgUnitTrackedEntityAttribute(attribute, dataExpression)); } columns.add( AnalyticsTableColumn.builder() @@ -559,15 +559,16 @@ private List getColumnFromDataElement( List columns = new ArrayList<>(); DataType dataType = getColumnType(dataElement.getValueType(), isSpatialSupport()); - String dataClause = getDataValueExpression(dataElement.getUid(), dataElement.getValueType()); + String dataExpression = + getDataValueExpression(dataElement.getUid(), dataElement.getValueType()); String columnName = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); String selectExpression = getSelectExpression(dataElement.getValueType(), columnName); - String sql = getSelectForInsert(dataElement, selectExpression, dataClause); + String sql = getSelectForInsert(dataElement, selectExpression, dataExpression); Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); if (dataElement.getValueType().isOrganisationUnit()) { - columns.addAll(getColumnFromOrgUnitDataElement(dataElement, dataClause)); + columns.addAll(getColumnFromOrgUnitDataElement(dataElement, dataExpression)); } columns.add( AnalyticsTableColumn.builder() @@ -579,7 +580,7 @@ private List getColumnFromDataElement( .build()); return withLegendSet - ? getColumnFromDataElementWithLegendSet(dataElement, selectExpression, dataClause) + ? getColumnFromDataElementWithLegendSet(dataElement, selectExpression, dataExpression) : columns; } @@ -728,9 +729,9 @@ private String getDataValueExpression(String uid, ValueType valueType) { if (valueType.isNumeric() || valueType.isDate()) { String regex = valueType.isNumeric() ? NUMERIC_LENIENT_REGEXP : DATE_REGEXP; - String jsonValue = sqlBuilder.jsonExtractNested("eventdatavalues", uid, "value"); + String jsonExpression = sqlBuilder.jsonExtractNested("eventdatavalues", uid, "value"); - return " and " + sqlBuilder.regexpMatch(jsonValue, "'" + regex + "'"); + return " and " + sqlBuilder.regexpMatch(jsonExpression, "'" + regex + "'"); } return EMPTY; From 243dc423591630001a797c2afeb8a12d7d303e4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:43:08 +0100 Subject: [PATCH 16/35] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 156 +++++++++--------- 1 file changed, 79 insertions(+), 77 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 1ff5b7589c66..62b298fa4bf8 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -480,6 +480,84 @@ protected AnalyticsTableColumn getPartitionColumn() { .build(); } + /** + * Returns a column for the given data element. If the value type of the data element is {@link + * ValueType#ORGANISATION_UNIT}, an extra column will be included. + * + * @param dataElement the {@link DataElement}. + * @param withLegendSet indicates + * @return + */ + private List getColumnFromDataElement( + DataElement dataElement, boolean withLegendSet) { + List columns = new ArrayList<>(); + + DataType dataType = getColumnType(dataElement.getValueType(), isSpatialSupport()); + String dataExpression = + getDataValueExpression(dataElement.getUid(), dataElement.getValueType()); + String columnName = + sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); + String selectExpression = getSelectExpression(dataElement.getValueType(), columnName); + String sql = getSelectForInsert(dataElement, selectExpression, dataExpression); + Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); + + if (dataElement.getValueType().isOrganisationUnit()) { + columns.addAll(getColumnFromOrgUnitDataElement(dataElement, dataExpression)); + } + + columns.add( + AnalyticsTableColumn.builder() + .name(dataElement.getUid()) + .columnType(AnalyticsColumnType.DYNAMIC) + .dataType(dataType) + .selectExpression(sql) + .skipIndex(skipIndex) + .build()); + + return withLegendSet + ? getColumnFromDataElementWithLegendSet(dataElement, selectExpression, dataExpression) + : columns; + } + + private List getColumnFromOrgUnitDataElement( + DataElement dataElement, String dataClause) { + final List columns = new ArrayList<>(); + + final String columnName = + sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); + + final String fromClause = + qualifyVariables("from ${organisationunit} ou where ou.uid = " + columnName); + + if (isSpatialSupport()) { + String fromType = "ou.geometry " + fromClause; + String geoSql = getSelectForInsert(dataElement, fromType, dataClause); + + columns.add( + AnalyticsTableColumn.builder() + .name((dataElement.getUid() + OU_GEOMETRY_COL_SUFFIX)) + .columnType(AnalyticsColumnType.DYNAMIC) + .dataType(GEOMETRY) + .selectExpression(geoSql) + .indexType(IndexType.GIST) + .build()); + } + + String fromTypeSql = "ou.name " + fromClause; + String ouNameSql = getSelectForInsert(dataElement, fromTypeSql, dataClause); + + columns.add( + AnalyticsTableColumn.builder() + .name((dataElement.getUid() + OU_NAME_COL_SUFFIX)) + .columnType(AnalyticsColumnType.DYNAMIC) + .dataType(TEXT) + .selectExpression(ouNameSql) + .skipIndex(SKIP) + .build()); + + return columns; + } + private List getColumnFromTrackedEntityAttribute( TrackedEntityAttribute attribute, String numericClause, @@ -497,6 +575,7 @@ private List getColumnFromTrackedEntityAttribute( if (attribute.getValueType().isOrganisationUnit()) { columns.addAll(getColumnsFromOrgUnitTrackedEntityAttribute(attribute, dataExpression)); } + columns.add( AnalyticsTableColumn.builder() .name(attribute.getUid()) @@ -546,44 +625,6 @@ private List getColumnFromTrackedEntityAttributeWithLegend .collect(toList()); } - /** - * Returns a column for the given data element. If the value type of the data element is {@link - * ValueType#ORGANISATION_UNIT}, an extra column will be included. - * - * @param dataElement the {@link DataElement}. - * @param withLegendSet indicates - * @return - */ - private List getColumnFromDataElement( - DataElement dataElement, boolean withLegendSet) { - List columns = new ArrayList<>(); - - DataType dataType = getColumnType(dataElement.getValueType(), isSpatialSupport()); - String dataExpression = - getDataValueExpression(dataElement.getUid(), dataElement.getValueType()); - String columnName = - sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); - String selectExpression = getSelectExpression(dataElement.getValueType(), columnName); - String sql = getSelectForInsert(dataElement, selectExpression, dataExpression); - Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); - - if (dataElement.getValueType().isOrganisationUnit()) { - columns.addAll(getColumnFromOrgUnitDataElement(dataElement, dataExpression)); - } - columns.add( - AnalyticsTableColumn.builder() - .name(dataElement.getUid()) - .columnType(AnalyticsColumnType.DYNAMIC) - .dataType(dataType) - .selectExpression(sql) - .skipIndex(skipIndex) - .build()); - - return withLegendSet - ? getColumnFromDataElementWithLegendSet(dataElement, selectExpression, dataExpression) - : columns; - } - private List getColumnsFromOrgUnitTrackedEntityAttribute( TrackedEntityAttribute attribute, String dataClause) { final List columns = new ArrayList<>(); @@ -619,45 +660,6 @@ private List getColumnsFromOrgUnitTrackedEntityAttribute( return columns; } - private List getColumnFromOrgUnitDataElement( - DataElement dataElement, String dataClause) { - final List columns = new ArrayList<>(); - - final String columnName = - sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); - - final String fromClause = - qualifyVariables("from ${organisationunit} ou where ou.uid = " + columnName); - - if (isSpatialSupport()) { - String fromType = "ou.geometry " + fromClause; - String geoSql = getSelectForInsert(dataElement, fromType, dataClause); - - columns.add( - AnalyticsTableColumn.builder() - .name((dataElement.getUid() + OU_GEOMETRY_COL_SUFFIX)) - .columnType(AnalyticsColumnType.DYNAMIC) - .dataType(GEOMETRY) - .selectExpression(geoSql) - .indexType(IndexType.GIST) - .build()); - } - - String fromTypeSql = "ou.name " + fromClause; - String ouNameSql = getSelectForInsert(dataElement, fromTypeSql, dataClause); - - columns.add( - AnalyticsTableColumn.builder() - .name((dataElement.getUid() + OU_NAME_COL_SUFFIX)) - .columnType(AnalyticsColumnType.DYNAMIC) - .dataType(TEXT) - .selectExpression(ouNameSql) - .skipIndex(SKIP) - .build()); - - return columns; - } - /** * Creates a select statement for data element insertion. * From 5f6b566b89b65d896649ea2c86475ed8c3044c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:44:01 +0100 Subject: [PATCH 17/35] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 62b298fa4bf8..71e7dcf95c74 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -493,8 +493,7 @@ private List getColumnFromDataElement( List columns = new ArrayList<>(); DataType dataType = getColumnType(dataElement.getValueType(), isSpatialSupport()); - String dataExpression = - getDataValueExpression(dataElement.getUid(), dataElement.getValueType()); + String dataExpression = getDataExpression(dataElement.getUid(), dataElement.getValueType()); String columnName = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); String selectExpression = getSelectExpression(dataElement.getValueType(), columnName); @@ -727,7 +726,7 @@ private List getColumnFromDataElementWithLegendSet( * @param valueType the {@link ValueType}. * @return an expression for extracting a data value. */ - private String getDataValueExpression(String uid, ValueType valueType) { + private String getDataExpression(String uid, ValueType valueType) { if (valueType.isNumeric() || valueType.isDate()) { String regex = valueType.isNumeric() ? NUMERIC_LENIENT_REGEXP : DATE_REGEXP; From 3e98d63580c98bb9baf32a9c415b0069626a3a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:44:44 +0100 Subject: [PATCH 18/35] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManager.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 71e7dcf95c74..4e106be2bce6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -520,12 +520,11 @@ private List getColumnFromDataElement( private List getColumnFromOrgUnitDataElement( DataElement dataElement, String dataClause) { - final List columns = new ArrayList<>(); + List columns = new ArrayList<>(); - final String columnName = + String columnName = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); - - final String fromClause = + String fromClause = qualifyVariables("from ${organisationunit} ou where ou.uid = " + columnName); if (isSpatialSupport()) { From 089182d863293bde7df6698bdf09f5210d2c3b9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:44:59 +0100 Subject: [PATCH 19/35] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 4e106be2bce6..d701947a8009 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -625,9 +625,9 @@ private List getColumnFromTrackedEntityAttributeWithLegend private List getColumnsFromOrgUnitTrackedEntityAttribute( TrackedEntityAttribute attribute, String dataClause) { - final List columns = new ArrayList<>(); + List columns = new ArrayList<>(); - final String fromClause = + String fromClause = qualifyVariables("from ${organisationunit} ou where ou.uid = (select value"); if (isSpatialSupport()) { From eb0bd0d699af32b495f75752bfa70afba55c50e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:46:05 +0100 Subject: [PATCH 20/35] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index d701947a8009..5bc41fd861a7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -719,7 +719,8 @@ private List getColumnFromDataElementWithLegendSet( } /** - * Returns an expression for extracting a data value. + * Returns an expression for extracting a data value for numeric and date value types, otherwise + * returns the empty string. * * @param uid the identifier. * @param valueType the {@link ValueType}. From 6f72165fe9202cce98b5fb8a97603bbfa9408244 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:47:07 +0100 Subject: [PATCH 21/35] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 5bc41fd861a7..2320e7f9e72a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -493,10 +493,10 @@ private List getColumnFromDataElement( List columns = new ArrayList<>(); DataType dataType = getColumnType(dataElement.getValueType(), isSpatialSupport()); - String dataExpression = getDataExpression(dataElement.getUid(), dataElement.getValueType()); String columnName = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); String selectExpression = getSelectExpression(dataElement.getValueType(), columnName); + String dataExpression = getDataExpression(dataElement.getUid(), dataElement.getValueType()); String sql = getSelectForInsert(dataElement, selectExpression, dataExpression); Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); @@ -564,9 +564,9 @@ private List getColumnFromTrackedEntityAttribute( List columns = new ArrayList<>(); DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); + String selectExpression = getSelectExpressionForTea(attribute.getValueType(), "value"); String dataExpression = attribute.isNumericType() ? numericClause : attribute.isDateType() ? dateClause : ""; - String selectExpression = getSelectExpressionForTea(attribute.getValueType(), "value"); String sql = selectForInsert(attribute, selectExpression, dataExpression); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); From 719155bddb942d719a28b2b4e9898cbca8310e4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:52:27 +0100 Subject: [PATCH 22/35] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 9ed57b532275..81a07fe1de2b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -118,14 +118,13 @@ protected String getSelectExpressionForTea(ValueType valueType, String columnNam /** * Returns the select expression, potentially with a cast statement, based on the given value - * type. This internal method handles both Data Value and Tracked Entity Attribute (TEA) select - * expressions. + * type. Handles data element and tracked entity attribute select expressions. * - * @param valueType the value type to represent as database column type - * @param columnName the name of the column to be selected - * @param isTeaContext whether the selection is in the context of a Tracked Entity Attribute. When - * true, organization unit selections will include an additional subquery wrapper - * @return A SQL select expression appropriate for the given value type and context + * @param valueType the value type to represent as database column type. + * @param columnName the name of the column to be selected. + * @param isTeaContext whether the selection is in the context of a tracked entity attribute. When + * true, organization unit selections will include an additional subquery wrapper. + * @return A SQL select expression appropriate for the given value type and context. */ private String getSelectExpressionInternal( ValueType valueType, String columnName, boolean isTeaContext) { From d811508a786451da6e062f50e2cb2d4db4cc346f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:53:19 +0100 Subject: [PATCH 23/35] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 2320e7f9e72a..4d33d699a867 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -719,8 +719,8 @@ private List getColumnFromDataElementWithLegendSet( } /** - * Returns an expression for extracting a data value for numeric and date value types, otherwise - * returns the empty string. + * Returns an expression for extracting a data value for numeric and date value types and checking + * whether to value is valid according to the value type. Otherwise returns the empty string. * * @param uid the identifier. * @param valueType the {@link ValueType}. From 5df47358663464d8ed8de6f8a4bef5263f2b79c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 16:57:06 +0100 Subject: [PATCH 24/35] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 4d33d699a867..08dca0d463de 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -493,9 +493,9 @@ private List getColumnFromDataElement( List columns = new ArrayList<>(); DataType dataType = getColumnType(dataElement.getValueType(), isSpatialSupport()); - String columnName = + String columnExpression = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); - String selectExpression = getSelectExpression(dataElement.getValueType(), columnName); + String selectExpression = getSelectExpression(dataElement.getValueType(), columnExpression); String dataExpression = getDataExpression(dataElement.getUid(), dataElement.getValueType()); String sql = getSelectForInsert(dataElement, selectExpression, dataExpression); Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); @@ -522,10 +522,10 @@ private List getColumnFromOrgUnitDataElement( DataElement dataElement, String dataClause) { List columns = new ArrayList<>(); - String columnName = + String columnExpression = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); String fromClause = - qualifyVariables("from ${organisationunit} ou where ou.uid = " + columnName); + qualifyVariables("from ${organisationunit} ou where ou.uid = " + columnExpression); if (isSpatialSupport()) { String fromType = "ou.geometry " + fromClause; @@ -662,11 +662,12 @@ private List getColumnsFromOrgUnitTrackedEntityAttribute( * Creates a select statement for data element insertion. * * @param dataElement the data element to create the select statement for. - * @param fromType the SQL snippet for the "from" part of the query. + * @param selectExpression the select expression. * @param dataClause the data type related clause. * @return A SQL select expression for the data element. */ - private String getSelectForInsert(DataElement dataElement, String fromType, String dataClause) { + private String getSelectForInsert( + DataElement dataElement, String selectExpression, String dataClause) { String sqlTemplate = dataElement.getValueType().isOrganisationUnit() ? "(select ${fromType} ${dataClause})${closingParentheses} as ${uid}" @@ -675,11 +676,11 @@ private String getSelectForInsert(DataElement dataElement, String fromType, Stri Map variables = Map.of( "fromType", - fromType, + selectExpression, "dataClause", dataClause, "closingParentheses", - getClosingParentheses(fromType), + getClosingParentheses(selectExpression), "uid", quote(dataElement.getUid())); From 49569a1243577b464ed3af487ef292425f38a892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 17:00:54 +0100 Subject: [PATCH 25/35] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 28 +++++++++++++++---- .../table/JdbcEventAnalyticsTableManager.java | 7 +++-- ...dbcTrackedEntityAnalyticsTableManager.java | 2 +- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 81a07fe1de2b..448a02ef5124 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -108,19 +108,35 @@ protected Skip skipIndex(ValueType valueType, boolean hasOptionSet) { return skipIndex ? Skip.SKIP : Skip.INCLUDE; } - protected String getSelectExpression(ValueType valueType, String columnName) { + /** + * Returns a select expression for a data element value, handling casting to the appropriate data + * type based on the given value type. + * + * @param valueType the {@link ValueType}. + * @param columnName the column name. + * @return a select expression. + */ + protected String getSelectExpressionForDataElement(ValueType valueType, String columnName) { return getSelectExpressionInternal(valueType, columnName, false); } - protected String getSelectExpressionForTea(ValueType valueType, String columnName) { + /** + * Returns a select expression for a tracked entity attribute, handling casting to the appropriate + * data type based on the given value type. + * + * @param valueType the {@link ValueType}. + * @param columnName the column name. + * @return a select expression. + */ + protected String getSelectExpressionForAttribute(ValueType valueType, String columnName) { return getSelectExpressionInternal(valueType, columnName, true); } /** - * Returns the select expression, potentially with a cast statement, based on the given value - * type. Handles data element and tracked entity attribute select expressions. + * Returns a select expression, potentially with a cast statement, based on the given value type. + * Handles data element and tracked entity attribute select expressions. * - * @param valueType the value type to represent as database column type. + * @param valueType the {@link ValueType} to represent as database column type. * @param columnName the name of the column to be selected. * @param isTeaContext whether the selection is in the context of a tracked entity attribute. When * true, organization unit selections will include an additional subquery wrapper. @@ -201,7 +217,7 @@ protected List getTrackedEntityAttributeColumns(Program pr attribute.isNumericType() ? getNumericClause() : attribute.isDateType() ? getDateClause() : ""; - String select = getSelectExpressionForTea(attribute.getValueType(), "value"); + String select = getSelectExpressionForAttribute(attribute.getValueType(), "value"); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); String sql = diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 08dca0d463de..2664a486f04f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -495,7 +495,8 @@ private List getColumnFromDataElement( DataType dataType = getColumnType(dataElement.getValueType(), isSpatialSupport()); String columnExpression = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); - String selectExpression = getSelectExpression(dataElement.getValueType(), columnExpression); + String selectExpression = + getSelectExpressionForDataElement(dataElement.getValueType(), columnExpression); String dataExpression = getDataExpression(dataElement.getUid(), dataElement.getValueType()); String sql = getSelectForInsert(dataElement, selectExpression, dataExpression); Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); @@ -564,7 +565,7 @@ private List getColumnFromTrackedEntityAttribute( List columns = new ArrayList<>(); DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); - String selectExpression = getSelectExpressionForTea(attribute.getValueType(), "value"); + String selectExpression = getSelectExpressionForAttribute(attribute.getValueType(), "value"); String dataExpression = attribute.isNumericType() ? numericClause : attribute.isDateType() ? dateClause : ""; String sql = selectForInsert(attribute, selectExpression, dataExpression); @@ -590,7 +591,7 @@ private List getColumnFromTrackedEntityAttribute( private List getColumnFromTrackedEntityAttributeWithLegendSet( TrackedEntityAttribute attribute, String numericClause) { - String selectClause = getSelectExpression(attribute.getValueType(), "value"); + String selectClause = getSelectExpressionForDataElement(attribute.getValueType(), "value"); String query = """ \s(select l.uid from ${maplegend} l \ diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index a4e1fc4b8997..6a6369262c01 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -255,7 +255,7 @@ private Stream getAllTrackedEntityAttributes( /** * Returns the select clause, potentially with a cast statement, based on the given value type. * (this method is an adapted version of {@link - * JdbcEventAnalyticsTableManager#getSelectExpression(ValueType, String)}) + * JdbcEventAnalyticsTableManager#getSelectExpressionForDataElement(ValueType, String)}) * * @param valueType the value type to represent as database column type. */ From ec6c90672d28a0762ca2ddf88784d1c843f53102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 17:03:26 +0100 Subject: [PATCH 26/35] fix: Update code --- .../table/AbstractEventJdbcTableManager.java | 2 +- .../table/JdbcEventAnalyticsTableManager.java | 25 +++++++++---------- ...dbcTrackedEntityAnalyticsTableManager.java | 2 +- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 448a02ef5124..65ef1a37ab32 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -116,7 +116,7 @@ protected Skip skipIndex(ValueType valueType, boolean hasOptionSet) { * @param columnName the column name. * @return a select expression. */ - protected String getSelectExpressionForDataElement(ValueType valueType, String columnName) { + protected String getSelectExpression(ValueType valueType, String columnName) { return getSelectExpressionInternal(valueType, columnName, false); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 2664a486f04f..421e8448d6bc 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -495,14 +495,13 @@ private List getColumnFromDataElement( DataType dataType = getColumnType(dataElement.getValueType(), isSpatialSupport()); String columnExpression = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); - String selectExpression = - getSelectExpressionForDataElement(dataElement.getValueType(), columnExpression); - String dataExpression = getDataExpression(dataElement.getUid(), dataElement.getValueType()); - String sql = getSelectForInsert(dataElement, selectExpression, dataExpression); + String selectExpression = getSelectExpression(dataElement.getValueType(), columnExpression); + String dataFilterClause = getDataFilterClause(dataElement.getUid(), dataElement.getValueType()); + String sql = getSelectForInsert(dataElement, selectExpression, dataFilterClause); Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); if (dataElement.getValueType().isOrganisationUnit()) { - columns.addAll(getColumnFromOrgUnitDataElement(dataElement, dataExpression)); + columns.addAll(getColumnFromOrgUnitDataElement(dataElement, dataFilterClause)); } columns.add( @@ -515,7 +514,7 @@ private List getColumnFromDataElement( .build()); return withLegendSet - ? getColumnFromDataElementWithLegendSet(dataElement, selectExpression, dataExpression) + ? getColumnFromDataElementWithLegendSet(dataElement, selectExpression, dataFilterClause) : columns; } @@ -591,7 +590,7 @@ private List getColumnFromTrackedEntityAttribute( private List getColumnFromTrackedEntityAttributeWithLegendSet( TrackedEntityAttribute attribute, String numericClause) { - String selectClause = getSelectExpressionForDataElement(attribute.getValueType(), "value"); + String selectClause = getSelectExpression(attribute.getValueType(), "value"); String query = """ \s(select l.uid from ${maplegend} l \ @@ -664,11 +663,11 @@ private List getColumnsFromOrgUnitTrackedEntityAttribute( * * @param dataElement the data element to create the select statement for. * @param selectExpression the select expression. - * @param dataClause the data type related clause. + * @param dataFilterClause the data filter clause. * @return A SQL select expression for the data element. */ private String getSelectForInsert( - DataElement dataElement, String selectExpression, String dataClause) { + DataElement dataElement, String selectExpression, String dataFilterClause) { String sqlTemplate = dataElement.getValueType().isOrganisationUnit() ? "(select ${fromType} ${dataClause})${closingParentheses} as ${uid}" @@ -679,7 +678,7 @@ private String getSelectForInsert( "fromType", selectExpression, "dataClause", - dataClause, + dataFilterClause, "closingParentheses", getClosingParentheses(selectExpression), "uid", @@ -721,14 +720,14 @@ private List getColumnFromDataElementWithLegendSet( } /** - * Returns an expression for extracting a data value for numeric and date value types and checking - * whether to value is valid according to the value type. Otherwise returns the empty string. + * For numeric and date value types, returns a data filter clause for checking whether the value + * is valid according to the value type. For other value types, returns the empty string. * * @param uid the identifier. * @param valueType the {@link ValueType}. * @return an expression for extracting a data value. */ - private String getDataExpression(String uid, ValueType valueType) { + private String getDataFilterClause(String uid, ValueType valueType) { if (valueType.isNumeric() || valueType.isDate()) { String regex = valueType.isNumeric() ? NUMERIC_LENIENT_REGEXP : DATE_REGEXP; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index 6a6369262c01..a4e1fc4b8997 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -255,7 +255,7 @@ private Stream getAllTrackedEntityAttributes( /** * Returns the select clause, potentially with a cast statement, based on the given value type. * (this method is an adapted version of {@link - * JdbcEventAnalyticsTableManager#getSelectExpressionForDataElement(ValueType, String)}) + * JdbcEventAnalyticsTableManager#getSelectExpression(ValueType, String)}) * * @param valueType the value type to represent as database column type. */ From 7db77c5ef64ef5c8f91d1435044c4ec9e783a654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 17:05:43 +0100 Subject: [PATCH 27/35] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 421e8448d6bc..a663baef04d0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -659,7 +659,7 @@ private List getColumnsFromOrgUnitTrackedEntityAttribute( } /** - * Creates a select statement for data element insertion. + * Creates a select statement for the given select expression. * * @param dataElement the data element to create the select statement for. * @param selectExpression the select expression. @@ -687,8 +687,16 @@ private String getSelectForInsert( return replaceQualify(sqlTemplate, variables); } + /** + * Returns a list of columns. + * + * @param dataElement the {@link DataElement}. + * @param selectExpression the select expression. + * @param dataFilterClause the data filter clause. + * @return a list of {@link AnayticsTableColumn}. + */ private List getColumnFromDataElementWithLegendSet( - DataElement dataElement, String select, String dataClause) { + DataElement dataElement, String selectExpression, String dataFilterClause) { String query = """ (select l.uid from ${maplegend} l @@ -705,9 +713,9 @@ private List getColumnFromDataElementWithLegendSet( replaceQualify( query, Map.of( - "select", select, + "select", selectExpression, "legendSetId", String.valueOf(ls.getId()), - "dataClause", dataClause, + "dataClause", dataFilterClause, "column", column)); return AnalyticsTableColumn.builder() From 4d2ad591189c5abf40688bbde2032f59911d14ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 17:06:35 +0100 Subject: [PATCH 28/35] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index a663baef04d0..0c3e85fe9e0a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -519,7 +519,7 @@ private List getColumnFromDataElement( } private List getColumnFromOrgUnitDataElement( - DataElement dataElement, String dataClause) { + DataElement dataElement, String dataFilterClause) { List columns = new ArrayList<>(); String columnExpression = @@ -529,7 +529,7 @@ private List getColumnFromOrgUnitDataElement( if (isSpatialSupport()) { String fromType = "ou.geometry " + fromClause; - String geoSql = getSelectForInsert(dataElement, fromType, dataClause); + String geoSql = getSelectForInsert(dataElement, fromType, dataFilterClause); columns.add( AnalyticsTableColumn.builder() @@ -542,7 +542,7 @@ private List getColumnFromOrgUnitDataElement( } String fromTypeSql = "ou.name " + fromClause; - String ouNameSql = getSelectForInsert(dataElement, fromTypeSql, dataClause); + String ouNameSql = getSelectForInsert(dataElement, fromTypeSql, dataFilterClause); columns.add( AnalyticsTableColumn.builder() @@ -624,7 +624,7 @@ private List getColumnFromTrackedEntityAttributeWithLegend } private List getColumnsFromOrgUnitTrackedEntityAttribute( - TrackedEntityAttribute attribute, String dataClause) { + TrackedEntityAttribute attribute, String dataFilterClause) { List columns = new ArrayList<>(); String fromClause = @@ -632,7 +632,7 @@ private List getColumnsFromOrgUnitTrackedEntityAttribute( if (isSpatialSupport()) { String fromType = "ou.geometry " + fromClause; - String geoSql = selectForInsert(attribute, fromType, dataClause); + String geoSql = selectForInsert(attribute, fromType, dataFilterClause); columns.add( AnalyticsTableColumn.builder() .name((attribute.getUid() + OU_GEOMETRY_COL_SUFFIX)) @@ -644,7 +644,7 @@ private List getColumnsFromOrgUnitTrackedEntityAttribute( } String fromTypeSql = "ou.name " + fromClause; - String ouNameSql = selectForInsert(attribute, fromTypeSql, dataClause); + String ouNameSql = selectForInsert(attribute, fromTypeSql, dataFilterClause); columns.add( AnalyticsTableColumn.builder() From 9af754079ad863f0ab1c1d8614dde5b5f94c23b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 18:30:11 +0100 Subject: [PATCH 29/35] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 0c3e85fe9e0a..1017cf56be83 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -670,12 +670,12 @@ private String getSelectForInsert( DataElement dataElement, String selectExpression, String dataFilterClause) { String sqlTemplate = dataElement.getValueType().isOrganisationUnit() - ? "(select ${fromType} ${dataClause})${closingParentheses} as ${uid}" - : "(select ${fromType} from ${event} where eventid=ev.eventid ${dataClause})${closingParentheses} as ${uid}"; + ? "(select ${selectExpression} ${dataClause})${closingParentheses} as ${uid}" + : "(select ${selectExpression} from ${event} where eventid=ev.eventid ${dataClause})${closingParentheses} as ${uid}"; Map variables = Map.of( - "fromType", + "selectExpression", selectExpression, "dataClause", dataFilterClause, From a5ac41ccc2ce439667e6a8c214747548e3f9d2c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 19:42:41 +0100 Subject: [PATCH 30/35] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 1017cf56be83..cfdf6bdd869b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -671,7 +671,7 @@ private String getSelectForInsert( String sqlTemplate = dataElement.getValueType().isOrganisationUnit() ? "(select ${selectExpression} ${dataClause})${closingParentheses} as ${uid}" - : "(select ${selectExpression} from ${event} where eventid=ev.eventid ${dataClause})${closingParentheses} as ${uid}"; + : "${selectExpression} as ${uid}"; Map variables = Map.of( From 952b0e8e13b4a134fc5e38c539440deeac495ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 20:23:35 +0100 Subject: [PATCH 31/35] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 40 +++++++++---------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index cfdf6bdd869b..6ad038ebb7d6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -441,19 +441,13 @@ private List getColumns(Program program) { columns.addAll( program.getNonConfidentialTrackedEntityAttributes().stream() - .map( - tea -> - getColumnFromTrackedEntityAttribute( - tea, getNumericClause(), getDateClause(), false)) + .map(tea -> getColumnFromTrackedEntityAttribute(tea, false)) .flatMap(Collection::stream) .toList()); columns.addAll( program.getNonConfidentialTrackedEntityAttributesWithLegendSet().stream() - .map( - tea -> - getColumnFromTrackedEntityAttribute( - tea, getNumericClause(), getDateClause(), true)) + .map(tea -> getColumnFromTrackedEntityAttribute(tea, true)) .flatMap(Collection::stream) .toList()); @@ -496,7 +490,7 @@ private List getColumnFromDataElement( String columnExpression = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); String selectExpression = getSelectExpression(dataElement.getValueType(), columnExpression); - String dataFilterClause = getDataFilterClause(dataElement.getUid(), dataElement.getValueType()); + String dataFilterClause = getDataFilterClause(dataElement); String sql = getSelectForInsert(dataElement, selectExpression, dataFilterClause); Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); @@ -556,17 +550,19 @@ private List getColumnFromOrgUnitDataElement( return columns; } + private String getDataFilterClause(TrackedEntityAttribute attribute) { + return attribute.isNumericType() + ? getNumericClause() + : attribute.isDateType() ? getDateClause() : EMPTY; + } + private List getColumnFromTrackedEntityAttribute( - TrackedEntityAttribute attribute, - String numericClause, - String dateClause, - boolean withLegendSet) { + TrackedEntityAttribute attribute, boolean withLegendSet) { List columns = new ArrayList<>(); DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); String selectExpression = getSelectExpressionForAttribute(attribute.getValueType(), "value"); - String dataExpression = - attribute.isNumericType() ? numericClause : attribute.isDateType() ? dateClause : ""; + String dataExpression = getDataFilterClause(attribute); String sql = selectForInsert(attribute, selectExpression, dataExpression); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); @@ -583,14 +579,13 @@ private List getColumnFromTrackedEntityAttribute( .skipIndex(skipIndex) .build()); - return withLegendSet - ? getColumnFromTrackedEntityAttributeWithLegendSet(attribute, numericClause) - : columns; + return withLegendSet ? getColumnFromAttributeWithLegendSet(attribute) : columns; } - private List getColumnFromTrackedEntityAttributeWithLegendSet( - TrackedEntityAttribute attribute, String numericClause) { + private List getColumnFromAttributeWithLegendSet( + TrackedEntityAttribute attribute) { String selectClause = getSelectExpression(attribute.getValueType(), "value"); + String numericClause = getNumericClause(); String query = """ \s(select l.uid from ${maplegend} l \ @@ -735,7 +730,10 @@ private List getColumnFromDataElementWithLegendSet( * @param valueType the {@link ValueType}. * @return an expression for extracting a data value. */ - private String getDataFilterClause(String uid, ValueType valueType) { + private String getDataFilterClause(DataElement dataElement) { + String uid = dataElement.getUid(); + ValueType valueType = dataElement.getValueType(); + if (valueType.isNumeric() || valueType.isDate()) { String regex = valueType.isNumeric() ? NUMERIC_LENIENT_REGEXP : DATE_REGEXP; From c7e5bf06fd5ea46c4200c82d49a16d02e2bf2519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 20:25:52 +0100 Subject: [PATCH 32/35] fix: Update code --- .../dhis/analytics/table/JdbcEventAnalyticsTableManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 6ad038ebb7d6..04cd8e4a589a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -666,7 +666,7 @@ private String getSelectForInsert( String sqlTemplate = dataElement.getValueType().isOrganisationUnit() ? "(select ${selectExpression} ${dataClause})${closingParentheses} as ${uid}" - : "${selectExpression} as ${uid}"; + : "(select ${selectExpression} from ${event} where eventid=ev.eventid ${dataClause})${closingParentheses} as ${uid}"; Map variables = Map.of( From 4e2b03f36b0a6b5388774e6c0a805de5c873534d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 20:29:39 +0100 Subject: [PATCH 33/35] fix: Update code --- .../table/JdbcEventAnalyticsTableManager.java | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 04cd8e4a589a..c5c4a28490b5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -225,7 +225,7 @@ private List getLatestAnalyticsTables(AnalyticsTableUpdateParams Assert.isTrue( lastFullTableUpdate.getTime() > 0L, - "A full analytics table update process must be run prior to a latest partition update process"); + "A full analytics table update process must be run prior to a latest partition update"); Date startDate = lastFullTableUpdate; Date endDate = params.getStartTime(); @@ -550,12 +550,6 @@ private List getColumnFromOrgUnitDataElement( return columns; } - private String getDataFilterClause(TrackedEntityAttribute attribute) { - return attribute.isNumericType() - ? getNumericClause() - : attribute.isDateType() ? getDateClause() : EMPTY; - } - private List getColumnFromTrackedEntityAttribute( TrackedEntityAttribute attribute, boolean withLegendSet) { List columns = new ArrayList<>(); @@ -668,7 +662,8 @@ private String getSelectForInsert( ? "(select ${selectExpression} ${dataClause})${closingParentheses} as ${uid}" : "(select ${selectExpression} from ${event} where eventid=ev.eventid ${dataClause})${closingParentheses} as ${uid}"; - Map variables = + return replaceQualify( + sqlTemplate, Map.of( "selectExpression", selectExpression, @@ -677,9 +672,7 @@ private String getSelectForInsert( "closingParentheses", getClosingParentheses(selectExpression), "uid", - quote(dataElement.getUid())); - - return replaceQualify(sqlTemplate, variables); + quote(dataElement.getUid()))); } /** @@ -726,9 +719,8 @@ private List getColumnFromDataElementWithLegendSet( * For numeric and date value types, returns a data filter clause for checking whether the value * is valid according to the value type. For other value types, returns the empty string. * - * @param uid the identifier. - * @param valueType the {@link ValueType}. - * @return an expression for extracting a data value. + * @param dataElement the {@link DataElement}. + * @return an filter expression. */ private String getDataFilterClause(DataElement dataElement) { String uid = dataElement.getUid(); @@ -745,6 +737,19 @@ private String getDataFilterClause(DataElement dataElement) { return EMPTY; } + /** + * For numeric and date value types, returns a data filter clause for checking whether the value + * is valid according to the value type. For other value types, returns the empty string. + * + * @param attribute the {@link TrackedEntityAttribute}. + * @return an filter expression. + */ + private String getDataFilterClause(TrackedEntityAttribute attribute) { + return attribute.isNumericType() + ? getNumericClause() + : attribute.isDateType() ? getDateClause() : EMPTY; + } + /** * Returns a list of years for which data exist. * From f7aa4c5063d6b72e7bae1aaa2cf7066bc888e446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 20:33:27 +0100 Subject: [PATCH 34/35] fix: Update code --- .../JdbcEnrollmentAnalyticsTableManager.java | 6 +++ .../table/JdbcEventAnalyticsTableManager.java | 37 +++++++++++++------ 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index ac747204d146..a6f0841dc2c0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -165,6 +165,12 @@ left join analytics_rs_dateperiodstructure dps on cast(en.enrollmentdate as date populateTableInternal(partition, fromClause); } + /** + * Returns a list of columns for the given program. + * + * @param program the {@link Program}. + * @return a list of {@link AnalyticsTableColumn}. + */ private List getColumns(Program program) { List columns = new ArrayList<>(); columns.addAll(fixedColumns); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index c5c4a28490b5..b9d51ad48146 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -429,25 +429,25 @@ private List getColumns(Program program) { columns.addAll( program.getAnalyticsDataElements().stream() - .map(de -> getColumnFromDataElement(de, false)) + .map(de -> getColumnForDataElement(de, false)) .flatMap(Collection::stream) .toList()); columns.addAll( program.getAnalyticsDataElementsWithLegendSet().stream() - .map(de -> getColumnFromDataElement(de, true)) + .map(de -> getColumnForDataElement(de, true)) .flatMap(Collection::stream) .toList()); columns.addAll( program.getNonConfidentialTrackedEntityAttributes().stream() - .map(tea -> getColumnFromTrackedEntityAttribute(tea, false)) + .map(tea -> getColumnForTrackedEntityAttribute(tea, false)) .flatMap(Collection::stream) .toList()); columns.addAll( program.getNonConfidentialTrackedEntityAttributesWithLegendSet().stream() - .map(tea -> getColumnFromTrackedEntityAttribute(tea, true)) + .map(tea -> getColumnForTrackedEntityAttribute(tea, true)) .flatMap(Collection::stream) .toList()); @@ -482,7 +482,7 @@ protected AnalyticsTableColumn getPartitionColumn() { * @param withLegendSet indicates * @return */ - private List getColumnFromDataElement( + private List getColumnForDataElement( DataElement dataElement, boolean withLegendSet) { List columns = new ArrayList<>(); @@ -495,7 +495,7 @@ private List getColumnFromDataElement( Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); if (dataElement.getValueType().isOrganisationUnit()) { - columns.addAll(getColumnFromOrgUnitDataElement(dataElement, dataFilterClause)); + columns.addAll(getColumnForOrgUnitDataElement(dataElement, dataFilterClause)); } columns.add( @@ -512,7 +512,7 @@ private List getColumnFromDataElement( : columns; } - private List getColumnFromOrgUnitDataElement( + private List getColumnForOrgUnitDataElement( DataElement dataElement, String dataFilterClause) { List columns = new ArrayList<>(); @@ -550,7 +550,7 @@ private List getColumnFromOrgUnitDataElement( return columns; } - private List getColumnFromTrackedEntityAttribute( + private List getColumnForTrackedEntityAttribute( TrackedEntityAttribute attribute, boolean withLegendSet) { List columns = new ArrayList<>(); @@ -561,7 +561,7 @@ private List getColumnFromTrackedEntityAttribute( Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); if (attribute.getValueType().isOrganisationUnit()) { - columns.addAll(getColumnsFromOrgUnitTrackedEntityAttribute(attribute, dataExpression)); + columns.addAll(getColumnsForOrgUnitTrackedEntityAttribute(attribute, dataExpression)); } columns.add( @@ -573,10 +573,16 @@ private List getColumnFromTrackedEntityAttribute( .skipIndex(skipIndex) .build()); - return withLegendSet ? getColumnFromAttributeWithLegendSet(attribute) : columns; + return withLegendSet ? getColumnForAttributeWithLegendSet(attribute) : columns; } - private List getColumnFromAttributeWithLegendSet( + /** + * Returns a list of columns based on the given attribute with legend set. + * + * @param attribute the {@link TrackedEntityAttribute}. + * @return a list of {@link AnalyticsTableColumn}. + */ + private List getColumnForAttributeWithLegendSet( TrackedEntityAttribute attribute) { String selectClause = getSelectExpression(attribute.getValueType(), "value"); String numericClause = getNumericClause(); @@ -612,7 +618,14 @@ private List getColumnFromAttributeWithLegendSet( .collect(toList()); } - private List getColumnsFromOrgUnitTrackedEntityAttribute( + /** + * Returns a list of columns based on the given attribute. + * + * @param attribute the {@link TrackedEntityAttribute}. + * @param dataFilterClause the data filter clause. + * @return a list of {@link AnalyticsTableColumn}. + */ + private List getColumnsForOrgUnitTrackedEntityAttribute( TrackedEntityAttribute attribute, String dataFilterClause) { List columns = new ArrayList<>(); From f767809c349f49250d10073e2578014e5d45c4bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 24 Nov 2024 20:43:44 +0100 Subject: [PATCH 35/35] fix: Update code --- .../analytics/table/JdbcEventAnalyticsTableManager.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index b9d51ad48146..3c6f5c905ec3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -758,9 +758,10 @@ private String getDataFilterClause(DataElement dataElement) { * @return an filter expression. */ private String getDataFilterClause(TrackedEntityAttribute attribute) { - return attribute.isNumericType() - ? getNumericClause() - : attribute.isDateType() ? getDateClause() : EMPTY; + if (attribute.isNumericType()) { + return getNumericClause(); + } + return attribute.isDateType() ? getDateClause() : EMPTY; } /**