diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index c164dc6d6989..543c686379ac 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -38,6 +38,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.partition.PartitionManager; import org.hisp.dhis.analytics.table.model.AnalyticsDimensionType; @@ -46,6 +47,7 @@ import org.hisp.dhis.analytics.table.model.Skip; import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; import org.hisp.dhis.category.CategoryService; +import org.hisp.dhis.common.IdentifiableObject; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.commons.util.TextUtils; @@ -55,6 +57,7 @@ import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.organisationunit.OrganisationUnitService; import org.hisp.dhis.period.PeriodDataProvider; +import org.hisp.dhis.program.Program; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingsProvider; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; @@ -117,14 +120,13 @@ protected Skip skipIndex(ValueType valueType, boolean hasOptionSet) { } /** - * Returns a select expression, potentially with a cast statement, based on the given value type. - * Handles data element and tracked entity attribute select expressions. + * Returns a column expression, potentially with a cast statement, based on the given value type. * * @param valueType the {@link ValueType} to represent as database column type. * @param columnExpression the expression or name of the column to be selected. * @return a select expression appropriate for the given value type and context. */ - protected String getSelectExpression(ValueType valueType, String columnExpression) { + protected String getColumnExpression(ValueType valueType, String columnExpression) { if (valueType.isDecimal()) { return getCastExpression(columnExpression, NUMERIC_REGEXP, sqlBuilder.dataTypeDouble()); } else if (valueType.isInteger()) { @@ -138,7 +140,8 @@ protected String getSelectExpression(ValueType valueType, String columnExpressio } else if (valueType.isGeo() && isSpatialSupport()) { return String.format( """ - ST_GeomFromGeoJSON('{"type":"Point", "coordinates":' || (%s) || ', "crs":{"type":"name", "properties":{"name":"EPSG:4326"}}}')""", + ST_GeomFromGeoJSON('{"type":"Point", "coordinates":' || (%s) || \ + ', "crs":{"type":"name", "properties":{"name":"EPSG:4326"}}}')""", columnExpression); } else { return columnExpression; @@ -219,14 +222,14 @@ protected void populateTableInternal(AnalyticsTablePartition partition, String f protected List getColumnForAttribute(TrackedEntityAttribute attribute) { List columns = new ArrayList<>(); + String valueColumn = String.format("%s.%s", quote(attribute.getUid()), "value"); DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); - String selectExpression = getSelectExpression(attribute.getValueType(), "value"); + String selectExpression = getColumnExpression(attribute.getValueType(), valueColumn); String dataFilterClause = getDataFilterClause(attribute); - String sql = getSelectSubquery(attribute, selectExpression, dataFilterClause); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); if (attribute.getValueType().isOrganisationUnit()) { - columns.addAll(getColumnForOrgUnitTrackedEntityAttribute(attribute, dataFilterClause)); + columns.addAll(getColumnForOrgUnitAttribute(attribute, dataFilterClause)); } columns.add( @@ -234,7 +237,7 @@ protected List getColumnForAttribute(TrackedEntityAttribut .name(attribute.getUid()) .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(dataType) - .selectExpression(sql) + .selectExpression(selectExpression) .skipIndex(skipIndex) .build()); @@ -248,7 +251,7 @@ protected List getColumnForAttribute(TrackedEntityAttribut * @param dataFilterClause the data filter clause. * @return a list of {@link AnalyticsTableColumn}. */ - private List getColumnForOrgUnitTrackedEntityAttribute( + private List getColumnForOrgUnitAttribute( TrackedEntityAttribute attribute, String dataFilterClause) { List columns = new ArrayList<>(); @@ -306,4 +309,34 @@ private String getSelectSubquery( "closingParentheses", getClosingParentheses(selectExpression), "attributeUid", quote(attribute.getUid()))); } + + /** + * Returns a join clause for attribute value for every attribute of the given program. + * + * @param program the {@link Program}. + * @return a join clause. + */ + protected String getAttributeValueJoinClause(Program program) { + String template = + """ + left join ${trackedentityattributevalue} as ${uid} \ + on en.trackedentityid=${uid}.trackedentityid \ + and ${uid}.trackedentityattributeid = ${id}\s"""; + + return program.getNonConfidentialTrackedEntityAttributes().stream() + .map(attribute -> replaceQualify(template, toVariableMap(attribute))) + .collect(Collectors.joining()); + } + + /** + * Returns a map of identifiable properties and values. + * + * @param object the {@link IdentifiableObject}. + * @return a {@link Map}. + */ + protected Map toVariableMap(IdentifiableObject object) { + return Map.of( + "id", String.valueOf(object.getId()), + "uid", quote(object.getUid())); + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index d18fbdffd80b..172745e793d3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -136,6 +136,7 @@ protected List getPartitionChecks(Integer year, Date endDate) { @Override public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTablePartition partition) { Program program = partition.getMasterTable().getProgram(); + String attributeJoinClause = getAttributeValueJoinClause(program); String fromClause = replaceQualify( @@ -148,6 +149,7 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti left join analytics_rs_dateperiodstructure dps on cast(en.enrollmentdate as date)=dps.dateperiod \ left join analytics_rs_orgunitstructure ous on en.organisationunitid=ous.organisationunitid \ left join analytics_rs_organisationunitgroupsetstructure ougs on en.organisationunitid=ougs.organisationunitid \ + ${attributeJoinClause}\ where pr.programid=${programId} \ and en.organisationunitid is not null \ and (ougs.startdate is null or dps.monthstartdate=ougs.startdate) \ @@ -155,6 +157,7 @@ left join analytics_rs_dateperiodstructure dps on cast(en.enrollmentdate as date and en.occurreddate is not null \ and en.deleted = false\s""", Map.of( + "attributeJoinClause", attributeJoinClause, "programId", String.valueOf(program.getId()), "startTime", toLongDate(params.getStartTime()))); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 7a4ff2e0ad6c..95a45283c8a6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -328,6 +328,7 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti Integer latestDataYear = availableDataYears.get(availableDataYears.size() - 1); Program program = partition.getMasterTable().getProgram(); String partitionClause = getPartitionClause(partition); + String attributeJoinClause = getAttributeValueJoinClause(program); String fromClause = replaceQualify( @@ -344,6 +345,7 @@ left join analytics_rs_dateperiodstructure dps on cast(${eventDateExpression} as left join analytics_rs_organisationunitgroupsetstructure ougs on ev.organisationunitid=ougs.organisationunitid \ left join ${organisationunit} enrollmentou on en.organisationunitid=enrollmentou.organisationunitid \ inner join analytics_rs_categorystructure acs on ev.attributeoptioncomboid=acs.categoryoptioncomboid \ + ${attributeJoinClause}\ where ev.lastupdated < '${startTime}' ${partitionClause} \ and pr.programid=${programId} \ and ev.organisationunitid is not null \ @@ -356,6 +358,7 @@ and ev.status in (${exportableEventStatues}) \ Map.of( "eventDateExpression", eventDateExpression, "partitionClause", partitionClause, + "attributeJoinClause", attributeJoinClause, "startTime", toLongDate(params.getStartTime()), "programId", String.valueOf(program.getId()), "firstDataYear", String.valueOf(firstDataYear), @@ -486,15 +489,16 @@ private List getColumnForDataElement( List columns = new ArrayList<>(); DataType dataType = getColumnType(dataElement.getValueType(), isSpatialSupport()); - String columnExpression = + String jsonExpression = sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); - String selectExpression = getSelectExpression(dataElement.getValueType(), columnExpression); + String columnExpression = getColumnExpression(dataElement.getValueType(), jsonExpression); String dataFilterClause = getDataFilterClause(dataElement); - String sql = String.format("%s as %s", selectExpression, quote(dataElement.getUid())); + String selectExpression = + String.format("%s as %s", columnExpression, quote(dataElement.getUid())); Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); if (withLegendSet) { - return getColumnFromDataElementWithLegendSet(dataElement, selectExpression, dataFilterClause); + return getColumnFromDataElementWithLegendSet(dataElement, columnExpression, dataFilterClause); } if (dataElement.getValueType().isOrganisationUnit()) { @@ -506,7 +510,7 @@ private List getColumnForDataElement( .name(dataElement.getUid()) .dimensionType(AnalyticsDimensionType.DYNAMIC) .dataType(dataType) - .selectExpression(sql) + .selectExpression(selectExpression) .skipIndex(skipIndex) .build()); @@ -587,7 +591,7 @@ private List getAttributeColumns(Program program) { */ private List getColumnForAttributeWithLegendSet( TrackedEntityAttribute attribute) { - String selectClause = getSelectExpression(attribute.getValueType(), "value"); + String columnExpression = getColumnExpression(attribute.getValueType(), "value"); String numericClause = getNumericClause(); String query = """ @@ -602,11 +606,11 @@ private List getColumnForAttributeWithLegendSet( .map( ls -> { String column = attribute.getUid() + PartitionUtils.SEP + ls.getUid(); - String sql = + String selectExpression = replaceQualify( query, Map.of( - "selectClause", selectClause, + "selectClause", columnExpression, "legendSetId", String.valueOf(ls.getId()), "column", column, "attributeId", String.valueOf(attribute.getId()), @@ -615,14 +619,14 @@ private List getColumnForAttributeWithLegendSet( return AnalyticsTableColumn.builder() .name(column) .dataType(CHARACTER_11) - .selectExpression(sql) + .selectExpression(selectExpression) .build(); }) .toList(); } /** - * Returns a select statement for the given data element with value type org unit. + * Returns a select statement for the given select expression. * * @param dataElement the data element to create the select statement for. * @param selectExpression the select expression. @@ -717,15 +721,14 @@ private List getDataYears( Program program, Integer firstDataYear, Integer lastDataYear) { + String fromDate = toMediumDate(params.getFromDate()); String fromDateClause = params.getFromDate() != null ? replace( "and (${eventDateExpression}) >= '${fromDate}'", Map.of( - "eventDateExpression", - eventDateExpression, - "fromDate", - toMediumDate(params.getFromDate()))) + "eventDateExpression", eventDateExpression, + "fromDate", fromDate)) : EMPTY; String sql = diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java index 05f4337cc10a..ddb386b7bd0b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java @@ -76,7 +76,7 @@ then cast(eventdatavalues #>> '{GieVkTxp4HH, value}' as double precision) \ else null end"""; String actual = - manager.getSelectExpression(ValueType.NUMBER, "eventdatavalues #>> '{GieVkTxp4HH, value}'"); + manager.getColumnExpression(ValueType.NUMBER, "eventdatavalues #>> '{GieVkTxp4HH, value}'"); assertEquals(expected, actual); } @@ -88,7 +88,7 @@ void testGetSelectExpressionBoolean() { case when eventdatavalues #>> '{Xl3voRRcmpo, value}' = 'true' then 1 when eventdatavalues #>> '{Xl3voRRcmpo, value}' = 'false' then 0 else null end"""; String actual = - manager.getSelectExpression( + manager.getColumnExpression( ValueType.BOOLEAN, "eventdatavalues #>> '{Xl3voRRcmpo, value}'"); assertEquals(expected, actual); @@ -103,7 +103,7 @@ then cast(eventdatavalues #>> '{AL04Wbutskk, value}' as timestamp) \ else null end"""; String actual = - manager.getSelectExpression(ValueType.DATE, "eventdatavalues #>> '{AL04Wbutskk, value}'"); + manager.getColumnExpression(ValueType.DATE, "eventdatavalues #>> '{AL04Wbutskk, value}'"); assertEquals(expected, actual); } @@ -115,7 +115,7 @@ void testGetSelectExpressionText() { eventdatavalues #>> '{FwUzmc49Pcr, value}'"""; String actual = - manager.getSelectExpression(ValueType.TEXT, "eventdatavalues #>> '{FwUzmc49Pcr, value}'"); + manager.getColumnExpression(ValueType.TEXT, "eventdatavalues #>> '{FwUzmc49Pcr, value}'"); assertEquals(expected, actual); } @@ -129,7 +129,7 @@ void testGetSelectExpressionGeometry() { ST_GeomFromGeoJSON('{"type":"Point", "coordinates":' || (eventdatavalues #>> '{C6bh7GevJfH, value}') || ', "crs":{"type":"name", "properties":{"name":"EPSG:4326"}}}')"""; String actual = - manager.getSelectExpression( + manager.getColumnExpression( ValueType.GEOJSON, "eventdatavalues #>> '{C6bh7GevJfH, value}'"); assertEquals(expected, actual); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java index d1b9f4443599..f2f2d642cf42 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java @@ -137,13 +137,7 @@ void verifyTeiTypeOrgUnitFetchesOuUidWhenPopulatingEventAnalyticsTable() { subject.populateTable(params, partition); verify(jdbcTemplate).execute(sql.capture()); - String ouQuery = - format( - """ - (select value from "trackedentityattributevalue" \ - where trackedentityid=en.trackedentityid and \ - trackedentityattributeid=9999) as %s""", - quote(tea.getUid())); + String ouQuery = format("%s.value", quote(tea.getUid())); assertThat(sql.getValue(), containsString(ouQuery)); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java index b7da2447b4ac..b2da976c129e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java @@ -501,10 +501,7 @@ void verifyGetTableWithTrackedEntityAttribute() { String aliasD1 = """ eventdatavalues #>> '{deabcdefghZ, value}' as "deabcdefghZ\""""; - String aliasTeaUid = - """ - (select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid \ - and trackedentityattributeid=%d) as "%s\""""; + String aliasTeaUid = "%s.value"; String aliasTea1 = """ @@ -537,12 +534,8 @@ void verifyGetTableWithTrackedEntityAttribute() { .withTableType(AnalyticsTableType.EVENT) .withColumnSize(59 + OU_NAME_HIERARCHY_COUNT) .addColumns(periodColumns) - .addColumn( - d1.getUid(), - TEXT, - toSelectExpression(aliasD1, d1.getUid()), - Skip.SKIP) // ValueType.TEXT - .addColumn(tea1.getUid(), TEXT, String.format(aliasTeaUid, tea1.getId(), tea1.getUid())) + .addColumn(d1.getUid(), TEXT, toSelectExpression(aliasD1, d1.getUid()), Skip.SKIP) + .addColumn(tea1.getUid(), TEXT, String.format(aliasTeaUid, quote(tea1.getUid()))) // Org unit geometry column .addColumn( tea1.getUid() + "_geom", @@ -650,12 +643,7 @@ void verifyTeiTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() { subject.populateTable(params, partition); verify(jdbcTemplate).execute(sql.capture()); - String ouUidQuery = - String.format( - """ - (select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid and \ - trackedentityattributeid=9999) as %s""", - quote(tea.getUid())); + String ouUidQuery = String.format("%s.value", quote(tea.getUid())); String ouNameQuery = String.format( @@ -944,12 +932,7 @@ void verifyTeaTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() { verify(jdbcTemplate).execute(sql.capture()); - String ouUidQuery = - String.format( - """ - select value from "trackedentityattributevalue" where trackedentityid=en.trackedentityid \ - and trackedentityattributeid=9999) as %s""", - quote(tea.getUid())); + String ouUidQuery = String.format("%s.value", quote(tea.getUid())); String ouNameQuery = String.format(