refactor: Join attribute value table for attributes [DHIS2-18417]

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java

@@ -172,9 +172,9 @@
     } else if (valueType.isOrganisationUnit()) {
       String ouClause =
           isTea
-              ? "ou.uid from ${organisationunit} ou where ou.uid = (select ${columnName}"
-              : "ou.uid from ${organisationunit} ou where ou.uid = ${columnName}";
-      return replaceQualify(ouClause, Map.of("columnName", columnExpression));
+              ? "ou.uid from ${organisationunit} ou where ou.uid = (select ${columnExpression}"
+              : "ou.uid from ${organisationunit} ou where ou.uid = ${columnExpression}";
+      return replaceQualify(ouClause, Map.of("columnExpression", columnExpression));
     } else {
       return columnExpression;
     }
@@ -261,7 +261,7 @@
     Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet());
 
     if (attribute.getValueType().isOrganisationUnit()) {
-      columns.addAll(getColumnForOrgUnitTrackedEntityAttribute(attribute, dataFilterClause));
+      columns.addAll(getColumnForOrgUnitAttribute(attribute, dataFilterClause));
     }
 
     columns.add(
@@ -283,7 +283,7 @@
    * @param dataFilterClause the data filter clause.
    * @return a list of {@link AnalyticsTableColumn}.
    */
-  private List<AnalyticsTableColumn> getColumnForOrgUnitTrackedEntityAttribute(
+  private List<AnalyticsTableColumn> getColumnForOrgUnitAttribute(
       TrackedEntityAttribute attribute, String dataFilterClause) {
     List<AnalyticsTableColumn> columns = new ArrayList<>();
 
@@ -318,6 +318,13 @@
     return columns;
   }
 
+  private String getSelectForInsert(
+      TrackedEntityAttribute attribute, String selectExpression, String dataFilterClause) {
+    return String.format(
+        """
+        %s.%s as %s, """);
+  }
+
   /**
    * The select subquery statement.
    *

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java

@@ -431,7 +431,7 @@ protected AnalyticsTable getLatestAnalyticsTable(
   protected void invokeTimeAndLog(String sql, String logPattern, Object... args) {
     Timer timer = new SystemTimer().start();
 
-    log.debug("Populate table SQL: '{}'", sql);
+    log.info("Populate table SQL: '{}'", sql);
 
     jdbcTemplate.execute(sql);
 

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java

@@ -50,6 +50,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.hisp.dhis.analytics.AnalyticsTableHookService;
 import org.hisp.dhis.analytics.AnalyticsTableType;
@@ -65,6 +66,7 @@
 import org.hisp.dhis.calendar.Calendar;
 import org.hisp.dhis.category.Category;
 import org.hisp.dhis.category.CategoryService;
+import org.hisp.dhis.common.IdentifiableObject;
 import org.hisp.dhis.common.IdentifiableObjectManager;
 import org.hisp.dhis.common.ValueType;
 import org.hisp.dhis.commons.collection.ListUtils;
@@ -330,6 +332,7 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti
     Integer latestDataYear = availableDataYears.get(availableDataYears.size() - 1);
     Program program = partition.getMasterTable().getProgram();
     String partitionClause = getPartitionClause(partition);
+    String attributeJoinClause = getAttributeValueJoinClause(program);
 
     String fromClause =
         replaceQualify(
@@ -346,6 +349,7 @@ left join analytics_rs_dateperiodstructure dps on cast(${eventDateExpression} as
             left join analytics_rs_organisationunitgroupsetstructure ougs on ev.organisationunitid=ougs.organisationunitid \
             left join ${organisationunit} enrollmentou on en.organisationunitid=enrollmentou.organisationunitid \
             inner join analytics_rs_categorystructure acs on ev.attributeoptioncomboid=acs.categoryoptioncomboid \
+            ${attributeJoinClause}\
             where ev.lastupdated < '${startTime}' ${partitionClause} \
             and pr.programid=${programId} \
             and ev.organisationunitid is not null \
@@ -358,6 +362,7 @@ and ev.status in (${exportableEventStatues}) \
             Map.of(
                 "eventDateExpression", eventDateExpression,
                 "partitionClause", partitionClause,
+                "attributeJoinClause", attributeJoinClause,
                 "startTime", toLongDate(params.getStartTime()),
                 "programId", String.valueOf(program.getId()),
                 "firstDataYear", String.valueOf(firstDataYear),
@@ -367,6 +372,36 @@ and ev.status in (${exportableEventStatues}) \
     populateTableInternal(partition, fromClause);
   }
 
+  /**
+   * Returns a join clause for attribute value for every attribute of the given program.
+   *
+   * @param program the {@link Program}.
+   * @return a join clause.
+   */
+  private String getAttributeValueJoinClause(Program program) {
+    String template =
+        """
+        left join ${trackedentityattributevalue} as ${uid} \
+        on en.trackedentityid=${uid}.trackedentityid \
+        and ${uid}.trackedentityattributeid = ${id}\s""";
+
+    return program.getNonConfidentialTrackedEntityAttributes().stream()
+        .map(attribute -> replaceQualify(template, toVariableMap(attribute)))
+        .collect(Collectors.joining());
+  }
+
+  /**
+   * Returns a variable map.
+   *
+   * @param object the object.
+   * @return a {@link Map}.
+   */
+  private Map<String, String> toVariableMap(IdentifiableObject object) {
+    return Map.of(
+        "id", String.valueOf(object.getId()),
+        "uid", quote(object.getUid()));
+  }
+
   /**
    * Returns a partition SQL clause.
    *
@@ -624,7 +659,7 @@ private List<AnalyticsTableColumn> getColumnForAttributeWithLegendSet(
   }
 
   /**
-   * Retyrns a select statement for the given select expression.
+   * Returns a select statement for the given select expression.
    *
    * @param dataElement the data element to create the select statement for.
    * @param selectExpression the select expression.
@@ -633,6 +668,7 @@ private List<AnalyticsTableColumn> getColumnForAttributeWithLegendSet(
    */
   private String getSelectForInsert(
       DataElement dataElement, String selectExpression, String dataFilterClause) {
+    String parentheses = getClosingParentheses(selectExpression);
     String sqlTemplate =
         dataElement.getValueType().isOrganisationUnit()
             ? "(select ${selectExpression} ${dataClause})${closingParentheses} as ${uid}"
@@ -641,14 +677,10 @@ private String getSelectForInsert(
     return replaceQualify(
         sqlTemplate,
         Map.of(
-            "selectExpression",
-            selectExpression,
-            "dataClause",
-            dataFilterClause,
-            "closingParentheses",
-            getClosingParentheses(selectExpression),
-            "uid",
-            quote(dataElement.getUid())));
+            "selectExpression", selectExpression,
+            "dataClause", dataFilterClause,
+            "closingParentheses", parentheses,
+            "uid", quote(dataElement.getUid())));
   }
 
   /**
@@ -726,15 +758,14 @@ private List<Integer> getDataYears(
       Program program,
       Integer firstDataYear,
       Integer lastDataYear) {
+    String fromDate = toMediumDate(params.getFromDate());
     String fromDateClause =
         params.getFromDate() != null
             ? replace(
                 "and (${eventDateExpression}) >= '${fromDate}'",
                 Map.of(
-                    "eventDateExpression",
-                    eventDateExpression,
-                    "fromDate",
-                    toMediumDate(params.getFromDate())))
+                    "eventDateExpression", eventDateExpression,
+                    "fromDate", fromDate))
             : EMPTY;
 
     String sql =