Skip to content

Commit

Permalink
refactor: Improve method structure for analytics tables [DHIS2-18417] (…
Browse files Browse the repository at this point in the history 
…#19271)
  • Loading branch information
@larshelge
larshelge authored Nov 24, 2024
1 parent d75afb5 commit 3751fc6
Show file tree
Hide file tree
Showing 2 changed files with 63 additions and 40 deletions.
6 changes: 6 additions & 0 deletions ...tics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,12 @@ left join analytics_rs_dateperiodstructure dps on cast(en.enrollmentdate as date
populateTableInternal(partition, fromClause);
}

/**
* Returns a list of columns for the given program.
*
* @param program the {@link Program}.
* @return a list of {@link AnalyticsTableColumn}.
*/
private List<AnalyticsTableColumn> getColumns(Program program) {
List<AnalyticsTableColumn> columns = new ArrayList<>();
columns.addAll(fixedColumns);
Expand Down
97 changes: 57 additions & 40 deletions ...analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ private List<AnalyticsTable> getLatestAnalyticsTables(AnalyticsTableUpdateParams

Assert.isTrue(
lastFullTableUpdate.getTime() > 0L,
"A full analytics table update process must be run prior to a latest partition update process");
"A full analytics table update process must be run prior to a latest partition update");

Date startDate = lastFullTableUpdate;
Date endDate = params.getStartTime();
Expand Down Expand Up @@ -429,31 +429,25 @@ private List<AnalyticsTableColumn> getColumns(Program program) {

columns.addAll(
program.getAnalyticsDataElements().stream()
.map(de -> getColumnFromDataElement(de, false))
.map(de -> getColumnForDataElement(de, false))
.flatMap(Collection::stream)
.toList());

columns.addAll(
program.getAnalyticsDataElementsWithLegendSet().stream()
.map(de -> getColumnFromDataElement(de, true))
.map(de -> getColumnForDataElement(de, true))
.flatMap(Collection::stream)
.toList());

columns.addAll(
program.getNonConfidentialTrackedEntityAttributes().stream()
.map(
tea ->
getColumnFromTrackedEntityAttribute(
tea, getNumericClause(), getDateClause(), false))
.map(tea -> getColumnForTrackedEntityAttribute(tea, false))
.flatMap(Collection::stream)
.toList());

columns.addAll(
program.getNonConfidentialTrackedEntityAttributesWithLegendSet().stream()
.map(
tea ->
getColumnFromTrackedEntityAttribute(
tea, getNumericClause(), getDateClause(), true))
.map(tea -> getColumnForTrackedEntityAttribute(tea, true))
.flatMap(Collection::stream)
.toList());

Expand Down Expand Up @@ -488,20 +482,20 @@ protected AnalyticsTableColumn getPartitionColumn() {
* @param withLegendSet indicates
* @return
*/
private List<AnalyticsTableColumn> getColumnFromDataElement(
private List<AnalyticsTableColumn> getColumnForDataElement(
DataElement dataElement, boolean withLegendSet) {
List<AnalyticsTableColumn> columns = new ArrayList<>();

DataType dataType = getColumnType(dataElement.getValueType(), isSpatialSupport());
String columnExpression =
sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value");
String selectExpression = getSelectExpression(dataElement.getValueType(), columnExpression);
String dataFilterClause = getDataFilterClause(dataElement.getUid(), dataElement.getValueType());
String dataFilterClause = getDataFilterClause(dataElement);
String sql = getSelectForInsert(dataElement, selectExpression, dataFilterClause);
Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet());

if (dataElement.getValueType().isOrganisationUnit()) {
columns.addAll(getColumnFromOrgUnitDataElement(dataElement, dataFilterClause));
columns.addAll(getColumnForOrgUnitDataElement(dataElement, dataFilterClause));
}

columns.add(
Expand All @@ -518,7 +512,7 @@ private List<AnalyticsTableColumn> getColumnFromDataElement(
: columns;
}

private List<AnalyticsTableColumn> getColumnFromOrgUnitDataElement(
private List<AnalyticsTableColumn> getColumnForOrgUnitDataElement(
DataElement dataElement, String dataFilterClause) {
List<AnalyticsTableColumn> columns = new ArrayList<>();

Expand Down Expand Up @@ -556,22 +550,18 @@ private List<AnalyticsTableColumn> getColumnFromOrgUnitDataElement(
return columns;
}

private List<AnalyticsTableColumn> getColumnFromTrackedEntityAttribute(
TrackedEntityAttribute attribute,
String numericClause,
String dateClause,
boolean withLegendSet) {
private List<AnalyticsTableColumn> getColumnForTrackedEntityAttribute(
TrackedEntityAttribute attribute, boolean withLegendSet) {
List<AnalyticsTableColumn> columns = new ArrayList<>();

DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport());
String selectExpression = getSelectExpressionForAttribute(attribute.getValueType(), "value");
String dataExpression =
attribute.isNumericType() ? numericClause : attribute.isDateType() ? dateClause : "";
String dataExpression = getDataFilterClause(attribute);
String sql = selectForInsert(attribute, selectExpression, dataExpression);
Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet());

if (attribute.getValueType().isOrganisationUnit()) {
columns.addAll(getColumnsFromOrgUnitTrackedEntityAttribute(attribute, dataExpression));
columns.addAll(getColumnsForOrgUnitTrackedEntityAttribute(attribute, dataExpression));
}

columns.add(
Expand All @@ -583,14 +573,19 @@ private List<AnalyticsTableColumn> getColumnFromTrackedEntityAttribute(
.skipIndex(skipIndex)
.build());

return withLegendSet
? getColumnFromTrackedEntityAttributeWithLegendSet(attribute, numericClause)
: columns;
return withLegendSet ? getColumnForAttributeWithLegendSet(attribute) : columns;
}

private List<AnalyticsTableColumn> getColumnFromTrackedEntityAttributeWithLegendSet(
TrackedEntityAttribute attribute, String numericClause) {
/**
* Returns a list of columns based on the given attribute with legend set.
*
* @param attribute the {@link TrackedEntityAttribute}.
* @return a list of {@link AnalyticsTableColumn}.
*/
private List<AnalyticsTableColumn> getColumnForAttributeWithLegendSet(
TrackedEntityAttribute attribute) {
String selectClause = getSelectExpression(attribute.getValueType(), "value");
String numericClause = getNumericClause();
String query =
"""
\s(select l.uid from ${maplegend} l \
Expand Down Expand Up @@ -623,7 +618,14 @@ private List<AnalyticsTableColumn> getColumnFromTrackedEntityAttributeWithLegend
.collect(toList());
}

private List<AnalyticsTableColumn> getColumnsFromOrgUnitTrackedEntityAttribute(
/**
* Returns a list of columns based on the given attribute.
*
* @param attribute the {@link TrackedEntityAttribute}.
* @param dataFilterClause the data filter clause.
* @return a list of {@link AnalyticsTableColumn}.
*/
private List<AnalyticsTableColumn> getColumnsForOrgUnitTrackedEntityAttribute(
TrackedEntityAttribute attribute, String dataFilterClause) {
List<AnalyticsTableColumn> columns = new ArrayList<>();

Expand Down Expand Up @@ -670,21 +672,20 @@ private String getSelectForInsert(
DataElement dataElement, String selectExpression, String dataFilterClause) {
String sqlTemplate =
dataElement.getValueType().isOrganisationUnit()
? "(select ${fromType} ${dataClause})${closingParentheses} as ${uid}"
: "(select ${fromType} from ${event} where eventid=ev.eventid ${dataClause})${closingParentheses} as ${uid}";
? "(select ${selectExpression} ${dataClause})${closingParentheses} as ${uid}"
: "(select ${selectExpression} from ${event} where eventid=ev.eventid ${dataClause})${closingParentheses} as ${uid}";

Map<String, String> variables =
return replaceQualify(
sqlTemplate,
Map.of(
"fromType",
"selectExpression",
selectExpression,
"dataClause",
dataFilterClause,
"closingParentheses",
getClosingParentheses(selectExpression),
"uid",
quote(dataElement.getUid()));

return replaceQualify(sqlTemplate, variables);
quote(dataElement.getUid())));
}

/**
Expand Down Expand Up @@ -731,11 +732,13 @@ private List<AnalyticsTableColumn> getColumnFromDataElementWithLegendSet(
* For numeric and date value types, returns a data filter clause for checking whether the value
* is valid according to the value type. For other value types, returns the empty string.
*
* @param uid the identifier.
* @param valueType the {@link ValueType}.
* @return an expression for extracting a data value.
* @param dataElement the {@link DataElement}.
* @return an filter expression.
*/
private String getDataFilterClause(String uid, ValueType valueType) {
private String getDataFilterClause(DataElement dataElement) {
String uid = dataElement.getUid();
ValueType valueType = dataElement.getValueType();

if (valueType.isNumeric() || valueType.isDate()) {
String regex = valueType.isNumeric() ? NUMERIC_LENIENT_REGEXP : DATE_REGEXP;

Expand All @@ -747,6 +750,20 @@ private String getDataFilterClause(String uid, ValueType valueType) {
return EMPTY;
}

/**
* For numeric and date value types, returns a data filter clause for checking whether the value
* is valid according to the value type. For other value types, returns the empty string.
*
* @param attribute the {@link TrackedEntityAttribute}.
* @return an filter expression.
*/
private String getDataFilterClause(TrackedEntityAttribute attribute) {
if (attribute.isNumericType()) {
return getNumericClause();
}
return attribute.isDateType() ? getDateClause() : EMPTY;
}

/**
* Returns a list of years for which data exist.
*
Expand Down

0 comments on commit 3751fc6

Please sign in to comment.