Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: Improve method structure for analytics tables [DHIS2-18417] #19271

Merged
merged 36 commits into from
Nov 24, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
a335311
fix: Update code
larshelge Nov 24, 2024
6d20921
fix: Update code
larshelge Nov 24, 2024
38c1df8
fix: Update code
larshelge Nov 24, 2024
49b58ea
fix: Update code
larshelge Nov 24, 2024
3b8c3ff
fix: Update code
larshelge Nov 24, 2024
3fec6c8
fix: Update code
larshelge Nov 24, 2024
87f0a72
fix: Update code
larshelge Nov 24, 2024
0dcaacf
fix: Update code
larshelge Nov 24, 2024
a397973
fix: Update code
larshelge Nov 24, 2024
74abf23
fix: Update code
larshelge Nov 24, 2024
a78d712
fix: Update code
larshelge Nov 24, 2024
4d6ce78
fix: Update code
larshelge Nov 24, 2024
7adfda5
fix: Update code
larshelge Nov 24, 2024
11f688f
fix: Update code
larshelge Nov 24, 2024
db7a54b
fix: Update code
larshelge Nov 24, 2024
243dc42
fix: Update code
larshelge Nov 24, 2024
5f6b566
fix: Update code
larshelge Nov 24, 2024
3e98d63
fix: Update code
larshelge Nov 24, 2024
089182d
fix: Update code
larshelge Nov 24, 2024
eb0bd0d
fix: Update code
larshelge Nov 24, 2024
6f72165
fix: Update code
larshelge Nov 24, 2024
719155b
fix: Update code
larshelge Nov 24, 2024
d811508
fix: Update code
larshelge Nov 24, 2024
5df4735
fix: Update code
larshelge Nov 24, 2024
49569a1
fix: Update code
larshelge Nov 24, 2024
ec6c906
fix: Update code
larshelge Nov 24, 2024
7db77c5
fix: Update code
larshelge Nov 24, 2024
4d2ad59
fix: Update code
larshelge Nov 24, 2024
c4a882f
Merge branch 'master' into DHIS2-18417
larshelge Nov 24, 2024
9af7540
fix: Update code
larshelge Nov 24, 2024
a5ac41c
fix: Update code
larshelge Nov 24, 2024
952b0e8
fix: Update code
larshelge Nov 24, 2024
c7e5bf0
fix: Update code
larshelge Nov 24, 2024
4e2b03f
fix: Update code
larshelge Nov 24, 2024
f7aa4c5
fix: Update code
larshelge Nov 24, 2024
f767809
fix: Update code
larshelge Nov 24, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,12 @@ left join analytics_rs_dateperiodstructure dps on cast(en.enrollmentdate as date
populateTableInternal(partition, fromClause);
}

/**
* Returns a list of columns for the given program.
*
* @param program the {@link Program}.
* @return a list of {@link AnalyticsTableColumn}.
*/
private List<AnalyticsTableColumn> getColumns(Program program) {
List<AnalyticsTableColumn> columns = new ArrayList<>();
columns.addAll(fixedColumns);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ private List<AnalyticsTable> getLatestAnalyticsTables(AnalyticsTableUpdateParams

Assert.isTrue(
lastFullTableUpdate.getTime() > 0L,
"A full analytics table update process must be run prior to a latest partition update process");
"A full analytics table update process must be run prior to a latest partition update");

Date startDate = lastFullTableUpdate;
Date endDate = params.getStartTime();
Expand Down Expand Up @@ -429,31 +429,25 @@ private List<AnalyticsTableColumn> getColumns(Program program) {

columns.addAll(
program.getAnalyticsDataElements().stream()
.map(de -> getColumnFromDataElement(de, false))
.map(de -> getColumnForDataElement(de, false))
.flatMap(Collection::stream)
.toList());

columns.addAll(
program.getAnalyticsDataElementsWithLegendSet().stream()
.map(de -> getColumnFromDataElement(de, true))
.map(de -> getColumnForDataElement(de, true))
.flatMap(Collection::stream)
.toList());

columns.addAll(
program.getNonConfidentialTrackedEntityAttributes().stream()
.map(
tea ->
getColumnFromTrackedEntityAttribute(
tea, getNumericClause(), getDateClause(), false))
.map(tea -> getColumnForTrackedEntityAttribute(tea, false))
.flatMap(Collection::stream)
.toList());

columns.addAll(
program.getNonConfidentialTrackedEntityAttributesWithLegendSet().stream()
.map(
tea ->
getColumnFromTrackedEntityAttribute(
tea, getNumericClause(), getDateClause(), true))
.map(tea -> getColumnForTrackedEntityAttribute(tea, true))
.flatMap(Collection::stream)
.toList());

Expand Down Expand Up @@ -488,20 +482,20 @@ protected AnalyticsTableColumn getPartitionColumn() {
* @param withLegendSet indicates
* @return
*/
private List<AnalyticsTableColumn> getColumnFromDataElement(
private List<AnalyticsTableColumn> getColumnForDataElement(
DataElement dataElement, boolean withLegendSet) {
List<AnalyticsTableColumn> columns = new ArrayList<>();

DataType dataType = getColumnType(dataElement.getValueType(), isSpatialSupport());
String columnExpression =
sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value");
String selectExpression = getSelectExpression(dataElement.getValueType(), columnExpression);
String dataFilterClause = getDataFilterClause(dataElement.getUid(), dataElement.getValueType());
String dataFilterClause = getDataFilterClause(dataElement);
String sql = getSelectForInsert(dataElement, selectExpression, dataFilterClause);
Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet());

if (dataElement.getValueType().isOrganisationUnit()) {
columns.addAll(getColumnFromOrgUnitDataElement(dataElement, dataFilterClause));
columns.addAll(getColumnForOrgUnitDataElement(dataElement, dataFilterClause));
}

columns.add(
Expand All @@ -518,7 +512,7 @@ private List<AnalyticsTableColumn> getColumnFromDataElement(
: columns;
}

private List<AnalyticsTableColumn> getColumnFromOrgUnitDataElement(
private List<AnalyticsTableColumn> getColumnForOrgUnitDataElement(
DataElement dataElement, String dataFilterClause) {
List<AnalyticsTableColumn> columns = new ArrayList<>();

Expand Down Expand Up @@ -556,22 +550,18 @@ private List<AnalyticsTableColumn> getColumnFromOrgUnitDataElement(
return columns;
}

private List<AnalyticsTableColumn> getColumnFromTrackedEntityAttribute(
TrackedEntityAttribute attribute,
String numericClause,
String dateClause,
boolean withLegendSet) {
private List<AnalyticsTableColumn> getColumnForTrackedEntityAttribute(
TrackedEntityAttribute attribute, boolean withLegendSet) {
List<AnalyticsTableColumn> columns = new ArrayList<>();

DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport());
String selectExpression = getSelectExpressionForAttribute(attribute.getValueType(), "value");
String dataExpression =
attribute.isNumericType() ? numericClause : attribute.isDateType() ? dateClause : "";
String dataExpression = getDataFilterClause(attribute);
String sql = selectForInsert(attribute, selectExpression, dataExpression);
Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet());

if (attribute.getValueType().isOrganisationUnit()) {
columns.addAll(getColumnsFromOrgUnitTrackedEntityAttribute(attribute, dataExpression));
columns.addAll(getColumnsForOrgUnitTrackedEntityAttribute(attribute, dataExpression));
}

columns.add(
Expand All @@ -583,14 +573,19 @@ private List<AnalyticsTableColumn> getColumnFromTrackedEntityAttribute(
.skipIndex(skipIndex)
.build());

return withLegendSet
? getColumnFromTrackedEntityAttributeWithLegendSet(attribute, numericClause)
: columns;
return withLegendSet ? getColumnForAttributeWithLegendSet(attribute) : columns;
}

private List<AnalyticsTableColumn> getColumnFromTrackedEntityAttributeWithLegendSet(
TrackedEntityAttribute attribute, String numericClause) {
/**
* Returns a list of columns based on the given attribute with legend set.
*
* @param attribute the {@link TrackedEntityAttribute}.
* @return a list of {@link AnalyticsTableColumn}.
*/
private List<AnalyticsTableColumn> getColumnForAttributeWithLegendSet(
TrackedEntityAttribute attribute) {
String selectClause = getSelectExpression(attribute.getValueType(), "value");
String numericClause = getNumericClause();
String query =
"""
\s(select l.uid from ${maplegend} l \
Expand Down Expand Up @@ -623,7 +618,14 @@ private List<AnalyticsTableColumn> getColumnFromTrackedEntityAttributeWithLegend
.collect(toList());
}

private List<AnalyticsTableColumn> getColumnsFromOrgUnitTrackedEntityAttribute(
/**
* Returns a list of columns based on the given attribute.
*
* @param attribute the {@link TrackedEntityAttribute}.
* @param dataFilterClause the data filter clause.
* @return a list of {@link AnalyticsTableColumn}.
*/
private List<AnalyticsTableColumn> getColumnsForOrgUnitTrackedEntityAttribute(
TrackedEntityAttribute attribute, String dataFilterClause) {
List<AnalyticsTableColumn> columns = new ArrayList<>();

Expand Down Expand Up @@ -670,21 +672,20 @@ private String getSelectForInsert(
DataElement dataElement, String selectExpression, String dataFilterClause) {
String sqlTemplate =
dataElement.getValueType().isOrganisationUnit()
? "(select ${fromType} ${dataClause})${closingParentheses} as ${uid}"
: "(select ${fromType} from ${event} where eventid=ev.eventid ${dataClause})${closingParentheses} as ${uid}";
? "(select ${selectExpression} ${dataClause})${closingParentheses} as ${uid}"
: "(select ${selectExpression} from ${event} where eventid=ev.eventid ${dataClause})${closingParentheses} as ${uid}";

Map<String, String> variables =
return replaceQualify(
sqlTemplate,
Map.of(
"fromType",
"selectExpression",
selectExpression,
"dataClause",
dataFilterClause,
"closingParentheses",
getClosingParentheses(selectExpression),
"uid",
quote(dataElement.getUid()));

return replaceQualify(sqlTemplate, variables);
quote(dataElement.getUid())));
}

/**
Expand Down Expand Up @@ -731,11 +732,13 @@ private List<AnalyticsTableColumn> getColumnFromDataElementWithLegendSet(
* For numeric and date value types, returns a data filter clause for checking whether the value
* is valid according to the value type. For other value types, returns the empty string.
*
* @param uid the identifier.
* @param valueType the {@link ValueType}.
* @return an expression for extracting a data value.
* @param dataElement the {@link DataElement}.
* @return an filter expression.
*/
private String getDataFilterClause(String uid, ValueType valueType) {
private String getDataFilterClause(DataElement dataElement) {
String uid = dataElement.getUid();
ValueType valueType = dataElement.getValueType();

if (valueType.isNumeric() || valueType.isDate()) {
String regex = valueType.isNumeric() ? NUMERIC_LENIENT_REGEXP : DATE_REGEXP;

Expand All @@ -747,6 +750,20 @@ private String getDataFilterClause(String uid, ValueType valueType) {
return EMPTY;
}

/**
* For numeric and date value types, returns a data filter clause for checking whether the value
* is valid according to the value type. For other value types, returns the empty string.
*
* @param attribute the {@link TrackedEntityAttribute}.
* @return an filter expression.
*/
private String getDataFilterClause(TrackedEntityAttribute attribute) {
if (attribute.isNumericType()) {
return getNumericClause();
}
return attribute.isDateType() ? getDateClause() : EMPTY;
}

/**
* Returns a list of years for which data exist.
*
Expand Down
Loading