diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelection.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelection.java new file mode 100644 index 000000000000..fa89dd4d2e53 --- /dev/null +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelection.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics; + +import java.util.Set; +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor +@Getter +public class OptionSetSelection { + private String qualifiedUid; + private String optionSetUid; + private Set options; + private OptionSetSelectionMode optionSetSelectionMode; + + @Override + public String toString() { + return "OptionSetSelection{" + + "qualifiedUid='" + + qualifiedUid + + '\'' + + "optionSetUid='" + + optionSetUid + + '\'' + + ", options=" + + options + + ", optionSetSelectionMode=" + + optionSetSelectionMode + + '}'; + } +} diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteria.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteria.java new file mode 100644 index 000000000000..155fe44856d2 --- /dev/null +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteria.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics; + +import static org.apache.commons.lang3.StringUtils.EMPTY; + +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public class OptionSetSelectionCriteria { + private Map optionSetSelections; + + @Override + public String toString() { + if (optionSetSelections == null || optionSetSelections.isEmpty()) { + return EMPTY; + } + + return "OptionSetSelectionCriteria{" + "optionSetSelections=" + optionSetSelections + '}'; + } +} diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionMode.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionMode.java new file mode 100644 index 000000000000..b7c0dc76139a --- /dev/null +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionMode.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics; + +import java.util.Arrays; +import java.util.List; + +/** The selection modes for dimension items with {@link OptionSet}. */ +public enum OptionSetSelectionMode { + /** + * All options in an option set are chosen and aggregated into a single column. This selection is + * relative, so any new options added to the option set are included. This is the default mode. + */ + AGGREGATED, + /** + * All options in an option set are chosen and displayed as data items. This selection is + * relative, so any new options added to the option set are included. + */ + DISAGGREGATED; + + public static List getOptionSetSelectionModes() { + return Arrays.stream(OptionSetSelectionMode.values()).map(Enum::toString).toList(); + } +} diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java index a7786ad46813..847a31093076 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java @@ -33,6 +33,7 @@ import static org.apache.commons.lang3.StringUtils.defaultIfBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.commons.lang3.StringUtils.substringAfterLast; +import static org.hisp.dhis.analytics.OptionSetSelectionMode.AGGREGATED; import static org.hisp.dhis.common.DimensionalObject.DIMENSION_NAME_SEP; import static org.hisp.dhis.common.DimensionalObject.DIMENSION_SEP; import static org.hisp.dhis.common.DimensionalObject.ITEM_SEP; @@ -57,6 +58,7 @@ import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Triple; +import org.hisp.dhis.analytics.OptionSetSelectionMode; import org.hisp.dhis.common.comparator.ObjectStringValueComparator; import org.hisp.dhis.dataelement.DataElementOperand; import org.hisp.dhis.eventvisualization.Attribute; @@ -69,8 +71,6 @@ */ @NoArgsConstructor(access = PRIVATE) public class DimensionalObjectUtils { - public static final String COMPOSITE_DIM_OBJECT_ESCAPED_SEP = "\\."; - public static final String COMPOSITE_DIM_OBJECT_PLAIN_SEP = "."; public static final String TITLE_ITEM_SEP = ", "; @@ -81,12 +81,20 @@ public class DimensionalObjectUtils { public static final String COL_SEP = " "; + /** + * Regex to ignore splitting by ";" inside square brackets []. ie: + * dx:FTRrcoaog83;WSGAb5XwJ3Y.QFX1FLWBwtq.R3ShQczKnI9[l8S7SjnQ58G;rexqxNDqUKg], splits into + * FTRrcoaog83 and WSGAb5XwJ3Y.QFX1FLWBwtq.R3ShQczKnI9[l8S7SjnQ58G;rexqxNDqUKg] + */ + private static final Pattern DX_REGEX_PATTERN = Pattern.compile(";(?![^\\(\\[]*[\\]\\)])"); + /** * Matching data element operand, program data element, program attribute, data set reporting rate - * metric. + * metric. ie: Luqe6ps5KZ9.uTLkjHWtSL8.R0jROOT3zni-AGGREGATED */ private static final Pattern COMPOSITE_DIM_OBJECT_PATTERN = - Pattern.compile("(?\\w+)\\.(?\\w+|\\*)(\\.(?\\w+|\\*))?"); + Pattern.compile( + "(?\\w+)\\.(?\\w+|\\*)(\\.(?\\w+|\\*))?(\\[(?[^\\]]*?)\\])?(-(?AGGREGATED|DISAGGREGATED)?)?"); private static final Set IGNORED_OPERATORS = Set.of(QueryOperator.LIKE, QueryOperator.IN, QueryOperator.SW, QueryOperator.EW); @@ -359,6 +367,21 @@ public static String getDimensionFromParam(String param) { return param.split(DIMENSION_NAME_SEP).length > 0 ? param.split(DIMENSION_NAME_SEP)[0] : param; } + /** + * Retrieves the value from the given dimension param. Returns the part of the string after the + * dimension name separator, or the whole string if the separator is not present. ie: + * dx:WSGAb5XwJ3Y.QFX1FLWBwtq, becomes WSGAb5XwJ3Y.QFX1FLWBwtq + * + * @param param the parameter. + */ + public static String getValueFromDimensionParam(String param) { + if (param == null) { + return null; + } + + return param.split(DIMENSION_NAME_SEP).length > 1 ? param.split(DIMENSION_NAME_SEP)[1] : param; + } + /** * Retrieves the dimension options from the given string. Looks for the part succeeding the * dimension name separator, if exists, splits the string part on the option separator and returns @@ -371,11 +394,10 @@ public static List getDimensionItemsFromParam(String param) { } if (param.split(DIMENSION_NAME_SEP).length > 1) { - // Extracts dimension items by removing dimension name and separator + // Extracts dimension items by removing dimension name and separator. String dimensionItems = param.substring(param.indexOf(DIMENSION_NAME_SEP) + 1); - // Returns them as List - return Arrays.asList(dimensionItems.split(OPTION_SEP)); + return Arrays.asList(DX_REGEX_PATTERN.split(dimensionItems)); } return new ArrayList<>(); @@ -521,20 +543,78 @@ public static boolean isCompositeDimensionalObject(String expression) { * @param compositeItem the composite dimension object identifier. * @return the first identifier, or null if not a valid composite identifier or no match. */ - public static String getFirstIdentifer(String compositeItem) { + public static String getFirstIdentifier(String compositeItem) { + if (compositeItem == null) { + return null; + } + Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); - return matcher.matches() ? matcher.group(1) : null; + return matcher.matches() ? matcher.group("id1") : null; } /** * Returns the second identifier in a composite dimension object identifier. * * @param compositeItem the composite dimension object identifier. - * @return the second identifier, or null if not a valid composite identifier or no match. + * @return the second identifier, or null if thr composite identifier is not valid or do not + * match. */ - public static String getSecondIdentifer(String compositeItem) { + public static String getSecondIdentifier(String compositeItem) { + if (compositeItem == null) { + return null; + } + Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); - return matcher.matches() ? matcher.group(2) : null; + return matcher.matches() ? matcher.group("id2") : null; + } + + /** + * Returns the third identifier in a composite dimension object identifier. + * + * @param compositeItem the composite dimension object identifier. + * @return the third identifier, or null if thr composite identifier is not valid or do not match. + */ + public static String getThirdIdentifier(String compositeItem) { + if (compositeItem == null) { + return null; + } + + Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); + + return matcher.matches() ? matcher.group("id3") : null; + } + + /** + * Based on the given argument, it will return the associated {@link OptionSetSelectionMode}. If + * the argument is null, or no association is found, it returns the default mode "AGGREGATED". + * + * @param composedOptionSetId the full option set dimension, ie: + * Luqe6ps5KZ9.uTLkjHWtSL8.R0jROOT3zni-AGGREGATED. + * @return the respective {@link OptionSetSelectionMode}, or the default mode. + */ + public static OptionSetSelectionMode getOptionSetSelectionMode(String composedOptionSetId) { + if (composedOptionSetId == null) { + return null; + } + + Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(composedOptionSetId); + if (matcher.matches()) { + String suffix = matcher.group("suffix"); + return suffix != null ? OptionSetSelectionMode.valueOf(suffix) : AGGREGATED; + } + + return AGGREGATED; + } + + /** + * Luqe6ps5KZ9.uTLkjHWtSL8.R0jROOT3zni-AGGREGATED + * + * @param optionSetParam + * @return the options specified for the option set param, or null. + */ + public static String getOptionsParam(String optionSetParam) { + Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(optionSetParam); + return matcher.matches() ? matcher.group("list") : null; } /** diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java index e21ab1118fc3..2647dc353b7d 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java @@ -72,6 +72,7 @@ public enum ErrorCode { E1126("Category combo {0} cannot combine more than {1} categories, but had: {2}"), E1127("Category {0} cannot have more than {1} options, but had: {2} "), E1128("Category combo {0} cannot have more than {1} combinations, but requires: {2}"), + E1129("OptionSet selection mode is not valid: `{0}`"), /* Org unit merge */ E1500("At least two source orgs unit must be specified"), diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/option/OptionSet.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/option/OptionSet.java index 91f469b9e66b..6c605f931a9a 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/option/OptionSet.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/option/OptionSet.java @@ -139,6 +139,17 @@ public List getOptionCodes() { .collect(Collectors.toList()); } + public Set getOptionUids() { + if (options != null) { + return options.stream() + .filter(Objects::nonNull) + .map(Option::getUid) + .collect(Collectors.toSet()); + } + + return Set.of(); + } + public Set getOptionCodesAsSet() { return options.stream() .filter(Objects::nonNull) diff --git a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/DimensionalObjectUtilsTest.java b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/DimensionalObjectUtilsTest.java index d68df61a942f..c4a124a31b32 100644 --- a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/DimensionalObjectUtilsTest.java +++ b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/DimensionalObjectUtilsTest.java @@ -177,15 +177,15 @@ void testGetDataElementOperandIdSchemeCodeMap() { @Test void testGetFirstSecondIdentifier() { assertEquals( - "A123456789A", DimensionalObjectUtils.getFirstIdentifer("A123456789A.P123456789A")); - assertNull(DimensionalObjectUtils.getFirstIdentifer("A123456789A")); + "A123456789A", DimensionalObjectUtils.getFirstIdentifier("A123456789A.P123456789A")); + assertNull(DimensionalObjectUtils.getFirstIdentifier("A123456789A")); } @Test void testGetSecondIdentifier() { assertEquals( - "P123456789A", DimensionalObjectUtils.getSecondIdentifer("A123456789A.P123456789A")); - assertNull(DimensionalObjectUtils.getSecondIdentifer("A123456789A")); + "P123456789A", DimensionalObjectUtils.getSecondIdentifier("A123456789A.P123456789A")); + assertNull(DimensionalObjectUtils.getSecondIdentifier("A123456789A")); } @Test diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java index c350db389e18..20553bc612b1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java @@ -33,6 +33,7 @@ import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.hisp.dhis.analytics.OrgUnitField.DEFAULT_ORG_UNIT_FIELD; import static org.hisp.dhis.analytics.TimeField.DEFAULT_TIME_FIELDS; +import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; import static org.hisp.dhis.common.DimensionType.CATEGORY; import static org.hisp.dhis.common.DimensionType.CATEGORY_OPTION_GROUP_SET; import static org.hisp.dhis.common.DimensionType.DATA_X; @@ -234,6 +235,9 @@ public class DataQueryParams { /** The aggregation type. */ protected AnalyticsAggregationType aggregationType; + /** The option set selection criteria. */ + protected OptionSetSelectionCriteria optionSetSelectionCriteria; + /** The measure criteria, which is measure filters and corresponding values. */ protected Map measureCriteria = new HashMap<>(); @@ -498,6 +502,7 @@ public T copyTo(T params) { params.dimensions = DimensionalObjectUtils.getCopies(this.dimensions); params.filters = DimensionalObjectUtils.getCopies(this.filters); params.aggregationType = this.aggregationType != null ? this.aggregationType.instance() : null; + params.optionSetSelectionCriteria = this.optionSetSelectionCriteria; params.measureCriteria = new HashMap<>(this.measureCriteria); params.preAggregateMeasureCriteria = new HashMap<>(this.preAggregateMeasureCriteria); params.skipMeta = this.skipMeta; @@ -591,6 +596,7 @@ protected QueryKey getQueryKey() { (k, v) -> key.add("preAggregateMeasureCriteria", (String.valueOf(k) + v))); return key.add("aggregationType", aggregationType) + .add("optionSetSelectionCriteria", optionSetSelectionCriteria) .add("skipMeta", skipMeta) .add("skipData", skipData) .add("skipHeaders", skipHeaders) @@ -746,6 +752,18 @@ public boolean hasOrganisationUnitGroupSets() { return !getDimensionsAndFilters(ORGANISATION_UNIT_GROUP_SET).isEmpty(); } + /** Indicates whether an option set selection criteria is present as param in this object. */ + public boolean hasOptionSetSelectionCriteria() { + return optionSetSelectionCriteria != null; + } + + /** Indicates whether option set selections are present as param ins this objct. */ + public boolean hasOptionSetSelections() { + return hasOptionSetSelectionCriteria() + && optionSetSelectionCriteria.getOptionSetSelections() != null + && !optionSetSelectionCriteria.getOptionSetSelections().isEmpty(); + } + /** * Returns the period type of the first period specified as filter, or null if there is no period * filter. @@ -861,6 +879,25 @@ public boolean isOutputFormat(OutputFormat format) { return this.outputFormat != null && this.outputFormat == format; } + /** + * Checks if there is an {@OptionSet} object inside data elements present in the current + * "dimensions" attribute of this class. + * + * @return boolean if found, false otherwise. + */ + public boolean hasOptionSetInDimensionItemsTypeDataElement() { + for (DimensionalObject d : dimensions) { + for (DimensionalItemObject it : d.getItems()) { + if (it.getDimensionItemType() == DATA_ELEMENT + && ((DataElement) it).getOptionSet() != null) { + return true; + } + } + } + + return false; + } + /** * Creates a mapping between the data periods, based on the data period type for this query, and * the aggregation periods for this query. @@ -1952,6 +1989,10 @@ public AnalyticsAggregationType getAggregationType() { return aggregationType; } + public OptionSetSelectionCriteria getOptionSetSelectionCriteria() { + return optionSetSelectionCriteria; + } + public Map getMeasureCriteria() { return measureCriteria; } @@ -2796,6 +2837,12 @@ public Builder withAggregationType(AnalyticsAggregationType aggregationType) { return this; } + public Builder withOptionSetSelectionCriteria( + OptionSetSelectionCriteria optionSetSelectionCriteria) { + this.params.optionSetSelectionCriteria = optionSetSelectionCriteria; + return this; + } + public Builder withSkipMeta(boolean skipMeta) { this.params.skipMeta = skipMeta; return this; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java index b5511f5ebad6..e8bafc0220e2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java @@ -27,6 +27,7 @@ */ package org.hisp.dhis.analytics.data; +import static java.util.Objects.requireNonNull; import static org.apache.commons.collections4.CollectionUtils.addIgnoreNull; import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; import static org.apache.commons.lang3.StringUtils.isNotEmpty; @@ -99,6 +100,8 @@ public class DefaultDataQueryService implements DataQueryService { private final AnalyticsSecurityManager securityManager; + private final OptionSetFacade optionSetFacade; + // ------------------------------------------------------------------------- // DataQueryService implementation // ------------------------------------------------------------------------- @@ -114,6 +117,8 @@ public DataQueryParams getFromRequest(DataQueryRequest request) { if (isNotEmpty(request.getDimension())) { params.addDimensions(getDimensionalObjects(request)); + params.withOptionSetSelectionCriteria( + optionSetFacade.getOptionSetSelectionCriteria(request.getDimension())); } if (isNotEmpty(request.getFilter())) { @@ -174,7 +179,7 @@ public DataQueryParams getFromRequest(DataQueryRequest request) { @Override @Transactional(readOnly = true) public DataQueryParams getFromAnalyticalObject(AnalyticalObject object) { - Objects.requireNonNull(object); + requireNonNull(object); DataQueryParams.Builder params = DataQueryParams.newBuilder(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java index 89935206684f..37bc25a10daf 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java @@ -28,6 +28,7 @@ package org.hisp.dhis.analytics.data; import static java.lang.String.join; +import static org.apache.commons.collections4.CollectionUtils.size; import static org.apache.commons.lang3.time.DateUtils.addYears; import static org.hisp.dhis.analytics.AggregationType.AVERAGE; import static org.hisp.dhis.analytics.AggregationType.COUNT; @@ -40,12 +41,16 @@ import static org.hisp.dhis.analytics.DataQueryParams.LEVEL_PREFIX; import static org.hisp.dhis.analytics.DataQueryParams.VALUE_ID; import static org.hisp.dhis.analytics.DataType.TEXT; +import static org.hisp.dhis.analytics.data.OptionSetFacade.addWhereClauseForOptions; +import static org.hisp.dhis.analytics.data.OptionSetFacade.getAggregatedOptionValueClause; +import static org.hisp.dhis.analytics.data.OptionSetFacade.getOptionSetSelectionMode; import static org.hisp.dhis.analytics.data.SubexpressionPeriodOffsetUtils.getParamsWithOffsetPeriods; import static org.hisp.dhis.analytics.util.AnalyticsUtils.throwIllegalQueryEx; import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; import static org.hisp.dhis.common.DimensionalObject.DIMENSION_SEP; import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; import static org.hisp.dhis.common.collection.CollectionUtils.concat; +import static org.hisp.dhis.system.util.SqlUtils.quote; import static org.hisp.dhis.util.DateUtils.toMediumDate; import static org.hisp.dhis.util.SqlExceptionUtils.ERR_MSG_SILENT_FALLBACK; import static org.hisp.dhis.util.SqlExceptionUtils.relationDoesNotExist; @@ -71,6 +76,7 @@ import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.DataType; import org.hisp.dhis.analytics.MeasureFilter; +import org.hisp.dhis.analytics.OptionSetSelectionMode; import org.hisp.dhis.analytics.QueryPlanner; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; import org.hisp.dhis.analytics.table.model.Partitions; @@ -343,6 +349,10 @@ private String getSelectClause(DataQueryParams params) { sql += getValueClause(params); + if (hasAggregation(params)) { + sql += getAggregatedOptionValueClause(params); + } + return sql; } @@ -355,7 +365,7 @@ private String getSelectClause(DataQueryParams params) { protected String getValueClause(DataQueryParams params) { String sql = ""; - if (params.isAggregation()) { + if (hasAggregation(params)) { sql += getAggregateValueColumn(params); } else { sql += params.getValueColumn(); @@ -364,6 +374,17 @@ protected String getValueClause(DataQueryParams params) { return sql + " as value "; } + private boolean hasAggregation(DataQueryParams params) { + // Analytics query is an item of sequential queries with one data element only. + if (size(params.getDataElements()) != 1) { + return params.isAggregation(); + } + + OptionSetSelectionMode mode = getOptionSetSelectionMode(params); + + return params.isAggregation() && mode == OptionSetSelectionMode.AGGREGATED; + } + /** * Returns an aggregate clause for the numeric value column. * @@ -371,10 +392,8 @@ protected String getValueClause(DataQueryParams params) { * @return a SQL numeric value column. */ protected String getAggregateValueColumn(DataQueryParams params) { - String sql = null; - + String sql; AnalyticsAggregationType aggType = params.getAggregationType(); - String valueColumn = params.getValueColumn(); if (aggType.isAggregationType(SUM) @@ -478,6 +497,8 @@ private void getWhereClauseDimensions( String items = sqlBuilder.singleQuotedCommaDelimited(getUids(dim.getItems())); sql.append(sqlHelper.whereAnd() + " " + col + " in (" + items + ") "); + + addWhereClauseForOptions(params, sqlHelper, sqlBuilder, sql, items); } } } @@ -640,13 +661,11 @@ private void getWhereClauseRestrictions( * @return a SQL group by clause. */ protected String getGroupByClause(DataQueryParams params) { - String sql = ""; - - if (params.isAggregation()) { - sql = "group by " + getCommaDelimitedQuotedDimensionColumns(params.getDimensions()) + " "; + if (hasAggregation(params)) { + return "group by " + getCommaDelimitedQuotedDimensionColumns(params.getDimensions()) + " "; } - return sql; + return ""; } /** @@ -957,7 +976,12 @@ private Map getKeyValueMap(DataQueryParams params, String sql, i if (params.isDataType(TEXT)) { String value = rowSet.getString(VALUE_ID); - map.put(key.toString(), value); + + if (params.hasOptionSetInDimensionItemsTypeDataElement()) { + map.put(key + DIMENSION_SEP + value, rowSet.getString("valuecount")); + } else { + map.put(key.toString(), value); + } } else // NUMERIC { Double value = rowSet.getDouble(VALUE_ID); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/OptionSetFacade.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/OptionSetFacade.java new file mode 100644 index 000000000000..51ddc5488a16 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/OptionSetFacade.java @@ -0,0 +1,340 @@ +/* + * Copyright (c) 2004-2025, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.data; + +import static org.apache.commons.collections4.CollectionUtils.isEmpty; +import static org.apache.commons.lang3.StringUtils.EMPTY; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.commons.lang3.StringUtils.substringBefore; +import static org.hisp.dhis.analytics.AnalyticsAggregationType.fromAggregationType; +import static org.hisp.dhis.analytics.OptionSetSelectionMode.AGGREGATED; +import static org.hisp.dhis.analytics.OptionSetSelectionMode.DISAGGREGATED; +import static org.hisp.dhis.common.DimensionalObject.DIMENSION_IDENTIFIER_SEP; +import static org.hisp.dhis.common.DimensionalObject.OPTION_SEP; +import static org.hisp.dhis.common.DimensionalObjectUtils.getDimensionItemsFromParam; +import static org.hisp.dhis.common.DimensionalObjectUtils.getFirstIdentifier; +import static org.hisp.dhis.common.DimensionalObjectUtils.getOptionsParam; +import static org.hisp.dhis.common.DimensionalObjectUtils.getSecondIdentifier; +import static org.hisp.dhis.common.DimensionalObjectUtils.getThirdIdentifier; +import static org.hisp.dhis.common.DimensionalObjectUtils.getValueFromDimensionParam; +import static org.hisp.dhis.common.collection.CollectionUtils.isNotEmpty; +import static org.hisp.dhis.util.ObjectUtils.firstNonNull; + +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import javax.annotation.Nonnull; +import lombok.RequiredArgsConstructor; +import org.hisp.dhis.analytics.AnalyticsAggregationType; +import org.hisp.dhis.analytics.DataQueryParams; +import org.hisp.dhis.analytics.OptionSetSelection; +import org.hisp.dhis.analytics.OptionSetSelectionCriteria; +import org.hisp.dhis.analytics.OptionSetSelectionMode; +import org.hisp.dhis.analytics.event.EventQueryParams; +import org.hisp.dhis.common.DimensionalItemObject; +import org.hisp.dhis.common.DimensionalObjectUtils; +import org.hisp.dhis.common.IdentifiableObjectManager; +import org.hisp.dhis.common.QueryItem; +import org.hisp.dhis.commons.util.SqlHelper; +import org.hisp.dhis.dataelement.DataElement; +import org.hisp.dhis.db.sql.SqlBuilder; +import org.hisp.dhis.option.Option; +import org.hisp.dhis.option.OptionSet; +import org.hisp.dhis.program.Program; +import org.springframework.stereotype.Component; + +@Component +@RequiredArgsConstructor +public class OptionSetFacade { + private final IdentifiableObjectManager idObjectManager; + + /** + * Add queries to the given list of {@link EventQueryParams} based on the params {@link + * EventQueryParams}. It respects rules related to AGGREGATED option set ({@link AGGREGATED}). + * + * @param params the {@link EventQueryParams}. + * @param queries the list of {@link EventQueryParams}. + * @param item the {@link QueryItem}. + */ + public void handleAggregatedOptionSet( + EventQueryParams params, List queries, QueryItem item) { + + OptionSetSelectionCriteria optionSetSelectionCriteria = params.getOptionSetSelectionCriteria(); + OptionSetSelectionMode optionSetSelectionMode = + optionSetSelectionCriteria.getOptionSetSelections().entrySet().stream() + .toList() + .get(0) + .getValue() + .getOptionSetSelectionMode(); + + if (optionSetSelectionMode == AGGREGATED) { + AnalyticsAggregationType aggregationType = + firstNonNull(params.getAggregationType(), fromAggregationType(item.getAggregationType())); + + EventQueryParams.Builder query = + new EventQueryParams.Builder(params) + .removeItems() + .removeItemProgramIndicators() + .withValue(item.getItem()) + .withAggregationType(aggregationType); + + if (item.hasProgram()) { + query.withProgram(item.getProgram()); + } + + queries.add(query.build()); + } + } + + /** + * Add queries to the given list of {@link EventQueryParams} based on the params {@link + * EventQueryParams}. It respects rules related to DISAGGREGATED option set ({@link + * OptionSetSelectionMode.DISAGGREGATED}). + * + * @param params the {@link EventQueryParams}. + * @param queries the list of {@link EventQueryParams}. + */ + public void handleDisaggregatedOptionSet( + EventQueryParams params, List queries) { + if (params.hasOptionSetSelections()) { + for (Map.Entry entry : + params.getOptionSetSelectionCriteria().getOptionSetSelections().entrySet()) { + if (entry.getValue() != null + && entry.getValue().getOptionSetSelectionMode() == DISAGGREGATED) { + for (String option : entry.getValue().getOptions()) { + Set optionSet = new LinkedHashSet<>(); + optionSet.add(option); + + OptionSetSelection optionSetSelection = + new OptionSetSelection( + entry.getValue().getQualifiedUid(), + entry.getValue().getOptionSetUid(), + optionSet, + DISAGGREGATED); + + Map optionSetSelectionMap = new HashMap<>(); + optionSetSelectionMap.put(entry.getKey(), optionSetSelection); + + OptionSetSelectionCriteria optionSetSelectionCriteria = + new OptionSetSelectionCriteria(optionSetSelectionMap); + + Program program = getProgram(params.getItems(), entry.getValue().getQualifiedUid()); + + EventQueryParams query = + new EventQueryParams.Builder(params) + .withAggregationType(params.getAggregationType()) + .withOptionSetSelectionCriteria(optionSetSelectionCriteria) + .withProgram(program) + .build(); + + queries.add(query); + } + } + } + } + } + + /** + * Creates a {@link OptionSetSelectionCriteria} object based on the given collection of + * dimensions. + * + * @param dimensions the collection of dimensions. + * @return the {@link OptionSetSelectionCriteria} object. + */ + public OptionSetSelectionCriteria getOptionSetSelectionCriteria(Set dimensions) { + Map optionSetSelections = new HashMap<>(); + Set splitDimensions = new LinkedHashSet<>(); + + for (String dimension : dimensions) { + if (dimension.startsWith("dx:")) { + splitDimensions.addAll(getDimensionItemsFromParam(dimension)); + } + } + + for (String dimension : splitDimensions) { + String dimValue = getValueFromDimensionParam(dimension); + + if (hasOptionSet(dimValue)) { + OptionSetSelectionMode mode = DimensionalObjectUtils.getOptionSetSelectionMode(dimValue); + String dimIdentifier = getDimensionIdentifier(dimValue); + String optionsParam = getOptionsParam(dimValue); + Set options = extractOptions(optionsParam); + + if (mode == DISAGGREGATED && isEmpty(options)) { + DataElement dataElement = + this.idObjectManager.get(DataElement.class, substringBefore(dimIdentifier, ".")); + if (dataElement.getOptionSet() != null) { + options = dataElement.getOptionSet().getOptionUids(); + } + } + + OptionSetSelection optionSetSelection = + new OptionSetSelection(dimValue, dimIdentifier, options, mode); + + optionSetSelections.put(dimIdentifier, optionSetSelection); + } + } + + return new OptionSetSelectionCriteria(optionSetSelections); + } + + /** + * Returns a "where" clause for options selected, if any. + * + * @param params the {@link DataQueryParams}. + * @param sqlHelper the {@link SqlHelper}. + * @param sqlBuilder the {@link SqlBuilder}. + * @param sql the {@link StringBuilder}. + */ + static void addWhereClauseForOptions( + DataQueryParams params, + SqlHelper sqlHelper, + SqlBuilder sqlBuilder, + StringBuilder sql, + String items) { + if (params.hasOptionSetSelections()) { + params + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .forEach( + (key, value) -> { + Set options = value.getOptions(); + if (isNotEmpty(options) + && items.contains(value.getOptionSetUid())) { // TODO: MAIKEL revisit + // OptionSetSelectionCriteria thing. + sql.append(" ") + .append(sqlHelper.whereAnd()) + .append(" ") + .append(sqlBuilder.quote("optionvalueuid")) + .append(" in ('") + .append(String.join("','", options)) + .append("') "); + } + }); + } + } + + static OptionSetSelectionMode getOptionSetSelectionMode(@Nonnull DataQueryParams params) { + Optional optionSetSelectionMode = Optional.empty(); + + for (DimensionalItemObject de : params.getDataElements()) { + if (params.hasOptionSetSelections() && ((DataElement) de).getOptionSet() != null) { + OptionSetSelectionMode setSelectionMode = + params + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .get( + de.getUid() + + DIMENSION_IDENTIFIER_SEP + + ((DataElement) de).getOptionSet().getUid()) + .getOptionSetSelectionMode(); + optionSetSelectionMode = Optional.of(setSelectionMode); + break; + } + } + + return optionSetSelectionMode.orElse(AGGREGATED); + } + + static String getAggregatedOptionValueClause(DataQueryParams params) { + if (params.hasOptionSetInDimensionItemsTypeDataElement()) { + return ", count(" + params.getValueColumn() + ") as valuecount "; + } + + return EMPTY; + } + + /** + * Extracts the dimension uid based on the given argument and internal rules, depending on the + * composition of the value. + * + * @param composedDimension ie: WSGAb5XwJ3Y.QFX1FLWBwtq.R3ShQczKnI9[l8S7SjnQ58G;x7H1HjJ0R64] + * @return the respective dimension uid. + */ + private String getDimensionIdentifier(String composedDimension) { + String dimIdentifier = getThirdIdentifier(composedDimension); + + if (dimIdentifier == null) { + dimIdentifier = + getFirstIdentifier(composedDimension) + + DIMENSION_IDENTIFIER_SEP + + getSecondIdentifier(composedDimension); + } else { + dimIdentifier = + getSecondIdentifier(composedDimension) + DIMENSION_IDENTIFIER_SEP + dimIdentifier; + } + + return dimIdentifier; + } + + /** + * Extracts the options uids specified in the URL param, if any. + * + * @param options the URL param options. + * @return the options uids found, or empty. + */ + private Set extractOptions(String options) { + if (isNotBlank(options)) { + Set optionSet = new LinkedHashSet<>(); + + for (String uid : options.split(OPTION_SEP)) { + Option option = this.idObjectManager.get(Option.class, uid); + if (option != null) { + optionSet.add(option.getUid()); + } + } + + return optionSet; + } + + return Set.of(); + } + + private boolean hasOptionSet(String param) { + String uid = getThirdIdentifier(param); + + if (uid == null) { + uid = getSecondIdentifier(param); + } + + return uid != null && idObjectManager.exists(OptionSet.class, uid); + } + + private Program getProgram(List items, String dimension) { + for (QueryItem item : items) { + if (item.hasProgram() && dimension.startsWith(item.getProgram().getUid())) { + return item.getProgram(); + } + } + + return null; + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java index 998fcb80d55c..ec9ca0eb36f2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java @@ -426,6 +426,7 @@ public void addProgramDataElementAttributeIndicatorValues(DataQueryParams params EventQueryParams eventQueryParams = new EventQueryParams.Builder(fromDataQueryParams(dataSourceParams)) .withSkipMeta(true) + .withOptionSetSelectionCriteria(params.getOptionSetSelectionCriteria()) .build(); Grid eventGrid = eventAggregatedService.getAggregatedData(eventQueryParams); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java index 4d6f0683f691..4e3ba61e2bc1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java @@ -59,6 +59,7 @@ import org.hisp.dhis.analytics.AnalyticsAggregationType; import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.EventOutputType; +import org.hisp.dhis.analytics.OptionSetSelectionCriteria; import org.hisp.dhis.analytics.OrgUnitField; import org.hisp.dhis.analytics.QueryKey; import org.hisp.dhis.analytics.QueryParamsBuilder; @@ -305,6 +306,7 @@ protected EventQueryParams instance() { params.rowContext = this.rowContext; params.multipleQueries = this.multipleQueries; params.userOrganisationUnitsCriteria = this.userOrganisationUnitsCriteria; + params.optionSetSelectionCriteria = this.optionSetSelectionCriteria; return params; } @@ -318,6 +320,7 @@ public static EventQueryParams fromDataQueryParams(DataQueryParams dataQueryPara for (DimensionalItemObject object : dataQueryParams.getProgramDataElements()) { ProgramDataElementDimensionItem element = (ProgramDataElementDimensionItem) object; DataElement dataElement = element.getDataElement(); + QueryItem item = new QueryItem( dataElement, @@ -1110,6 +1113,14 @@ public Builder removeItemProgramIndicators() { return this; } + public Builder removeOptionSetSelection() { + if (this.params.hasOptionSetSelections()) { + this.params.optionSetSelectionCriteria.getOptionSetSelections().clear(); + } + + return this; + } + public Builder withValue(DimensionalItemObject value) { this.params.value = value; return this; @@ -1360,5 +1371,11 @@ public Builder withMultipleQueries(boolean multipleQueries) { this.params.multipleQueries = multipleQueries; return this; } + + public Builder withOptionSetSelectionCriteria( + OptionSetSelectionCriteria optionSetSelectionCriteria) { + this.params.optionSetSelectionCriteria = optionSetSelectionCriteria; + return this; + } } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java index 6f5ccba6f53c..7b0f2795c9d0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java @@ -41,6 +41,14 @@ public interface EventQueryPlanner { */ List planAggregateQuery(EventQueryParams params); + /** + * Plans the given parameters and returns a list of parameters. + * + * @param params the event query parameters. + * @return a list of {@link EventQueryParams}. + */ + List planQuery(EventQueryParams params); + /** * Plans the given parameters and returns a list of parameters. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 47952b35a9d4..6e7cdf39cad2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -43,9 +43,11 @@ import static org.hisp.dhis.analytics.AnalyticsConstants.DATE_PERIOD_STRUCT_ALIAS; import static org.hisp.dhis.analytics.DataQueryParams.NUMERATOR_DENOMINATOR_PROPERTIES_COUNT; import static org.hisp.dhis.analytics.DataType.NUMERIC; +import static org.hisp.dhis.analytics.OptionSetSelectionMode.AGGREGATED; import static org.hisp.dhis.analytics.QueryKey.NV; import static org.hisp.dhis.analytics.SortOrder.ASC; import static org.hisp.dhis.analytics.SortOrder.DESC; +import static org.hisp.dhis.analytics.data.QueryPlannerUtils.getAggregationType; import static org.hisp.dhis.analytics.event.data.EnrollmentQueryHelper.getHeaderColumns; import static org.hisp.dhis.analytics.event.data.EnrollmentQueryHelper.getOrgUnitLevelColumns; import static org.hisp.dhis.analytics.event.data.EnrollmentQueryHelper.getPeriodColumns; @@ -56,6 +58,7 @@ import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT; import static org.hisp.dhis.common.DimensionItemType.PROGRAM_INDICATOR; +import static org.hisp.dhis.common.DimensionalObject.DIMENSION_IDENTIFIER_SEP; import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; import static org.hisp.dhis.common.DimensionalObjectUtils.COMPOSITE_DIM_OBJECT_PLAIN_SEP; import static org.hisp.dhis.common.QueryOperator.IN; @@ -89,6 +92,7 @@ import org.apache.commons.lang3.time.DateUtils; import org.hisp.dhis.analytics.AggregationType; import org.hisp.dhis.analytics.EventOutputType; +import org.hisp.dhis.analytics.OptionSetSelectionMode; import org.hisp.dhis.analytics.SortOrder; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; @@ -111,6 +115,7 @@ import org.hisp.dhis.commons.collection.ListUtils; import org.hisp.dhis.commons.util.SqlHelper; import org.hisp.dhis.commons.util.TextUtils; +import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.feedback.ErrorCode; import org.hisp.dhis.option.Option; @@ -591,6 +596,11 @@ public Grid getAggregatedEventData(EventQueryParams params, Grid grid, int maxLi private String getGroupByClause(EventQueryParams params) { String sql = ""; + AggregationType aggregationType = getAggregationType(params); + if (aggregationType == NONE && !params.hasOptionSetSelections()) { + return sql; + } + if (params.isAggregation()) { List selectColumnNames = getGroupByColumnNames(params, true); @@ -697,7 +707,7 @@ protected String getAggregateClause(EventQueryParams params) { EventOutputType outputType = params.getOutputType(); - AggregationType aggregationType = params.getAggregationTypeFallback().getAggregationType(); + AggregationType aggregationType = getAggregationType(params); String function = (aggregationType == NONE || aggregationType == CUSTOM) ? "" : aggregationType.getValue(); @@ -747,6 +757,28 @@ protected String getAggregateClause(EventQueryParams params) { } } + private AggregationType getAggregationType(EventQueryParams params) { + if (params.getValue() instanceof DataElement dataElement + && dataElement.hasOptionSet() + && params.hasOptionSetSelections()) { + String key = + dataElement.getUid() + DIMENSION_IDENTIFIER_SEP + dataElement.getOptionSet().getUid(); + + OptionSetSelectionMode mode = + params + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .get(key) + .getOptionSetSelectionMode(); + + if (mode != AGGREGATED) { + return NONE; + } + } + + return params.getAggregationTypeFallback().getAggregationType(); + } + /** * Creates a coordinate base column "selector" for the given item name. The item is expected to be * of type Coordinate. diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java index 2ee4a0fb3d9a..f01bdf3fd860 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java @@ -27,6 +27,7 @@ */ package org.hisp.dhis.analytics.event.data; +import static org.apache.commons.lang3.ObjectUtils.firstNonNull; import static org.hisp.dhis.analytics.AnalyticsAggregationType.fromAggregationType; import com.google.common.collect.ImmutableList; @@ -40,6 +41,7 @@ import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.OrgUnitField; import org.hisp.dhis.analytics.QueryPlanner; +import org.hisp.dhis.analytics.data.OptionSetFacade; import org.hisp.dhis.analytics.data.QueryPlannerUtils; import org.hisp.dhis.analytics.event.EventQueryParams; import org.hisp.dhis.analytics.event.EventQueryPlanner; @@ -51,7 +53,6 @@ import org.hisp.dhis.common.QueryItem; import org.hisp.dhis.period.Period; import org.hisp.dhis.program.ProgramIndicator; -import org.hisp.dhis.util.ObjectUtils; import org.springframework.stereotype.Service; /** @@ -62,6 +63,8 @@ public class DefaultEventQueryPlanner implements EventQueryPlanner { private final QueryPlanner queryPlanner; + private final OptionSetFacade optionSetQueryPlanner; + private final PartitionManager partitionManager; // ------------------------------------------------------------------------- @@ -90,6 +93,13 @@ public List planAggregateQuery(EventQueryParams params) { return withTableNameAndPartitions(queries); } + @Override + public List planQuery(EventQueryParams params) { + List queries = Lists.newArrayList(params); + + return withTableNameAndPartitions(queries); + } + @Override public EventQueryParams planEventQuery(EventQueryParams params) { return withTableNameAndPartitions(params); @@ -204,22 +214,26 @@ private List groupByQueryItems(EventQueryParams params) { if (params.isAggregateData()) { for (QueryItem item : params.getItemsAndItemFilters()) { - AnalyticsAggregationType aggregationType = - ObjectUtils.firstNonNull( - params.getAggregationType(), fromAggregationType(item.getAggregationType())); - - EventQueryParams.Builder query = - new EventQueryParams.Builder(params) - .removeItems() - .removeItemProgramIndicators() - .withValue(item.getItem()) - .withAggregationType(aggregationType); - - if (item.hasProgram()) { - query.withProgram(item.getProgram()); + if (params.hasOptionSetSelections()) { + optionSetQueryPlanner.handleAggregatedOptionSet(params, queries, item); + } else { + AnalyticsAggregationType aggregationType = + firstNonNull( + params.getAggregationType(), fromAggregationType(item.getAggregationType())); + + EventQueryParams.Builder query = + new EventQueryParams.Builder(params) + .removeItems() + .removeItemProgramIndicators() + .withValue(item.getItem()) + .withAggregationType(aggregationType); + + if (item.hasProgram()) { + query.withProgram(item.getProgram()); + } + + queries.add(query.build()); } - - queries.add(query.build()); } for (ProgramIndicator programIndicator : params.getItemProgramIndicators()) { @@ -236,6 +250,8 @@ private List groupByQueryItems(EventQueryParams params) { queries.add(query); } + + optionSetQueryPlanner.handleDisaggregatedOptionSet(params, queries); } else if (params.isCollapseDataDimensions() && !params.getItems().isEmpty()) { for (QueryItem item : params.getItems()) { EventQueryParams.Builder query = diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/EventQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/EventQueryService.java index 09fb1b9c0578..b58d79972f5d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/EventQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/EventQueryService.java @@ -70,6 +70,7 @@ import static org.hisp.dhis.feedback.ErrorCode.E7218; import java.util.List; +import java.util.Optional; import lombok.RequiredArgsConstructor; import org.hisp.dhis.analytics.AnalyticsSecurityManager; import org.hisp.dhis.analytics.Rectangle; @@ -82,6 +83,7 @@ import org.hisp.dhis.common.DimensionItemKeywords.Keyword; import org.hisp.dhis.common.Grid; import org.hisp.dhis.common.GridHeader; +import org.hisp.dhis.common.QueryItem; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.system.grid.ListGrid; import org.hisp.dhis.util.Timer; @@ -126,6 +128,9 @@ public Grid getEvents(EventQueryParams params) { params = new EventQueryParams.Builder(params).withStartEndDatesForPeriods().build(); + // Set program if null. + params = getEventQueryParamsWithProgram(params); + // Headers Grid grid = createGridWithHeaders(params); @@ -153,6 +158,20 @@ public Grid getEvents(EventQueryParams params) { return grid; } + private static EventQueryParams getEventQueryParamsWithProgram(EventQueryParams params) { + if (!params.hasProgram()) { + Optional itemWithProgram = + params.getItems().stream().filter(QueryItem::hasProgram).findFirst(); + if (itemWithProgram.isPresent()) { + EventQueryParams.Builder builder = + new EventQueryParams.Builder(params).withProgram(itemWithProgram.get().getProgram()); + params = builder.build(); + } + } + + return params; + } + /** * Returns a list of event clusters matching the given query. * @@ -269,7 +288,7 @@ private Grid createGridWithHeaders(EventQueryParams params) { false, true)); - if (params.getProgram().isRegistration()) { + if (params.getProgram() != null && params.getProgram().isRegistration()) { grid.addHeader( new GridHeader( ENROLLMENT_DATE.getItem(), diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java index 23da08181fd1..80709de1ecf7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java @@ -28,6 +28,8 @@ package org.hisp.dhis.analytics.event.data; import static java.util.stream.Collectors.joining; +import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; +import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.apache.commons.lang3.StringUtils.isNotEmpty; import static org.apache.commons.lang3.time.DateUtils.addYears; import static org.hisp.dhis.analytics.AnalyticsConstants.ANALYTICS_TBL_ALIAS; @@ -49,6 +51,7 @@ import com.google.common.collect.Lists; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Map; @@ -58,6 +61,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.math3.util.Precision; import org.hisp.dhis.analytics.AggregationType; +import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.OrgUnitField; import org.hisp.dhis.analytics.Rectangle; import org.hisp.dhis.analytics.TimeField; @@ -549,6 +553,8 @@ protected String getWhereClause(EventQueryParams params) { sql += getQueryItemsAndFiltersWhereClause(params, hlp); + sql += getWhereClauseOptions(params, hlp); + // --------------------------------------------------------------------- // Filter expression // --------------------------------------------------------------------- @@ -658,6 +664,35 @@ protected String getWhereClause(EventQueryParams params) { return sql; } + private String getWhereClauseOptions(DataQueryParams params, SqlHelper sqlHelper) { + if (params.hasOptionSetSelections()) { + StringBuilder sql = new StringBuilder(); + + params + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .forEach( + (key, value) -> { + List uids = Arrays.stream(key.split("\\.")).toList(); + Set options = value.getOptions(); + + if (!uids.isEmpty() && isNotEmpty(options)) { + sql.append(" ") + .append(sqlHelper.whereAnd()) + .append(" ") + .append(quote(uids.get(0) + ".optionvalueuid")) + .append(" in ('") + .append(String.join("','", options)) + .append("') "); + } + }); + + return sql.toString(); + } + + return EMPTY; + } + /** Generates a sub query which provides a filter by organisation descendant level. */ private String getOrgUnitDescendantsClause( OrgUnitField orgUnitField, List dimensionOrFilterItems) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java index 31d587e51392..7f79311cb32a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java @@ -125,6 +125,18 @@ public class JdbcAnalyticsTableManager extends AbstractJdbcTableManager { .selectExpression("acs.categoryoptioncombouid as ao") .indexColumns(List.of("dx", "ao")) .build(), + AnalyticsTableColumn.builder() + .name("optionsetuid") + .dataType(CHARACTER_11) + .nullable(NULL) + .selectExpression("deo.optionsetuid as optionsetuid") + .build(), + AnalyticsTableColumn.builder() + .name("optionvalueuid") + .dataType(CHARACTER_11) + .nullable(NULL) + .selectExpression("deo.optionvalueuid as optionvalueuid") + .build(), AnalyticsTableColumn.builder() .name("pestartdate") .dataType(DATE) @@ -366,7 +378,8 @@ private void populateTable( inner join analytics_rs_categorystructure dcs on dv.categoryoptioncomboid=dcs.categoryoptioncomboid \ inner join analytics_rs_categorystructure acs on dv.attributeoptioncomboid=acs.categoryoptioncomboid \ inner join analytics_rs_categoryoptioncomboname aon on dv.attributeoptioncomboid=aon.categoryoptioncomboid \ - inner join analytics_rs_categoryoptioncomboname con on dv.categoryoptioncomboid=con.categoryoptioncomboid\s""", + inner join analytics_rs_categoryoptioncomboname con on dv.categoryoptioncomboid=con.categoryoptioncomboid \ + left outer join analytics_rs_dataelementoption deo on dv.dataelementid = deo.dataelementid and dv.value = deo.optionvaluecode \s""", Map.of( "approvalSelectExpression", approvalSelectExpression, "valueExpression", valueExpression, diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 6fa5118509ca..fcb4035f5f82 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.analytics.table; +import static java.lang.String.join; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.hisp.dhis.analytics.table.model.Skip.SKIP; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; @@ -39,6 +41,7 @@ import static org.hisp.dhis.db.model.DataType.INTEGER; import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; +import static org.hisp.dhis.system.util.SqlUtils.singleQuote; import static org.hisp.dhis.util.DateUtils.toLongDate; import static org.hisp.dhis.util.DateUtils.toMediumDate; @@ -50,7 +53,9 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -362,7 +367,7 @@ and ev.status in (${exportableEventStatues}) \ "programId", String.valueOf(program.getId()), "firstDataYear", String.valueOf(firstDataYear), "latestDataYear", String.valueOf(latestDataYear), - "exportableEventStatues", String.join(",", EXPORTABLE_EVENT_STATUSES))); + "exportableEventStatues", join(",", EXPORTABLE_EVENT_STATUSES))); populateTableInternal(partition, fromClause); } @@ -462,11 +467,21 @@ private List getAttributeCategoryColumns(Program program) */ private List getDataElementColumns(Program program) { List columns = new ArrayList<>(); + Set dataElements = + program.getAnalyticsDataElements().stream().filter(Objects::nonNull).collect(toSet()); + columns.addAll( - program.getAnalyticsDataElements().stream() + dataElements.stream() .map(de -> getColumnForDataElement(de, false)) .flatMap(Collection::stream) .toList()); + columns.addAll( + dataElements.stream() + .filter(DataElement::hasOptionSet) + .map(this::getColumnFromDataElementOptionSet) + .flatMap(Collection::stream) + .toList()); + columns.addAll( program.getAnalyticsDataElementsWithLegendSet().stream() .map(de -> getColumnForDataElement(de, true)) @@ -788,4 +803,121 @@ private final String getNumericClause(String value) { private List getYearsForPartitionTable(List dataYears) { return ListUtils.mutableCopy(!dataYears.isEmpty() ? dataYears : List.of(Year.now().getValue())); } + + private List getColumnFromDataElementOptionSet(DataElement dataElement) { + List columns = new ArrayList<>(); + + if (!dataElement.hasOptionSet()) { + return columns; + } + + String dataClause = getDataClause(dataElement.getUid(), dataElement.getValueType()); + String columnName = "eventdatavalues #>> '{" + dataElement.getUid() + ", value}'"; + String select = getSelectClause(dataElement.getValueType(), columnName); + String sql = selectOptionValueCodeForInsert(dataElement, select, dataClause); + + columns.add( + AnalyticsTableColumn.builder() + .name(dataElement.getUid() + ".optionvalueuid") + .dataType(DataType.VARCHAR_255) + .selectExpression(sql) + .skipIndex(Skip.INCLUDE) + .build()); + + return columns; + } + + private String getDataClause(String uid, ValueType valueType) { + if (valueType.isNumeric() || valueType.isDate()) { + String regex = valueType.isNumeric() ? NUMERIC_LENIENT_REGEXP : DATE_REGEXP; + + return replace( + " and eventdatavalues #>> '{${uid},value}' ~* '${regex}'", + Map.of("uid", uid, "regex", regex)); + } + + return ""; + } + + private String selectOptionValueCodeForInsert( + DataElement dataElement, String fromType, String dataClause) { + String innerSql = + replaceQualify( + """ + (select ${fromType} from ${event} \ + where eventid=ev.eventid ${dataClause})${closingParentheses}""", + Map.of( + "fromType", + fromType, + "dataClause", + dataClause, + "closingParentheses", + getClosingParentheses(fromType), + "dataElementUid", + quote(dataElement.getUid()))); + + return replaceQualify( + """ + (select optionvalueuid \ + from analytics_rs_dataelementoption \ + where dataelementuid = ${dataElementUid} \ + and optionvaluecode = ${selectForInsert}::varchar) as ${alias}""", + Map.of( + "dataElementUid", + singleQuote(dataElement.getUid()), + "selectForInsert", + innerSql, + "alias", + quote(dataElement.getUid() + ".optionvalueuid"))); + } + + private String getClosingParentheses(String str) { + if (StringUtils.isEmpty(str)) { + return EMPTY; + } + + int open = 0; + + for (int i = 0; i < str.length(); i++) { + if (str.charAt(i) == '(') { + open++; + } else if ((str.charAt(i) == ')') && open >= 1) { + open--; + } + } + + return StringUtils.repeat(")", open); + } + + /** + * Returns the select clause, potentially with a cast statement, based on the given value type. + * + * @param valueType the value type to represent as database column type. + */ + private String getSelectClause(ValueType valueType, String columnName) { + String doubleType = sqlBuilder.dataTypeDouble(); + if (valueType.isDecimal()) { + return "cast(" + columnName + " as " + doubleType + ")"; + } else if (valueType.isInteger()) { + return "cast(" + columnName + " as bigint)"; + } else if (valueType.isBoolean()) { + return "case when " + + columnName + + " = 'true' then 1 when " + + columnName + + " = 'false' then 0 else null end"; + } else if (valueType.isDate()) { + return "cast(" + columnName + " as timestamp)"; + } else if (valueType.isGeo() && isGeospatialSupport()) { + return "ST_GeomFromGeoJSON('{\"type\":\"Point\", \"coordinates\":' || (" + + columnName + + ") || ', \"crs\":{\"type\":\"name\", \"properties\":{\"name\":\"EPSG:4326\"}}}')"; + } else if (valueType.isOrganisationUnit()) { + return replaceQualify( + "ou.uid from ${organisationunit} ou where ou.uid = (select ${columnName}", + Map.of("columnName", columnName)); + } else { + return columnName; + } + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java index b61b7a82bbaa..1d563cf13c26 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java @@ -31,6 +31,7 @@ import static org.hisp.dhis.common.DimensionalObject.ATTRIBUTEOPTIONCOMBO_DIM_ID; import static org.hisp.dhis.common.DimensionalObject.CATEGORYOPTIONCOMBO_DIM_ID; import static org.hisp.dhis.common.DimensionalObject.DATA_X_DIM_ID; +import static org.hisp.dhis.common.DimensionalObject.DIMENSION_IDENTIFIER_SEP; import static org.hisp.dhis.common.DimensionalObject.DIMENSION_SEP; import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; import static org.hisp.dhis.common.DimensionalObject.PERIOD_DIM_ID; @@ -56,6 +57,7 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -105,11 +107,13 @@ import org.hisp.dhis.feedback.ErrorMessage; import org.hisp.dhis.hibernate.HibernateProxyUtils; import org.hisp.dhis.indicator.Indicator; +import org.hisp.dhis.option.OptionSet; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.period.FinancialPeriodType; import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodType; import org.hisp.dhis.program.Program; +import org.hisp.dhis.program.ProgramDataElementDimensionItem; import org.hisp.dhis.program.ProgramIndicator; import org.hisp.dhis.program.ProgramStage; import org.hisp.dhis.system.grid.ListGrid; @@ -644,8 +648,8 @@ public static void handleGridForDataValueSet(DataQueryParams params, Grid grid) coc = dataItem.getAggregateExportCategoryOptionCombo(); aoc = dataItem.getAggregateExportAttributeOptionCombo(); } else if (DataElementOperand.class.isAssignableFrom(item.getClass())) { - row.set(dxInx, DimensionalObjectUtils.getFirstIdentifer(dx)); - coc = DimensionalObjectUtils.getSecondIdentifer(dx); + row.set(dxInx, DimensionalObjectUtils.getFirstIdentifier(dx)); + coc = DimensionalObjectUtils.getSecondIdentifier(dx); } cocCol.add(coc); @@ -825,6 +829,17 @@ public static Map getDimensionMetadataItemMap( coc.getDisplayProperty(params.getDisplayProperty()), includeMetadataDetails ? coc : null)); } + + addOptionSetToMap(dataElement.getOptionSet(), map, dataElement, includeMetadataDetails); + } + if (DimensionItemType.PROGRAM_DATA_ELEMENT == item.getDimensionItemType() + && item instanceof ProgramDataElementDimensionItem programDataElement) { + + addOptionSetToMap( + programDataElement.getOptionSet(), + map, + programDataElement.getDataElement(), + includeMetadataDetails); } } @@ -889,6 +904,30 @@ public static Map getDimensionMetadataItemMap( return map; } + /** + * Adds the given {@link OptionSet} data into the given map, respecting the internal business + * rules. + * + * @param optionSet the {@link OptionSet} to add. + * @param map the source map where to add the given {@link OptionSet}. + * @param dataElement the {@link DataElement} associated with the {@link OptionSet}. + * @param includeDetails include {@link OptionSet} details or not. + */ + private static void addOptionSetToMap( + OptionSet optionSet, + Map map, + DataElement dataElement, + boolean includeDetails) { + if (optionSet != null) { + map.put( + dataElement.getUid() + DIMENSION_IDENTIFIER_SEP + optionSet.getUid(), + includeDetails + ? new MetadataItem( + optionSet.getName(), optionSet, new LinkedHashSet<>(optionSet.getOptions())) + : new MetadataItem(optionSet.getName())); + } + } + /** * Returns a mapping between the category option combo identifiers and names for the given query. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java index 04ba319c9e30..d032ba88003f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java @@ -61,6 +61,7 @@ import org.hisp.dhis.resourcetable.table.DataApprovalMinLevelResourceTable; import org.hisp.dhis.resourcetable.table.DataApprovalRemapLevelResourceTable; import org.hisp.dhis.resourcetable.table.DataElementGroupSetResourceTable; +import org.hisp.dhis.resourcetable.table.DataElementOptionResourceTable; import org.hisp.dhis.resourcetable.table.DataElementResourceTable; import org.hisp.dhis.resourcetable.table.DataSetOrganisationUnitCategoryResourceTable; import org.hisp.dhis.resourcetable.table.DataSetResourceTable; @@ -162,7 +163,8 @@ private final List getResourceTables() { new DataElementResourceTable(logged, idObjectManager.getAllNoAcl(DataElement.class)), new DatePeriodResourceTable(logged, getAndValidateAvailableDataYears()), new PeriodResourceTable(logged, periodService.getAllPeriods()), - new CategoryOptionComboResourceTable(logged)); + new CategoryOptionComboResourceTable(logged), + new DataElementOptionResourceTable(logged)); } /** diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DataElementOptionResourceTable.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DataElementOptionResourceTable.java new file mode 100644 index 000000000000..66d7ccdb5fd4 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DataElementOptionResourceTable.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.resourcetable.table; + +import static org.hisp.dhis.commons.util.TextUtils.replace; +import static org.hisp.dhis.db.model.Table.toStaging; +import static org.hisp.dhis.system.util.SqlUtils.appendRandom; + +import java.util.List; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import org.hisp.dhis.db.model.Column; +import org.hisp.dhis.db.model.DataType; +import org.hisp.dhis.db.model.Index; +import org.hisp.dhis.db.model.Logged; +import org.hisp.dhis.db.model.Table; +import org.hisp.dhis.db.model.constraint.Nullable; +import org.hisp.dhis.resourcetable.ResourceTable; +import org.hisp.dhis.resourcetable.ResourceTableType; + +@RequiredArgsConstructor +public class DataElementOptionResourceTable implements ResourceTable { + public static final String TABLE_NAME = "analytics_rs_dataelementoption"; + + private final Logged logged; + + @Override + public Table getTable() { + return new Table(toStaging(TABLE_NAME), getColumns(), getPrimaryKey(), logged); + } + + @Override + public Table getMainTable() { + return new Table(TABLE_NAME, getColumns(), getPrimaryKey(), logged); + } + + private List getColumns() { + return List.of( + new Column("dataelementid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("optionsetid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("optionvalueid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("dataelementuid", DataType.CHARACTER_11, Nullable.NOT_NULL), + new Column("optionsetuid", DataType.CHARACTER_11, Nullable.NOT_NULL), + new Column("optionvalueuid", DataType.CHARACTER_11, Nullable.NOT_NULL), + new Column("optionvaluecode", DataType.VARCHAR_255, Nullable.NOT_NULL)); + } + + private List getPrimaryKey() { + return List.of("dataelementid", "optionsetid", "optionvalueid"); + } + + @Override + public List getIndexes() { + return List.of( + Index.builder() + .name(appendRandom("in_optionsetoptionvalue")) + .tableName(toStaging(TABLE_NAME)) + .columns(List.of("dataelementuid", "optionsetuid", "optionvalueuid")) + .build(), + Index.builder() + .name(appendRandom("in_dataelementoptioncode")) + .tableName(toStaging(TABLE_NAME)) + .columns(List.of("dataelementuid", "optionvaluecode")) + .build()); + } + + @Override + public ResourceTableType getTableType() { + return ResourceTableType.DATA_ELEMENT_CATEGORY_OPTION_COMBO; + } + + @Override + public Optional getPopulateTempTableStatement() { + String sql = + replace( + """ + insert into ${tableName} \ + (dataelementid, optionsetid, optionvalueid, dataelementuid, optionsetuid, optionvalueuid, optionvaluecode) \ + select de.dataelementid, os.optionsetid as optionsetid, ov.optionvalueid as optionvalueid, \ + de.uid as dataelementuid, os.uid as optionsetuid, ov.uid as optionvalueuid, ov.code as optionvaluecode from optionvalue ov \ + inner join optionset os on ov.optionsetid = os.optionsetid \ + inner join dataelement de on os.optionsetid = de.optionsetid;""", + "tableName", + toStaging(TABLE_NAME)); + + return Optional.of(sql); + } + + @Override + public Optional> getPopulateTempTableContent() { + return Optional.empty(); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DataQueryServiceDimensionItemKeywordTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DataQueryServiceDimensionItemKeywordTest.java index 622f2b45975f..777cef241f29 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DataQueryServiceDimensionItemKeywordTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DataQueryServiceDimensionItemKeywordTest.java @@ -122,6 +122,8 @@ class DataQueryServiceDimensionItemKeywordTest { @Mock private I18n i18n; + @Mock private OptionSetFacade optionSetFacade; + @InjectMocks private DimensionalObjectProvider dimensionalObjectProducer; private DefaultDataQueryService target; @@ -135,7 +137,8 @@ public void setUp() { lenient().when(settingsService.getCurrentSettings()).thenReturn(SystemSettings.of(Map.of())); target = - new DefaultDataQueryService(dimensionalObjectProducer, idObjectManager, securityManager); + new DefaultDataQueryService( + dimensionalObjectProducer, idObjectManager, securityManager, optionSetFacade); rb = new RequestBuilder(); diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DataDimensionExtractor.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DataDimensionExtractor.java index b7dea4affd99..6e1a229fd76d 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DataDimensionExtractor.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DataDimensionExtractor.java @@ -52,6 +52,7 @@ import org.hisp.dhis.dataelement.DataElementOperand; import org.hisp.dhis.dataset.DataSet; import org.hisp.dhis.indicator.Indicator; +import org.hisp.dhis.option.OptionSet; import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramDataElementDimensionItem; import org.hisp.dhis.program.ProgramIndicator; @@ -230,6 +231,26 @@ public ReportingRate getReportingRate(IdScheme idScheme, String dataSetId, Strin return new ReportingRate(dataSet, ReportingRateMetric.valueOf(metric)); } + /** + * Returns a {@link DataElement}. + * + * @param idScheme the identifier scheme. + * @param dataElementId the data element identifier. + * @param optionSetId the option set identifier. + */ + @Transactional(readOnly = true) + public DataElement getOptionSetDataElementDimensionItem( + IdScheme idScheme, String dataElementId, String optionSetId) { + DataElement dataElement = idObjectManager.getObject(DataElement.class, idScheme, dataElementId); + OptionSet optionSet = idObjectManager.getObject(OptionSet.class, idScheme, optionSetId); + + if (dataElement == null || optionSet == null) { + return null; + } + + return dataElement; + } + /** * Returns a {@link ProgramTrackedEntityAttributeDimensionItem}. * diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java index 8d37336eda60..7fc50b703e48 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java @@ -40,7 +40,7 @@ import static org.hisp.dhis.common.DimensionType.PROGRAM_ATTRIBUTE; import static org.hisp.dhis.common.DimensionType.PROGRAM_DATA_ELEMENT; import static org.hisp.dhis.common.DimensionType.PROGRAM_INDICATOR; -import static org.hisp.dhis.common.DimensionalObjectUtils.COMPOSITE_DIM_OBJECT_ESCAPED_SEP; +import static org.hisp.dhis.common.DimensionalObject.ITEM_SEP; import static org.hisp.dhis.common.IdScheme.UID; import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; import static org.hisp.dhis.commons.util.TextUtils.splitSafe; @@ -348,10 +348,18 @@ public DimensionalItemObject getDataDimensionalItemObject(String dimensionItem) public DimensionalItemObject getDataDimensionalItemObject( IdScheme idScheme, String dimensionItem) { if (DimensionalObjectUtils.isCompositeDimensionalObject(dimensionItem)) { - String id0 = splitSafe(dimensionItem, COMPOSITE_DIM_OBJECT_ESCAPED_SEP, 0); - String id1 = splitSafe(dimensionItem, COMPOSITE_DIM_OBJECT_ESCAPED_SEP, 1); - String id2 = splitSafe(dimensionItem, COMPOSITE_DIM_OBJECT_ESCAPED_SEP, 2); + String id0 = DimensionalObjectUtils.getFirstIdentifier(dimensionItem); + String id1 = DimensionalObjectUtils.getSecondIdentifier(dimensionItem); + String id2 = DimensionalObjectUtils.getThirdIdentifier(dimensionItem); + + String optionSetSelectionMode = splitSafe(dimensionItem, ITEM_SEP, 1); + if (optionSetSelectionMode != null && id2 != null) { + id2 = splitSafe(id2, ITEM_SEP, 0); + } else if (optionSetSelectionMode != null && id1 != null) { + id1 = splitSafe(id1, ITEM_SEP, 0); + } + DataElement dataElementWithOptionSet; DataElementOperand operand; ReportingRate reportingRate; ProgramDataElementDimensionItem programDataElement; @@ -373,6 +381,13 @@ public DimensionalItemObject getDataDimensionalItemObject( != null) { return programAttribute; } + + if ((dataElementWithOptionSet = + dataDimensionExtractor.getOptionSetDataElementDimensionItem(idScheme, id0, id1)) + != null) { + return dataElementWithOptionSet; + } + } else if (!idScheme.is(IdentifiableProperty.UID) || CodeGenerator.isValidUid(dimensionItem)) { return idObjectManager.get(DataDimensionItem.DATA_DIM_CLASSES, idScheme, dimensionItem); } diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/AnalyticsControllerTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/AnalyticsControllerTest.java index bdd79d005960..9ab2b91fe768 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/AnalyticsControllerTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/AnalyticsControllerTest.java @@ -50,6 +50,7 @@ import org.hisp.dhis.analytics.DataQueryService; import org.hisp.dhis.analytics.data.DefaultDataQueryService; import org.hisp.dhis.analytics.data.DimensionalObjectProvider; +import org.hisp.dhis.analytics.data.OptionSetFacade; import org.hisp.dhis.common.BaseDimensionalObject; import org.hisp.dhis.common.DimensionService; import org.hisp.dhis.common.DimensionType; @@ -94,14 +95,16 @@ class AnalyticsControllerTest { @Mock private DhisConfigurationProvider dhisConfigurationProvider; + @Mock private OptionSetFacade optionSetFacade; + @BeforeEach public void setUp() { - DataQueryService dataQueryService = new DefaultDataQueryService( dimensionalObjectProducer, mock(IdentifiableObjectManager.class), - mock(AnalyticsSecurityManager.class)); + mock(AnalyticsSecurityManager.class), + optionSetFacade); when(dimensionalObjectProducer.getPeriodDimension(Mockito.any(), Mockito.any())) .thenAnswer(