Skip to content

Commit

Permalink
Speed-up-program-and-property-insight-pages (#4890)
Browse files Browse the repository at this point in the history 
* Speed-up-program-and-property-insight-pages

* Add ali selection to property insights page

* lint

* Lint

* Lint

---------

Co-authored-by: kflemin <[email protected]>
  • Loading branch information
@haneslinger @kflemin
haneslinger and kflemin authored Dec 10, 2024
1 parent 24bb932 commit 207b4e6
Show file tree
Hide file tree
Showing 8 changed files with 166 additions and 58 deletions.
35 changes: 16 additions & 19 deletions seed/models/compliance_metrics.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,29 +66,26 @@ def evaluate(self, user_ali):
# get properties (no filter)
# property_response = properties_across_cycles(self.organization_id, -1, cycle_ids)
# get properties (applies filter group)
display_field_id = Column.objects.get(
display_field = Column.objects.get(
table_name="PropertyState", column_name=self.organization.property_display_field, organization=self.organization
).id
)
# array of columns to return
column_ids = [display_field_id]
columns = [display_field]

if self.actual_energy_column is not None:
column_ids.append(self.actual_energy_column.id)
columns.append(self.actual_energy_column)
if self.target_energy_column is not None:
column_ids.append(self.target_energy_column.id)
columns.append(self.target_energy_column)

if self.actual_emission_column is not None:
column_ids.append(self.actual_emission_column.id)
columns.append(self.actual_emission_column)
if self.target_emission_column is not None:
column_ids.append(self.target_emission_column.id)
columns.append(self.target_emission_column)

for col in self.x_axis_columns.all():
column_ids.append(col.id)
columns.append(col)

# Unique ids
column_ids = [*set(column_ids)]

property_response = properties_across_cycles_with_filters(self.organization_id, user_ali, cycle_ids, query_dict, column_ids)
property_response = properties_across_cycles_with_filters(self.organization_id, user_ali, cycle_ids, query_dict, columns)

datasets = {
"y": {"data": [], "label": "compliant"},
Expand Down Expand Up @@ -141,24 +138,24 @@ def evaluate(self, user_ali):

for p in property_response[cyc]:
# initialize
properties[p["property_view_id"]] = None
properties[p["id"]] = None
# energy metric
if metric["energy_metric"]:
properties[p["property_view_id"]] = self._calculate_compliance(p, metric["energy_bool"], "energy")
properties[p["id"]] = self._calculate_compliance(p, metric["energy_bool"], "energy")
# emission metric
if metric["emission_metric"] and properties[p["property_view_id"]] != "u":
if metric["emission_metric"] and properties[p["id"]] != "u":
temp_val = self._calculate_compliance(p, metric["emission_bool"], "emission")

# reconcile
if temp_val == "u":
# unknown stays unknown (missing data)
properties[p["property_view_id"]] = "u"
elif properties[p["property_view_id"]] is None:
properties[p["id"]] = "u"
elif properties[p["id"]] is None:
# only emission metric (not energy metric)
properties[p["property_view_id"]] = temp_val
properties[p["id"]] = temp_val
else:
# compliant if both are compliant
properties[p["property_view_id"]] = temp_val if temp_val == "n" else properties[p["property_view_id"]]
properties[p["id"]] = temp_val if temp_val == "n" else properties[p["id"]]

# count compliant, non-compliant, unknown for each property with data
for key in cnts:
Expand Down
64 changes: 60 additions & 4 deletions seed/static/seed/js/controllers/insights_property_controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ angular.module('SEED.controller.insights_property', []).controller('insights_pro
'filter_groups',
'property_columns',
'cycles',
'access_level_tree',
'user_service',
'ah_service',
'spinner_utility',
'auth_payload',
// eslint-disable-next-line func-names
Expand All @@ -29,6 +32,9 @@ angular.module('SEED.controller.insights_property', []).controller('insights_pro
filter_groups,
property_columns,
cycles,
access_level_tree,
user_service,
ah_service,
spinner_utility,
auth_payload
) {
Expand All @@ -37,6 +43,13 @@ angular.module('SEED.controller.insights_property', []).controller('insights_pro
$scope.organization = organization_payload.organization;
$scope.auth = auth_payload.auth;

$scope.access_level_tree = access_level_tree.access_level_tree;
$scope.level_names = access_level_tree.access_level_names;
$scope.level_name_index = null;
$scope.potential_level_instances = [];
$scope.access_level_instance_id = null;
$scope.users_access_level_instance_id = user_service.get_access_level_instance().id;

// used by modal
$scope.filter_groups = filter_groups;
$scope.property_columns = property_columns;
Expand All @@ -48,6 +61,28 @@ angular.module('SEED.controller.insights_property', []).controller('insights_pro
$scope.show_help = !$scope.show_help;
};

function path_to_string(path) {
const orderedPath = [];
for (const i in $scope.level_names) {
if (Object.prototype.hasOwnProperty.call(path, $scope.level_names[i])) {
orderedPath.push(path[$scope.level_names[i]]);
}
}
return orderedPath.join(' : ');
}

const access_level_instances_by_depth = ah_service.calculate_access_level_instances_by_depth($scope.access_level_tree);
// cannot select parents alis
const [users_depth] = Object.entries(access_level_instances_by_depth).find(([, x]) => x.length === 1 && x[0].id === parseInt($scope.users_access_level_instance_id, 10));
$scope.change_selected_level_index = () => {
const new_level_instance_depth = parseInt($scope.level_name_index, 10) + parseInt(users_depth, 10);
$scope.potential_level_instances = access_level_instances_by_depth[new_level_instance_depth];
for (const key in $scope.potential_level_instances) {
$scope.potential_level_instances[key].name = path_to_string($scope.potential_level_instances[key].path);
}
$scope.access_level_instance_id = null;
};

// configs ($scope.configs set to saved_configs where still applies.
// for example, if saved_configs.compliance_metric is 1, but 1 has been deleted, it does apply.)
const saved_configs = JSON.parse(localStorage.getItem(`insights.property.configs.${$scope.organization.id}`));
Expand Down Expand Up @@ -154,6 +189,12 @@ angular.module('SEED.controller.insights_property', []).controller('insights_pro
true
);

$scope.change_ali = () => {
localStorage.setItem(localStorageALIndex, JSON.stringify($scope.level_name_index));
localStorage.setItem(localStorageALIID, JSON.stringify($scope.access_level_instance_id));
_load_data();
};

// load data
const _load_data = () => {
if (_.isEmpty($scope.configs.compliance_metric)) {
Expand All @@ -163,11 +204,19 @@ angular.module('SEED.controller.insights_property', []).controller('insights_pro
}
spinner_utility.show();
compliance_metric_service
.evaluate_compliance_metric($scope.configs.compliance_metric.id)
.evaluate_compliance_metric($scope.configs.compliance_metric.id, $scope.organization.id, $scope.access_level_instance_id)
.then((data) => {
$scope.data = data;
})
.then(() => {
// if there's +3k properties, dont even bother charting them.
$scope.chartStatusMessage = '';
const num_properties = Object.values($scope.data.properties_by_cycles).reduce((acc, curr) => acc + curr.length, 0);
if (num_properties > 3000) {
$scope.data.properties_by_cycles = Object.keys($scope.data.properties_by_cycles).reduce((acc, k) => ({ ...acc, [k]: [] }), {});
$scope.chartStatusMessage = 'Too much data, try a different ali';
}

if ($scope.data) {
// set options
// cycles
Expand Down Expand Up @@ -307,7 +356,7 @@ angular.module('SEED.controller.insights_property', []).controller('insights_pro
$scope.annotations = {};

_.forEach($scope.data.properties_by_cycles[$scope.configs.chart_cycle], (prop) => {
const item = { id: prop.property_view_id };
const item = { id: prop.id };
item.name = _.find(prop, (v, k) => k.startsWith($scope.organization.property_display_field));
// x axis is easy
item.x = _.find(prop, (v, k) => k.endsWith(`_${String($scope.configs.chart_xaxis)}`));
Expand All @@ -333,10 +382,10 @@ angular.module('SEED.controller.insights_property', []).controller('insights_pro
}

// place in appropriate dataset
if (_.includes($scope.data.results_by_cycles[$scope.configs.chart_cycle].y, prop.property_view_id)) {
if (_.includes($scope.data.results_by_cycles[$scope.configs.chart_cycle].y, prop.id)) {
// compliant dataset
datasets[0].data.push(item);
} else if (_.includes($scope.data.results_by_cycles[$scope.configs.chart_cycle].n, prop.property_view_id)) {
} else if (_.includes($scope.data.results_by_cycles[$scope.configs.chart_cycle].n, prop.id)) {
// non-compliant dataset
datasets[1].data.push(item);
} else {
Expand Down Expand Up @@ -606,6 +655,13 @@ angular.module('SEED.controller.insights_property', []).controller('insights_pro
$scope.configs.annotation_visibility = $scope.display_annotation;
};

const localStorageALIndex = 'insights.property.configs.ALIndex';
const localStorageALIID = 'insights.property.configs.ALIID';
$scope.level_name_index = JSON.parse(localStorage.getItem(localStorageALIndex)) || '0';
const new_level_instance_depth = parseInt($scope.level_name_index, 10) + parseInt(users_depth, 10);
$scope.potential_level_instances = access_level_instances_by_depth[new_level_instance_depth];
$scope.access_level_instance_id = JSON.parse(localStorage.getItem(localStorageALIID)) || parseInt($scope.users_access_level_instance_id, 10);

setTimeout(_load_data, 0); // avoid race condition with route transition spinner.

$scope.visibleIds = () => {
Expand Down
10 changes: 8 additions & 2 deletions seed/static/seed/js/controllers/program_setup_controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,14 @@ angular.module('SEED.controller.program_setup', []).controller('program_setup_co
$scope.valid_column_data_types = ['number', 'float', 'integer', 'ghg', 'ghg_intensity', 'area', 'eui', 'boolean'];
$scope.valid_x_axis_data_types = ['number', 'string', 'float', 'integer', 'ghg', 'ghg_intensity', 'area', 'eui', 'boolean'];

$scope.property_columns = _.reject(property_columns, (item) => (item.related || !$scope.valid_column_data_types.includes(item.data_type)) && item.derived_column == null).sort((a, b) => naturalSort(a.displayName, b.displayName));
$scope.x_axis_columns = _.reject(property_columns, (item) => (item.related || !$scope.valid_x_axis_data_types.includes(item.data_type)) && item.derived_column == null).sort((a, b) => naturalSort(a.displayName, b.displayName));
$scope.property_columns = _.reject(property_columns, (item) => (
(item.related || !$scope.valid_column_data_types.includes(item.data_type)) &&
item.derived_column == null
)).sort((a, b) => naturalSort(a.displayName, b.displayName));
$scope.x_axis_columns = _.reject(property_columns, (item) => (
(item.related || !$scope.valid_x_axis_data_types.includes(item.data_type)) &&
item.derived_column == null
)).sort((a, b) => naturalSort(a.displayName, b.displayName));
$scope.x_axis_selection = '';
$scope.cycle_selection = '';
$scope.compliance_metrics_error = [];
Expand Down
10 changes: 9 additions & 1 deletion seed/static/seed/js/seed.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2586,7 +2586,15 @@
return inventory_service.get_property_columns_for_org(organization_id);
}
],
cycles: ['cycle_service', (cycle_service) => cycle_service.get_cycles()]
cycles: ['cycle_service', (cycle_service) => cycle_service.get_cycles()],
access_level_tree: [
'organization_service',
'user_service',
(organization_service, user_service) => {
const organization_id = user_service.get_organization().id;
return organization_service.get_organization_access_level_tree(organization_id);
}
]
}
})
.state({
Expand Down
5 changes: 3 additions & 2 deletions seed/static/seed/js/services/compliance_metric_service.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,11 @@ angular.module('SEED.service.compliance_metric', []).factory('compliance_metric_
};

// evaluate
const evaluate_compliance_metric = (metric_id, organization_id = user_service.get_organization().id) => $http
const evaluate_compliance_metric = (metric_id, organization_id = user_service.get_organization().id, access_level_instance_id = null) => $http
.get(`/api/v3/compliance_metrics/${metric_id}/evaluate/`, {
params: {
organization_id
organization_id,
...(access_level_instance_id ? { access_level_instance_id } : {})
}
})
.then((response) => response.data.data)
Expand Down
22 changes: 22 additions & 0 deletions seed/static/seed/partials/insights_property.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,22 @@
<select class="form-control" id="xSelect" ng-model="configs.chart_xaxis" ng-change="update()" ng-options="operator.id as operator.display_name for operator in x_axis_options"></select>
</div>

<!-- ali selection -->
<div class="form-group pad-bottom-10">
<label for="accessLevel" translate>Access Level:</label>
<select id="accessLevel" class="form-control col-sm-8" ng-model="level_name_index" ng-change="change_selected_level_index()" ng-options="i as name for (i, name) in level_names"></select>
</div>
<div class="form-group">
<label for="accessLevelInstance" translate>Access Level Instance:</label>
<select
for="accessLevelInstance"
class="form-control"
ng-change="change_ali()"
ng-model="access_level_instance_id"
ng-options="potential_level_instance.id as potential_level_instance.name for potential_level_instance in potential_level_instances"
></select>
</div>

<div ng-show="menu.user.organization.user_role !== 'viewer'" class="button-grid chart-buttons">
<div class="flex-button-child">
<button type="button" class="btn btn-primary" ng-click="open_update_labels_modal()" ng-disabled="!visibleIds().length" translate>Update Property Labels</button>
Expand All @@ -73,6 +89,12 @@
<div class="graph" style="margin-left: 3vw; margin-right: 2vw">
<canvas id="property-insights-chart"></canvas>
</div>
<div ng-show="chartStatusMessage" class="status-message">
{$ chartStatusMessage | translate $}
<div class="progress progress-striped active" ng-show="chartIsLoading">
<div class="progress-bar" role="progressbar" aria-valuenow="100" aria-valuemin="0" aria-valuemax="100" style="width: 100%"></div>
</div>
</div>
<div class="section_content r-columns insights-table" style="display: unset !important">
<div class="r-column r-shrink table_list_container" ng-if="data" style="margin-left: 3vw; margin-right: 2vw">
<table>
Expand Down
66 changes: 38 additions & 28 deletions seed/utils/properties.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
import json
import logging

from django.db.models import F

# Imports from Django
from django.http import JsonResponse
from rest_framework import status
Expand All @@ -24,7 +26,6 @@
TaxLotView,
)
from seed.serializers.pint import apply_display_unit_preferences
from seed.utils.search import build_view_filters_and_sorts

logging.basicConfig(format="%(asctime)s %(levelname)-8s %(message)s", level=logging.ERROR, datefmt="%Y-%m-%d %H:%M:%S")

Expand Down Expand Up @@ -170,41 +171,50 @@ def properties_across_cycles(org_id, ali, profile_id, cycle_ids=[]):
return results


def properties_across_cycles_with_filters(org_id, user_ali, cycle_ids=[], query_dict={}, column_ids=[]):
# Identify column preferences to be used to scope fields/values
columns_from_database = Column.retrieve_all(org_id, "property", False)
org = Organization.objects.get(pk=org_id)
def properties_across_cycles_with_filters(org_id, user_ali, cycle_ids=[], query_dict={}, columns=[]):
# get relevant views
views_list = PropertyView.objects.select_related("property", "state", "cycle").filter(
property__organization_id=org_id,
cycle_id__in=cycle_ids,
property__access_level_instance__lft__gte=user_ali.lft,
property__access_level_instance__rgt__lte=user_ali.rgt,
)
views_list = _serialize_views(views_list, columns, org_id)

# group by cycle
results = {cycle_id: [] for cycle_id in cycle_ids}
property_views = _get_filter_group_views(org_id, cycle_ids, query_dict, user_ali)
views_cycle_ids = [v.cycle_id for v in property_views]
related_results = TaxLotProperty.serialize(property_views, column_ids, columns_from_database, include_related=False)
unit_collapsed_results = [apply_display_unit_preferences(org, x) for x in related_results]

for cycle_id, unit_collapsed_result in zip(views_cycle_ids, unit_collapsed_results):
results[cycle_id].append(unit_collapsed_result)
for view in views_list:
cycle_id = view["cycle_id"]
del view["cycle_id"]
results[cycle_id].append(view)

return results


# helper function for getting filtered properties
def _get_filter_group_views(org_id, cycles, query_dict, user_ali):
columns = Column.retrieve_all(org_id=org_id, inventory_type="property", only_used=False, include_related=False)
def _serialize_views(views_list, columns, org_id):
org = Organization.objects.get(pk=org_id)

# build annotations
annotations = {}
try:
filters, annotations, _order_by = build_view_filters_and_sorts(query_dict, columns, "property")
except Exception:
return JsonResponse({"status": "error", "message": "error with filter group"}, status=status.HTTP_404_NOT_FOUND)

views_list = PropertyView.objects.select_related("property", "state", "cycle").filter(
property__organization_id=org_id,
cycle__in=cycles,
property__access_level_instance__lft__gte=user_ali.lft,
property__access_level_instance__rgt__lte=user_ali.rgt,
)

views_list = views_list.annotate(**annotations).filter(filters).order_by("id")
values_list = ["id", "cycle_id"] # django readable names
returned_name = ["id", "cycle_id"] # actual api names
for column in columns:
if column.is_extra_data:
anno_value = F("state__extra_data__" + column.column_name)
elif column.derived_column:
anno_value = F("state__derived_data__" + column.column_name)
else:
anno_value = F("state__" + column.column_name)

name = f"{column.column_name.replace(' ', '_')}_{column.id}" # django readable name
annotations[name] = anno_value
values_list.append(name)
returned_name.append(f"{column.column_name}_{column.id}")

# use api names and add units
views_list = views_list.annotate(**annotations).values_list(*values_list)
views_list = [dict(zip(returned_name, view)) for view in views_list] # replace django readable name with api name
views_list = [apply_display_unit_preferences(org, view) for view in views_list]

return views_list

Expand Down
Loading

0 comments on commit 207b4e6

Please sign in to comment.