From 0264539925aa3cd1f336169ca0bf6a15d5ed2052 Mon Sep 17 00:00:00 2001 From: Hannah Eslinger Date: Fri, 1 Nov 2024 15:59:15 -0600 Subject: [PATCH 01/14] Add export charts to default reports (#4869) * Add export charts to default reports * lint * update button ls layout * lint --------- Co-authored-by: Katherine Fleming <2205659+kflemin@users.noreply.github.com> --- .../inventory_reports_controller.js | 12 ++++++++++++ .../seed/partials/inventory_reports.html | 18 +++++++++++------- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/seed/static/seed/js/controllers/inventory_reports_controller.js b/seed/static/seed/js/controllers/inventory_reports_controller.js index e6bfd4657e..0d322b260e 100644 --- a/seed/static/seed/js/controllers/inventory_reports_controller.js +++ b/seed/static/seed/js/controllers/inventory_reports_controller.js @@ -718,6 +718,18 @@ angular.module('SEED.controller.inventory_reports', []).controller('inventory_re }); } + $scope.downloadChart = () => { + const a = document.createElement('a'); + a.href = $scope.barChart.toBase64Image(); + a.download = 'default_report_bar.png'; + a.click(); + + const b = document.createElement('a'); + b.href = $scope.scatterChart.toBase64Image(); + b.download = 'default_report_scatter.png'; + b.click(); + }; + function updateStorage() { // Save axis and cycle selections localStorage.setItem(localStorageXAxisKey, JSON.stringify($scope.xAxisSelectedItem ?? '')); diff --git a/seed/static/seed/partials/inventory_reports.html b/seed/static/seed/partials/inventory_reports.html index f012df3334..8044d5ac2d 100644 --- a/seed/static/seed/partials/inventory_reports.html +++ b/seed/static/seed/partials/inventory_reports.html @@ -122,7 +122,6 @@

{$:: 'X Axis' | translate $}: -
@@ -134,13 +133,10 @@

CONFIGURE_XY_AXES

- -
-
- -
- +
+ +
@@ -153,6 +149,14 @@

+
+
+ + +
+
From b8ec8cb156fa3caed49a40cf7d27f57953e728d0 Mon Sep 17 00:00:00 2001 From: Hannah Eslinger Date: Mon, 4 Nov 2024 17:04:01 -0700 Subject: [PATCH 02/14] Move show populated to backend (#4866) * Move show populated to backend * Include derived columns * Update seed/static/seed/js/controllers/show_populated_columns_modal_controller.js Co-authored-by: Ross Perry * Fix for detail page --------- Co-authored-by: Ross Perry Co-authored-by: Katherine Fleming <2205659+kflemin@users.noreply.github.com> --- seed/models/columns.py | 46 +++++++- .../inventory_detail_controller.js | 2 +- ...show_populated_columns_modal_controller.js | 110 ++---------------- .../seed/js/services/inventory_service.js | 20 ++++ .../show_populated_columns_modal.html | 2 +- seed/tests/test_column_list_profiles_views.py | 98 +++++++++++++++- seed/views/v3/analyses.py | 68 +---------- seed/views/v3/column_list_profiles.py | 37 ++++++ 8 files changed, 211 insertions(+), 172 deletions(-) diff --git a/seed/models/columns.py b/seed/models/columns.py index e2b10d043b..c701d10205 100644 --- a/seed/models/columns.py +++ b/seed/models/columns.py @@ -14,8 +14,8 @@ from django.apps import apps from django.core.exceptions import ValidationError -from django.db import IntegrityError, models, transaction -from django.db.models import Q +from django.db import IntegrityError, connection, models, transaction +from django.db.models import Count, Q from django.db.models.signals import pre_save from django.utils.translation import gettext_lazy as _ @@ -1665,6 +1665,48 @@ def retrieve_all_by_tuple(org_id): return result + @staticmethod + def get_num_of_nonnulls_by_column_name(state_ids, inventory_class, columns): + states = inventory_class.objects.filter(id__in=state_ids) + + # init dicts + num_of_nonnulls_by_column_name = {c.column_name: 0 for c in columns} + canonical_columns = [c.column_name for c in columns if not c.is_extra_data] + + # add non-null counts for extra_data columns + with connection.cursor() as cursor: + table_name = "seed_propertystate" if inventory_class.__name__ == "PropertyState" else "seed_taxlotstate" + non_null_extra_data_counts_query = ( + f'SELECT key, COUNT(*)\n' + f'FROM {table_name}, LATERAL JSONB_EACH_TEXT(extra_data) AS each_entry(key, value)\n' + f'WHERE id IN ({", ".join(map(str, state_ids))})\n' + f' AND value IS NOT NULL\n' + f'GROUP BY key;' + ) + cursor.execute(non_null_extra_data_counts_query) + extra_data_counts = dict(cursor.fetchall()) + num_of_nonnulls_by_column_name.update(extra_data_counts) + + # add non-null counts for derived_data columns + with connection.cursor() as cursor: + table_name = "seed_propertystate" if inventory_class.__name__ == "PropertyState" else "seed_taxlotstate" + non_null_derived_data_counts_query = ( + f'SELECT key, COUNT(*)\n' + f'FROM {table_name}, LATERAL JSONB_EACH_TEXT(derived_data) AS each_entry(key, value)\n' + f'WHERE id IN ({", ".join(map(str, state_ids))})\n' + f' AND value IS NOT NULL\n' + f'GROUP BY key;' + ) + cursor.execute(non_null_derived_data_counts_query) + derived_data_counts = dict(cursor.fetchall()) + num_of_nonnulls_by_column_name.update(derived_data_counts) + + # add non-null counts for canonical columns + canonical_counts = states.aggregate(**{col: Count(col) for col in canonical_columns}) + num_of_nonnulls_by_column_name.update(canonical_counts) + + return num_of_nonnulls_by_column_name + def validate_model(sender, **kwargs): instance = kwargs["instance"] diff --git a/seed/static/seed/js/controllers/inventory_detail_controller.js b/seed/static/seed/js/controllers/inventory_detail_controller.js index 3ca350b575..4dd9ad85d1 100644 --- a/seed/static/seed/js/controllers/inventory_detail_controller.js +++ b/seed/static/seed/js/controllers/inventory_detail_controller.js @@ -259,7 +259,7 @@ angular.module('SEED.controller.inventory_detail', []).controller('inventory_det resolve: { columns: () => columns, currentProfile: () => $scope.currentProfile, - cycle: () => null, + cycle: () => $scope.cycle, inventory_type: () => $stateParams.inventory_type, provided_inventory() { const provided_inventory = []; diff --git a/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js b/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js index aa5bd47e86..54e980a262 100644 --- a/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js +++ b/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js @@ -17,118 +17,22 @@ angular.module('SEED.controller.show_populated_columns_modal', []).controller('s 'inventory_type', // eslint-disable-next-line func-names function ($scope, $window, $uibModalInstance, Notification, inventory_service, modified_service, spinner_utility, columns, currentProfile, cycle, provided_inventory, inventory_type) { - $scope.columns = columns; - $scope.currentProfile = currentProfile; - $scope.cycle = cycle; - $scope.inventory_type = inventory_type; - - _.forEach($scope.columns, (col) => { - col.pinnedLeft = false; - col.visible = true; - }); - - const notEmpty = (value) => !_.isNil(value) && value !== ''; - - const fetch = (page, chunk) => { - let fn; - if ($scope.inventory_type === 'properties') { - fn = inventory_service.get_properties; - } else if ($scope.inventory_type === 'taxlots') { - fn = inventory_service.get_taxlots; - } - return fn(page, chunk, $scope.cycle, -1).then((data) => { - $scope.progress = Math.round((data.pagination.end / data.pagination.total) * 100); - if (data.pagination.has_next) { - return fetch(page + 1, chunk).then((data2) => data.results.concat(data2)); - } - return data.results; - }); - }; - - const update_profile_with_populated_columns = (inventory) => { - $scope.status = `Processing ${$scope.columns.length} columns in ${inventory.length} records`; - - const cols = _.reject($scope.columns, 'related'); - // console.log('cols', cols); - - const relatedCols = _.filter($scope.columns, 'related'); - // console.log('relatedCols', relatedCols); - - const col_key = provided_inventory ? 'column_name' : 'name'; - - _.forEach(inventory, (record, index) => { - // console.log(cols.length + ' remaining cols to check'); - _.forEachRight(cols, (col, colIndex) => { - if (notEmpty(record[col[col_key]])) { - // console.log('Removing ' + col[col_key] + ' from cols'); - cols.splice(colIndex, 1); - } - }); - - _.forEach(record.related, (relatedRecord) => { - // console.log(relatedCols.length + ' remaining related cols to check'); - _.forEachRight(relatedCols, (col, colIndex) => { - if (notEmpty(relatedRecord[col[col_key]])) { - // console.log('Removing ' + col[col_key] + ' from relatedCols'); - relatedCols.splice(colIndex, 1); - } - }); - }); - - $scope.progress = (index / inventory.length) * 50 + 50; - }); - - // determine hidden columns - const visible = _.reject($scope.columns, (col) => { - if (!col.related) { - return _.find(cols, { id: col.id }); - } - return _.find(relatedCols, { id: col.id }); - }); - - const hidden = _.reject($scope.columns, (col) => _.find(visible, { id: col.id })); - - _.forEach(hidden, (col) => { - col.visible = false; - }); - - const columns = []; - _.forEach(visible, (col) => { - columns.push({ - column_name: col.column_name, - id: col.id, - order: columns.length + 1, - pinned: col.pinnedLeft, - table_name: col.table_name - }); - }); + $scope.start = () => { + $scope.state = 'running'; + $scope.status = 'Processing...'; + $scope.inventory_type = inventory_type === 'properties' ? 'Property' : 'Tax lot'; - const { id } = $scope.currentProfile; - const profile = _.omit($scope.currentProfile, 'id'); - profile.columns = columns; - inventory_service.update_column_list_profile(id, profile).then((/* updatedProfile */) => { + inventory_service.update_column_list_profile_to_show_populated(currentProfile.id, cycle.id, $scope.inventory_type).then((/* updatedProfile */) => { modified_service.resetModified(); $scope.progress = 100; $scope.state = 'done'; - $scope.status = `Found ${visible.length} populated columns`; + $scope.refresh(); }); }; - $scope.start = () => { - $scope.state = 'running'; - $scope.status = 'Fetching Inventory'; - - if (provided_inventory) { - update_profile_with_populated_columns(provided_inventory); - } else { - const page = 1; - const chunk = 5000; - fetch(page, chunk).then(update_profile_with_populated_columns); - } - }; - $scope.refresh = () => { spinner_utility.show(); + $uibModalInstance.close(); $window.location.reload(); }; diff --git a/seed/static/seed/js/services/inventory_service.js b/seed/static/seed/js/services/inventory_service.js index a7d4905091..8e48e89b83 100644 --- a/seed/static/seed/js/services/inventory_service.js +++ b/seed/static/seed/js/services/inventory_service.js @@ -1200,6 +1200,26 @@ angular.module('SEED.service.inventory', []).factory('inventory_service', [ .then((response) => response.data.data); }; + inventory_service.update_column_list_profile_to_show_populated = (id, cycle_id, inventory_type) => { + if (id === null) { + Notification.error('This settings profile is protected from modifications'); + return $q.reject(); + } + return $http + .put( + `/api/v3/column_list_profiles/${id}/show_populated/`, + { + cycle_id, + inventory_type + }, + { + params: { + organization_id: user_service.get_organization().id + } + } + ).then((response) => response.data.data); + }; + inventory_service.remove_column_list_profile = (id) => { if (id === null) { Notification.error('This settings profile is protected from modifications'); diff --git a/seed/static/seed/partials/show_populated_columns_modal.html b/seed/static/seed/partials/show_populated_columns_modal.html index 7a0a2b44c9..edc415f29d 100644 --- a/seed/static/seed/partials/show_populated_columns_modal.html +++ b/seed/static/seed/partials/show_populated_columns_modal.html @@ -4,7 +4,7 @@
- - - - - - - - - - - - - - - - - - - - - -
AxisAccess Level InstanceSumMin5th Percentile25th PercentileMeanMedian75 Percentile95th PercentileMax
{$ item $}
+

Statistics

+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
AxisAccess Level InstanceSumMin5th Percentile25th PercentileMeanMedian75 Percentile95th PercentileMax
{$ key $} + {$ item $} + {$ item | tolerantNumber:2 $} + + + + +
+ + + + + + +
 {$ key2 $} + {$ item $} + {$ item | tolerantNumber:2 $} +
+
+
+
diff --git a/seed/static/seed/scss/style.scss b/seed/static/seed/scss/style.scss index a091629aaf..2367f5af8a 100755 --- a/seed/static/seed/scss/style.scss +++ b/seed/static/seed/scss/style.scss @@ -5683,3 +5683,9 @@ tags-input .tags .tag-item { white-space: normal; min-width: 350px; } + +.no-lr-pad { + padding-left: 0 !important; + padding-right: 0 !important; + padding-top: 0 !important; +} diff --git a/seed/views/v3/organizations.py b/seed/views/v3/organizations.py index 0367424072..364ef679b3 100644 --- a/seed/views/v3/organizations.py +++ b/seed/views/v3/organizations.py @@ -1024,6 +1024,12 @@ def report_aggregated(self, request, pk=None): ys = [building["y"] for datum in data for building in datum["chart_data"] if building["y"] is not None] if ys and isinstance(ys[0], Number): bins = np.histogram_bin_edges(ys, bins=5) + + # special case for year built: make bins integers + # year built is in x axis, but it shows up in y_var variable + if params["y_var"] == "year_built": + bins = bins.astype(int) + aggregate_data = self.continuous_aggregate_data else: bins = list(set(ys)) @@ -1243,15 +1249,19 @@ def get_axis_stats(self, organization, cycle, axis, axis_var, views, ali): return [axis_var, ali.name, 0, 0, 0, 0, 0, 0, 0, 0, 0] def get_axis_data(self, organization_id, access_level_instance, cycles, x_var, y_var, all_property_views, fields): - axis_data = [] + axis_data = {} axes = {"x": x_var, "y": y_var} organization = Organization.objects.get(pk=organization_id) + # initialize + for cycle in cycles: + axis_data[cycle.name] = {} + for axis in axes: if axes[axis] != "Count": columns = Column.objects.filter(organization_id=organization_id, column_name=axes[axis]) if not columns: - return [] + return {} column = columns[0] if not column.data_type or column.data_type == "None": @@ -1266,19 +1276,24 @@ def get_axis_data(self, organization_id, access_level_instance, cycles, x_var, y name_to_display = ( serialized_column["display_name"] if serialized_column["display_name"] != "" else serialized_column["column_name"] ) - axis_name = name_to_display + f" ({cycle.name})" + axis_data[cycle.name][name_to_display] = {} stats = self.get_axis_stats(organization, cycle, axis, axes[axis], all_property_views, access_level_instance) - axis_data.append(self.clean_axis_data(axis_name, data_type, stats)) - for child_ali in access_level_instance.get_children(): - stats = self.get_axis_stats(organization, cycle, axis, axes[axis], all_property_views, child_ali) - axis_data.append(self.clean_axis_data(axis_name, data_type, stats)) + axis_data[cycle.name][name_to_display]["values"] = self.clean_axis_data(data_type, stats) + + children = access_level_instance.get_children() + if len(children): + axis_data[cycle.name][name_to_display]["children"] = {} + for child_ali in children: + stats = self.get_axis_stats(organization, cycle, axis, axes[axis], all_property_views, child_ali) + axis_data[cycle.name][name_to_display]["children"][child_ali.name] = self.clean_axis_data(data_type, stats) + return axis_data - def clean_axis_data(self, column_name, data_type, data): + def clean_axis_data(self, data_type, data): if data_type == "float": - return [column_name] + data[1:3] + np.round(data[3:], decimals=2).tolist() + return data[1:3] + np.round(data[3:], decimals=2).tolist() elif data_type == "integer": - return [column_name] + data[1:3] + np.round(data[3:]).tolist() + return data[1:3] + np.round(data[3:]).tolist() @has_perm_class("requires_member") @ajax_request_class From de11e2a977dfdd6a1d4e33c0676c81df3cd18167 Mon Sep 17 00:00:00 2001 From: Katherine Fleming <2205659+kflemin@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:29:59 -0700 Subject: [PATCH 06/14] Fix stats table for various ali layouts (#4880) fix stats table for different alis --- .../js/controllers/inventory_reports_controller.js | 10 ++++++++++ seed/static/seed/partials/inventory_reports.html | 5 ++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/seed/static/seed/js/controllers/inventory_reports_controller.js b/seed/static/seed/js/controllers/inventory_reports_controller.js index 0d322b260e..2eed57e7ba 100644 --- a/seed/static/seed/js/controllers/inventory_reports_controller.js +++ b/seed/static/seed/js/controllers/inventory_reports_controller.js @@ -65,6 +65,16 @@ angular.module('SEED.controller.inventory_reports', []).controller('inventory_re $scope.filter_groups = filter_groups; $scope.report_configurations = report_configurations; $scope.filter_group_id = null; + + $scope.has_children = (obj) => { + // check if the access level selected has children levels for stats table + let children = false; + if ('children' in obj && Object.keys(obj.children).length > 0) { + children = true; + } + return children; + }; + function path_to_string(path) { const orderedPath = []; for (const i in $scope.level_names) { diff --git a/seed/static/seed/partials/inventory_reports.html b/seed/static/seed/partials/inventory_reports.html index f612743d0e..2e8b4936ce 100644 --- a/seed/static/seed/partials/inventory_reports.html +++ b/seed/static/seed/partials/inventory_reports.html @@ -267,14 +267,13 @@

Statistics

Max - + {$ key $} {$ item $} {$ item | tolerantNumber:2 $} - - +