diff --git a/locale/en_US/LC_MESSAGES/django.mo b/locale/en_US/LC_MESSAGES/django.mo index 50ec7730b3..19d51eb18d 100644 Binary files a/locale/en_US/LC_MESSAGES/django.mo and b/locale/en_US/LC_MESSAGES/django.mo differ diff --git a/locale/en_US/LC_MESSAGES/django.po b/locale/en_US/LC_MESSAGES/django.po index 056062f813..e449040656 100644 --- a/locale/en_US/LC_MESSAGES/django.po +++ b/locale/en_US/LC_MESSAGES/django.po @@ -1,15 +1,14 @@ msgid "" msgstr "" -"Project-Id-Version: PACKAGE VERSION\n" -"Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-10-06 12:15-0700\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: FULL NAME \n" -"Language-Team: LANGUAGE \n" -"Language: \n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" +"X-Generator: lokalise.com\n" +"Project-Id-Version: SEED Platform\n" +"PO-Revision-Date: 2026-03-10 20:51\n" +"Last-Translator: lokalise.com\n" +"Language-Team: lokalise.com\n\n" +"Language: en_US\n" msgid "# Of Files" msgstr "# Of Files" @@ -2948,7 +2947,7 @@ msgid "ORGANIZATION_WUI_DISPLAY_UNITS" msgstr "Measurement unit display for Water Use Intensity (WUI)" msgid "ORG_MATCH_MERGE_LINK_WARNING" -msgstr "WARNING: You have modified the criteria that your organization's data will be matched and merged against. Once a column has been added to an organization's matching criteria it cannot be removed while the organization has inventory." +msgstr "WARNING: You have modified the criteria that your organization's data will be matched and merged against. If the organization is using more than one access levels: once a column has been added to an organization's matching criteria it cannot be removed while the organization has inventory. Matching criteria can be removed in organizations not using access levels." msgid "OVERWRITE_COLUMN_DATA_QUESTION" msgstr "Overwrite data?" diff --git a/locale/es/LC_MESSAGES/django.mo b/locale/es/LC_MESSAGES/django.mo index 587316e6c5..1ac65a7c39 100644 Binary files a/locale/es/LC_MESSAGES/django.mo and b/locale/es/LC_MESSAGES/django.mo differ diff --git a/locale/es/LC_MESSAGES/django.po b/locale/es/LC_MESSAGES/django.po index 1794744b59..c789897607 100644 --- a/locale/es/LC_MESSAGES/django.po +++ b/locale/es/LC_MESSAGES/django.po @@ -5,7 +5,7 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "X-Generator: lokalise.com\n" "Project-Id-Version: SEED Platform\n" -"PO-Revision-Date: 2025-12-11 17:45\n" +"PO-Revision-Date: 2026-03-10 20:51\n" "Last-Translator: lokalise.com\n" "Language-Team: lokalise.com\n\n" "Language: es\n" @@ -3751,9 +3751,8 @@ msgstr "Visualización de la unidad de medida del consumo de agua" msgid "ORGANIZATION_WUI_DISPLAY_UNITS" msgstr "Visualización de la unidad de medida para la intensidad de uso del agua (WUI)" -#, fuzzy msgid "ORG_MATCH_MERGE_LINK_WARNING" -msgstr "ADVERTENCIA: Ha modificado los criterios con los que se compararán y combinarán los datos de su organización. Una vez que se ha añadido una columna a los criterios de correspondencia de una organización, no se puede eliminar mientras la organización tenga inventario." +msgstr "ADVERTENCIA: Ha modificado los criterios con los que se compararán y fusionarán los datos de su organización. Si la organización utiliza más de un nivel de acceso, una vez que se haya añadido una columna a los criterios de coincidencia de una organización, no se podrá eliminar mientras la organización tenga inventario. Los criterios de coincidencia se pueden eliminar en organizaciones que no utilicen niveles de acceso." #, fuzzy msgid "OVERWRITE_COLUMN_DATA_QUESTION" diff --git a/locale/fr_CA/LC_MESSAGES/django.mo b/locale/fr_CA/LC_MESSAGES/django.mo index bb0f74f724..1691fb455a 100644 Binary files a/locale/fr_CA/LC_MESSAGES/django.mo and b/locale/fr_CA/LC_MESSAGES/django.mo differ diff --git a/locale/fr_CA/LC_MESSAGES/django.po b/locale/fr_CA/LC_MESSAGES/django.po index d17750669b..141b0db10d 100644 --- a/locale/fr_CA/LC_MESSAGES/django.po +++ b/locale/fr_CA/LC_MESSAGES/django.po @@ -1,15 +1,14 @@ msgid "" msgstr "" -"Project-Id-Version: PACKAGE VERSION\n" -"Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-10-04 14:26-0700\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: FULL NAME \n" -"Language-Team: LANGUAGE \n" -"Language: \n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" +"X-Generator: lokalise.com\n" +"Project-Id-Version: SEED Platform\n" +"PO-Revision-Date: 2026-03-10 20:51\n" +"Last-Translator: lokalise.com\n" +"Language-Team: lokalise.com\n\n" +"Language: fr_CA\n" msgid "# Of Files" msgstr "Nombre de fichiers" @@ -2970,7 +2969,7 @@ msgid "ORGANIZATION_WUI_DISPLAY_UNITS" msgstr "Affichage de l'unité de mesure de l'intensité d'utilisation de l'eau (WUI)" msgid "ORG_MATCH_MERGE_LINK_WARNING" -msgstr "AVERTISSEMENT: Vous avez modifié les critères selon lesquels les données de votre organisation seront mises en correspondance et fusionnées. Une fois qu'une colonne a été ajoutée aux critères de correspondance d'une organisation, elle ne peut pas être supprimée tant que l'organisation dispose d'un inventaire." +msgstr "AVERTISSEMENT: Vous avez modifié les critères de correspondance et de fusion des données de votre organisation. Si l’organisation utilise plusieurs niveaux d’accès: une fois qu’une colonne a été ajoutée aux critères de correspondance, elle ne peut plus être supprimée tant que l’organisation dispose de stocks. Les critères de correspondance peuvent être supprimés dans les organisations n’utilisant pas de niveaux d’accès." msgid "OVERWRITE_COLUMN_DATA_QUESTION" msgstr "Écraser les données?" diff --git a/seed/static/seed/js/controllers/column_settings_controller.js b/seed/static/seed/js/controllers/column_settings_controller.js index 29c6638f30..2ae2e010fb 100644 --- a/seed/static/seed/js/controllers/column_settings_controller.js +++ b/seed/static/seed/js/controllers/column_settings_controller.js @@ -15,6 +15,8 @@ angular.module('SEED.controller.column_settings', []).controller('column_setting 'auth_payload', 'columns_service', 'modified_service', + 'organization_service', + 'uploader_service', 'spinner_utility', 'urls', 'naturalSort', @@ -33,6 +35,8 @@ angular.module('SEED.controller.column_settings', []).controller('column_setting auth_payload, columns_service, modified_service, + organization_service, + uploader_service, spinner_utility, urls, naturalSort, @@ -125,7 +129,7 @@ angular.module('SEED.controller.column_settings', []).controller('column_setting $scope.matching_status = (column) => { if (column.is_extra_data) return 'ineligible'; - if ($scope.org.inventory_count && initial_matching_ids.includes(column.id)) return 'locked'; + if ($scope.org.access_level_names.length > 1 && $scope.org.inventory_count && initial_matching_ids.includes(column.id)) return 'locked'; return 'eligible'; }; @@ -249,6 +253,30 @@ angular.module('SEED.controller.column_settings', []).controller('column_setting $scope.isModified = () => modified_service.isModified(); + $scope.complete_column_update = function () { + const matching_criteria_changed = _.find(_.values(diff), (delta) => _.has(delta, 'is_matching_criteria')); + + if (matching_criteria_changed) { + // reset the spinner and run whole org match merge link + spinner_utility.show(undefined, $('.display')[0]); + + organization_service.match_merge_link($scope.org.id, $scope.inventory_type).then((response) => { + uploader_service.check_progress_loop( + response.progress_key, + 0, + 1, + (response) => { + organization_service.get_match_merge_link_result($scope.org.id, response.unique_id).then(() => column_update_complete()); + }, + () => {}, + { progress: 0 } + ); + }); + } else { + column_update_complete(); + } + }; + // Table Sorting const default_sort_toggle = () => { $scope.column_sort = 'default'; @@ -360,6 +388,7 @@ angular.module('SEED.controller.column_settings', []).controller('column_setting } modified_service.resetModified(); + $scope.modal_instance.dismiss(); $state.reload(); }; @@ -378,14 +407,7 @@ angular.module('SEED.controller.column_settings', []).controller('column_setting return; } - const modal_instance = $scope.open_confirm_column_settings_modal(); - modal_instance.result - .then((data) => { - column_update_complete(data); - }) - .catch(() => { - // User cancelled - }); + $scope.modal_instance = $scope.open_confirm_column_settings_modal(); }; $scope.open_create_column_modal = () => $uibModal.open({ @@ -412,7 +434,8 @@ angular.module('SEED.controller.column_settings', []).controller('column_setting columns_service: () => columns_service, spinner_utility: () => spinner_utility, table_name: () => ($scope.inventory_type === 'properties' ? 'PropertyState' : 'TaxLotState'), - $q: () => $q + $q: () => $q, + complete_column_update: () => $scope.complete_column_update } }); diff --git a/seed/static/seed/js/controllers/confirm_column_settings_modal_controller.js b/seed/static/seed/js/controllers/confirm_column_settings_modal_controller.js index 0ab123c7d9..01b12d690d 100644 --- a/seed/static/seed/js/controllers/confirm_column_settings_modal_controller.js +++ b/seed/static/seed/js/controllers/confirm_column_settings_modal_controller.js @@ -15,7 +15,9 @@ angular.module('SEED.controller.confirm_column_settings_modal', []).controller(' 'proposed_changes', 'columns_service', 'spinner_utility', + 'uiGridGroupingConstants', '$q', + 'complete_column_update', '$interval', 'uploader_service', 'table_name', @@ -35,7 +37,9 @@ angular.module('SEED.controller.confirm_column_settings_modal', []).controller(' proposed_changes, columns_service, spinner_utility, + uiGridGroupingConstants, $q, + complete_column_update, $interval, uploader_service, table_name, @@ -220,6 +224,7 @@ angular.module('SEED.controller.confirm_column_settings_modal', []).controller(' 0, 1, (response) => { + complete_column_update(); $scope.result = `${response.message} in ${$scope.elapsed}`; $scope.state = 'done'; $interval.cancel($scope.interval); @@ -260,6 +265,205 @@ angular.module('SEED.controller.confirm_column_settings_modal', []).controller(' } }; + // Preview + // Agg function returning last value of matching criteria field (all should be the same if they match) + $scope.matching_field_value = function (aggregation, fieldValue) { + aggregation.value = fieldValue; + }; + + function prioritize_sort(grid, sortColumns) { + // To maintain grouping while giving users the ability to have some sorting, + // matching columns are given top priority followed by the hidden linking ID column. + // Lastly, non-matching columns are given next priority so that users can sort within a grouped set. + if (sortColumns.length > 1) { + const matching_cols = _.filter(sortColumns, (col) => col.colDef.is_matching_criteria); + const linking_id_col = _.find(sortColumns, ['name', 'id']); + const remaining_cols = _.filter(sortColumns, (col) => !col.colDef.is_matching_criteria && !(col.name === 'id')); + sortColumns = matching_cols.concat(linking_id_col).concat(remaining_cols); + _.forEach(sortColumns, (col, index) => { + col.sort.priority = index; + }); + } + } + + // Takes raw cycle-partitioned records and returns array of cycle-aware records + const format_preview_records = (raw_inventory) => _.reduce(raw_inventory, (all_records, records, cycle_id) => { + const cycle = _.find($scope.cycles, { id: parseInt(cycle_id, 10) }); + _.forEach(records, (record) => { + record.cycle_name = cycle.name; + record.cycle_start = cycle.start; + all_records.push(record); + }); + return all_records; + }, []); + + // Builds preview columns using non-extra_data columns + function build_preview_columns() { + // create copy in order to not change original column objects. + const preview_column_defs = _.reject(_.cloneDeep($scope.columns), 'is_extra_data'); + const default_min_width = 50; + const autopin_width = 100; + const column_def_defaults = { + headerCellFilter: 'translate', + minWidth: default_min_width, + width: 125, + groupingShowAggregationMenu: false + }; + + _.map(preview_column_defs, (col) => { + const options = {}; + if (col.data_type === 'datetime') { + options.cellFilter = 'date:\'yyyy-MM-dd h:mm a\''; + options.filter = inventory_service.dateFilter(); + } else { + options.filter = inventory_service.combinedFilter(); + } + + // For matching criteria values, always pin left and show values in aggregate rows. + if ($scope.proposed_matching_criteria_columns.includes(col.column_name)) { + col.pinnedLeft = true; + + // Help indicate matching columns are given preferred sort priority + col.displayName += '*'; + options.headerCellClass = 'matching-column-header'; + + options.customTreeAggregationFn = $scope.matching_field_value; + options.width = autopin_width; + } + return _.defaults(col, options, column_def_defaults); + }); + + // Grouping Settings + preview_column_defs.unshift( + { + displayName: 'Linking ID', + grouping: { groupPriority: 0 }, + name: 'id', + sort: { priority: 0, direction: 'desc' }, + pinnedLeft: true, + visible: false, + suppressRemoveSort: true, // since grouping relies on sorting + minWidth: default_min_width, + width: autopin_width + }, + { + name: 'cycle_name', + displayName: 'Cycle', + pinnedLeft: true, + treeAggregationType: uiGridGroupingConstants.aggregation.COUNT, + customTreeAggregationFinalizerFn: function (aggregation) { + aggregation.rendered = `total cycles: ${aggregation.value}`; + }, + minWidth: default_min_width, + width: autopin_width, + groupingShowAggregationMenu: false + }, + { + name: 'cycle_start', + displayName: 'Cycle Start', + cellFilter: 'date:\'yyyy-MM-dd\'', + filter: inventory_service.dateFilter(), + type: 'date', + sort: { priority: 1, direction: 'asc' }, + pinnedLeft: true, + minWidth: default_min_width, + width: autopin_width, + groupingShowAggregationMenu: false + } + ); + + return preview_column_defs; + } + + // Initialize preview table as empty for now. + $scope.match_merge_link_preview = { + data: 'data', + enableColumnResizing: true, + enableFiltering: true, + onRegisterApi: function (gridApi) { + $scope.gridApi = gridApi; + + // used to allow filtering for child branches of grouping tree + $scope.gridApi.table_category = 'year-over-year'; + + $scope.gridApi.core.on.filterChanged($scope, () => { + // This is a workaround for losing the state of expanded rows during filtering. + _.delay($scope.gridApi.treeBase.expandAllRows, 500); + }); + + // Prioritized to maintain grouping. + $scope.gridApi.core.on.sortChanged($scope, prioritize_sort); + } + }; + + // Preview Loading Helpers + function build_proposed_matching_columns(result) { + // Summarize proposed matching_criteria_columns for pinning and to create preview + const criteria_additions = _.filter($scope.change_summary_data, (change) => change.is_matching_criteria); + const criteria_removals = _.filter($scope.change_summary_data, (change) => change.is_matching_criteria === false); + + $scope.criteria_changes = { + add: _.map(criteria_additions, 'column_name'), + remove: _.map(criteria_removals, 'column_name') + }; + + let base_and_add; + if ($scope.inventory_type === 'properties') { + base_and_add = _.union(result.PropertyState, $scope.criteria_changes.add); + } else { + base_and_add = _.union(result.TaxLotState, $scope.criteria_changes.add); + } + $scope.proposed_matching_criteria_columns = _.difference(base_and_add, $scope.criteria_changes.remove); + } + + function build_preview(summary) { + $scope.data = format_preview_records(summary); + $scope.preview_columns = build_preview_columns(); + $scope.match_merge_link_preview.columnDefs = $scope.preview_columns; + } + + function preview_loading_complete() { + $scope.preview_loading = false; + spinner_utility.hide(); + } + + function get_preview() { + // Use new proposed matching_criteria_columns to request a preview then render this preview. + const spinner_options = { + scale: 0.40, + position: 'relative', + left: '100%' + }; + spinner_utility.show(spinner_options, $('#spinner_placeholder')[0]); + + organization_service.match_merge_link_preview($scope.org_id, $scope.inventory_type, $scope.criteria_changes) + .then((response) => { + uploader_service.check_progress_loop( + response.progress_key, + 0, + 1, + (completion_notice) => { + organization_service.get_match_merge_link_result($scope.org_id, completion_notice.unique_id) + .then(build_preview) + .then(preview_loading_complete); + }, + () => { /* Do nothing */ }, + { progress: 0 } + ); + }); + } + + // Get and Show Preview (If matching criteria changes exist.) + $scope.matching_criteria_exists = _.find(_.values($scope.change_summary_data), (delta) => _.has(delta, 'is_matching_criteria')); + + if ($scope.matching_criteria_exists) { + $scope.preview_loading = true; + + organization_service.matching_criteria_columns($scope.org_id) + .then(build_proposed_matching_columns) + .then(get_preview); + } + $scope.setRunningState = () => { $scope.eta = $scope.etaFn(); if ($scope.eta) { diff --git a/seed/static/seed/js/services/organization_service.js b/seed/static/seed/js/services/organization_service.js index d80c61eca8..13bd14229b 100644 --- a/seed/static/seed/js/services/organization_service.js +++ b/seed/static/seed/js/services/organization_service.js @@ -164,6 +164,22 @@ angular.module('SEED.service.organization', []).factory('organization_service', organization_factory.reset_all_passwords = (org_id) => $http.post(`/api/v3/organizations/${org_id}/reset_all_passwords/`).then((response) => response.data); + organization_factory.match_merge_link = (org_id, inventory_type) => $http.post(`/api/v3/organizations/${org_id}/match_merge_link/`, { + inventory_type + }).then((response) => response.data); + + organization_factory.geocoding_columns = (org_id) => $http.get(`/api/v3/organizations/${org_id}/geocoding_columns/`).then((response) => response.data); + + organization_factory.match_merge_link_preview = (org_id, inventory_type, criteria_change_columns) => $http.post(`/api/v3/organizations/${org_id}/match_merge_link_preview/`, { + inventory_type, + add: criteria_change_columns.add, + remove: criteria_change_columns.remove + }).then((response) => response.data); + + organization_factory.get_match_merge_link_result = (org_id, match_merge_link_id) => $http.get(`/api/v3/organizations/${org_id}/match_merge_link_result/?match_merge_link_id=${match_merge_link_id}`).then((response) => response.data); + + organization_factory.reset_all_passwords = (org_id) => $http.post(`/api/v3/organizations/${org_id}/reset_all_passwords/`).then((response) => response.data); + organization_factory.insert_sample_data = (org_id) => $http.get(`/api/v3/organizations/${org_id}/insert_sample_data/`).then((response) => response.data); /** diff --git a/seed/static/seed/locales/en_US.json b/seed/static/seed/locales/en_US.json index 96f92dc847..6a06a636a2 100644 --- a/seed/static/seed/locales/en_US.json +++ b/seed/static/seed/locales/en_US.json @@ -965,7 +965,7 @@ "OR": "OR", "ORGANIZATION_WATER_USE_DISPLAY_UNITS": "Measurement unit display for Water Use", "ORGANIZATION_WUI_DISPLAY_UNITS": "Measurement unit display for Water Use Intensity (WUI)", - "ORG_MATCH_MERGE_LINK_WARNING": "WARNING: You have modified the criteria that your organization's data will be matched and merged against. Once a column has been added to an organization's matching criteria it cannot be removed while the organization has inventory.", + "ORG_MATCH_MERGE_LINK_WARNING": "WARNING: You have modified the criteria that your organization's data will be matched and merged against. If the organization is using more than one access levels: once a column has been added to an organization's matching criteria it cannot be removed while the organization has inventory. Matching criteria can be removed in organizations not using access levels.", "OVERWRITE_COLUMN_DATA_QUESTION": "Overwrite data?", "Occupied Floor Area": "Occupied Floor Area", "Only Show Populated Columns": "Only Show Populated Columns", diff --git a/seed/static/seed/locales/es.json b/seed/static/seed/locales/es.json index cca7122dec..c60801e12d 100644 --- a/seed/static/seed/locales/es.json +++ b/seed/static/seed/locales/es.json @@ -965,7 +965,7 @@ "OR": "O", "ORGANIZATION_WATER_USE_DISPLAY_UNITS": "Visualización de la unidad de medida del consumo de agua", "ORGANIZATION_WUI_DISPLAY_UNITS": "Visualización de la unidad de medida para la intensidad de uso del agua (WUI)", - "ORG_MATCH_MERGE_LINK_WARNING": "ADVERTENCIA: Ha modificado los criterios con los que se compararán y combinarán los datos de su organización. Una vez que se ha añadido una columna a los criterios de correspondencia de una organización, no se puede eliminar mientras la organización tenga inventario.", + "ORG_MATCH_MERGE_LINK_WARNING": "ADVERTENCIA: Ha modificado los criterios con los que se compararán y fusionarán los datos de su organización. Si la organización utiliza más de un nivel de acceso, una vez que se haya añadido una columna a los criterios de coincidencia de una organización, no se podrá eliminar mientras la organización tenga inventario. Los criterios de coincidencia se pueden eliminar en organizaciones que no utilicen niveles de acceso.", "OVERWRITE_COLUMN_DATA_QUESTION": "¿Sobreescribir datos?", "Occupied Floor Area": "Superficie ocupada", "Only Show Populated Columns": "Mostrar sólo columnas rellenas", diff --git a/seed/static/seed/locales/fr_CA.json b/seed/static/seed/locales/fr_CA.json index 2104abcb3f..614248e33f 100644 --- a/seed/static/seed/locales/fr_CA.json +++ b/seed/static/seed/locales/fr_CA.json @@ -965,7 +965,7 @@ "OR": "OU", "ORGANIZATION_WATER_USE_DISPLAY_UNITS": "Affichage de l'unité de mesure pour l'utilisation de l'eau", "ORGANIZATION_WUI_DISPLAY_UNITS": "Affichage de l'unité de mesure de l'intensité d'utilisation de l'eau (WUI)", - "ORG_MATCH_MERGE_LINK_WARNING": "AVERTISSEMENT: Vous avez modifié les critères selon lesquels les données de votre organisation seront mises en correspondance et fusionnées. Une fois qu'une colonne a été ajoutée aux critères de correspondance d'une organisation, elle ne peut pas être supprimée tant que l'organisation dispose d'un inventaire.", + "ORG_MATCH_MERGE_LINK_WARNING": "AVERTISSEMENT: Vous avez modifié les critères de correspondance et de fusion des données de votre organisation. Si l’organisation utilise plusieurs niveaux d’accès: une fois qu’une colonne a été ajoutée aux critères de correspondance, elle ne peut plus être supprimée tant que l’organisation dispose de stocks. Les critères de correspondance peuvent être supprimés dans les organisations n’utilisant pas de niveaux d’accès.", "OVERWRITE_COLUMN_DATA_QUESTION": "Écraser les données?", "Occupied Floor Area": "Surface occupée", "Only Show Populated Columns": "Afficher uniquement les colonnes remplies", diff --git a/seed/static/seed/partials/confirm_column_settings_modal.html b/seed/static/seed/partials/confirm_column_settings_modal.html index 641ff047fd..4a09dbea28 100644 --- a/seed/static/seed/partials/confirm_column_settings_modal.html +++ b/seed/static/seed/partials/confirm_column_settings_modal.html @@ -1,10 +1,14 @@ -
-

Review Changes

+
+

Review Changes

+

+ Matching Criteria Preview +

-
+
Review Changes
-
+
@@ -32,7 +36,11 @@

Review Changes

ORG_MATCH_MERGE_LINK_WARNING
-
+
+ +
+
+
+
+
-
-
-
-
-
-

REHASH_TEXT

-

LONG_OP_CONFIRM_TEXT

-
-
-
- -

WAIT_TEXT_1

-
-
- -

WAIT_TEXT_2

+
+
+
+
+ Warning: REHASH_TEXT
+ LONG_OP_CONFIRM_TEXT
-
- {$ progressBar.status_message $} - - - - - - - - - -
Elapsed:{$ elapsed $}
Estimated Completion:{$ eta $}
-
-
{$ result $}
- -
+
+
+
+
+
+
+ +

WAIT_TEXT_1

+
+
+ +

WAIT_TEXT_2

+
+
+ {$ progressBar.status_message $} + + + + + + + + + +
Elapsed:{$ elapsed $}
Estimated Completion:{$ eta $}
+
+
{$ result $}
+
+
+ +
+ +
diff --git a/seed/tests/test_organization_views.py b/seed/tests/test_organization_views.py index 927371d015..50676936a8 100644 --- a/seed/tests/test_organization_views.py +++ b/seed/tests/test_organization_views.py @@ -11,17 +11,21 @@ from xlrd import open_workbook from seed.data_importer.models import ImportFile, ImportRecord +from seed.data_importer.tasks import geocode_and_match_buildings_task from seed.landing.models import SEEDUser as User from seed.lib.mcm.reader import ROW_DELIMITER -from seed.models import Column, Cycle, FilterGroup, PropertyView +from seed.lib.progress_data.progress_data import ProgressData +from seed.models import ASSESSED_RAW, DATA_STATE_MAPPING, Column, Cycle, FilterGroup, PropertyView from seed.models import StatusLabel as Label from seed.test_helpers.fake import ( FakeCycleFactory, FakePropertyFactory, FakePropertyStateFactory, FakePropertyViewFactory, + FakeTaxLotStateFactory, ) from seed.tests.util import AccessLevelBaseTestCase, DataMappingBaseTestCase +from seed.utils.cache import get_cache_raw from seed.utils.organizations import create_organization @@ -284,3 +288,228 @@ def test_filtered_report(self): url += f"&filter_group_id={self.office_filter_group.id}" resp = self.client.get(url, content_type="application/json") assert resp.json()["data"]["property_counts"][0]["num_properties"] == 1 + + def test_whole_org_match_merge_link_endpoint_properties(self): + url = reverse("api:v3:organizations-match-merge-link", args=[self.org.id]) + post_params = json.dumps({"inventory_type": "properties"}) + raw_result = self.client.post(url, post_params, content_type="application/json") + + self.assertEqual(200, raw_result.status_code) + + raw_content = json.loads(raw_result.content) + + identifier = ProgressData.from_key(raw_content["progress_key"]).data["unique_id"] + result_key = f"org_match_merge_link_result__{identifier}" + summary = get_cache_raw(result_key) + + summary_keys = list(summary.keys()) + + self.assertCountEqual(["PropertyState", "TaxLotState"], summary_keys) + + # try to get result using results endpoint + get_result_url = ( + reverse("api:v3:organizations-match-merge-link-result", args=[self.org.id]) + "?match_merge_link_id=" + str(identifier) + ) + + get_result_raw_response = self.client.get(get_result_url) + summary = json.loads(get_result_raw_response.content) + + summary_keys = list(summary.keys()) + + self.assertCountEqual(["PropertyState", "TaxLotState"], summary_keys) + + def test_whole_org_match_merge_link_endpoint_taxlots(self): + url = reverse("api:v3:organizations-match-merge-link", args=[self.org.id]) + post_params = json.dumps({"inventory_type": "taxlots"}) + raw_result = self.client.post(url, post_params, content_type="application/json") + + self.assertEqual(200, raw_result.status_code) + + raw_content = json.loads(raw_result.content) + + identifier = ProgressData.from_key(raw_content["progress_key"]).data["unique_id"] + result_key = f"org_match_merge_link_result__{identifier}" + summary = get_cache_raw(result_key) + + summary_keys = list(summary.keys()) + + self.assertCountEqual(["PropertyState", "TaxLotState"], summary_keys) + + # try to get result using results endpoint + get_result_url = ( + reverse("api:v3:organizations-match-merge-link-result", args=[self.org.id]) + "?match_merge_link_id=" + str(identifier) + ) + + get_result_raw_response = self.client.get(get_result_url) + summary = json.loads(get_result_raw_response.content) + + summary_keys = list(summary.keys()) + + self.assertCountEqual(["PropertyState", "TaxLotState"], summary_keys) + + +class TestOrganizationPreviewViews(DataMappingBaseTestCase): + def setUp(self): + selfvars = self.set_up(ASSESSED_RAW) + self.user, self.org, self.import_file_1, self.import_record_1, self.cycle_1 = selfvars + + cycle_factory = FakeCycleFactory(organization=self.org, user=self.user) + self.cycle_2 = cycle_factory.get_cycle(name="Cycle 2") + self.import_record_2, self.import_file_2 = self.create_import_file(self.user, self.org, self.cycle_2) + + user_details = {"username": "test_user@demo.com", "password": "test_pass", "email": "test_user@demo.com"} + + self.client.login(**user_details) + + self.property_state_factory = FakePropertyStateFactory(organization=self.org) + self.taxlot_state_factory = FakeTaxLotStateFactory(organization=self.org) + + def test_whole_org_match_merge_link_preview_endpoint_invalid_columns(self): + url = reverse("api:v3:organizations-match-merge-link-preview", args=[self.org.id]) + post_params = json.dumps({"inventory_type": "properties", "add": ["DNE col 1"], "remove": ["DNE col 2"]}) + raw_result = self.client.post(url, post_params, content_type="application/json") + self.assertEqual(404, raw_result.status_code) + + def test_whole_org_match_merge_link_preview_endpoint_properties(self): + # Cycle 1 / ImportFile 1 - Create 1 property + base_property_details = { + "pm_property_id": "1st Non-Match Set", + "city": "City 1", + "property_name": "Match Set", + "import_file_id": self.import_file_1.id, + "data_state": DATA_STATE_MAPPING, + "no_default_data": False, + "raw_access_level_instance_id": self.org.root.id, + } + print(PropertyView.objects.count()) + print() + + ps_1 = self.property_state_factory.get_property_state(**base_property_details) + + self.import_file_1.mapping_done = True + self.import_file_1.save() + geocode_and_match_buildings_task(self.import_file_1.id) + + # Cycle 2 / ImportFile 2 - Create 1 unlinked property + base_property_details["pm_property_id"] = "2nd Non-Match Set" + base_property_details["property_name"] = "Match Set" + base_property_details["import_file_id"] = self.import_file_2.id + ps_2 = self.property_state_factory.get_property_state(**base_property_details) + + self.import_file_2.mapping_done = True + self.import_file_2.save() + geocode_and_match_buildings_task(self.import_file_2.id) + + # Check there doesn't exist links + print(PropertyView.objects.count()) + print(ps_1.propertyview_set.values("id")) + print(ps_2.propertyview_set.values("id")) + self.assertNotEqual(ps_1.propertyview_set.first().property_id, ps_2.propertyview_set.first().property_id) + + url = reverse("api:v3:organizations-match-merge-link-preview", args=[self.org.id]) + post_params = json.dumps({"inventory_type": "properties", "add": ["property_name"], "remove": ["pm_property_id"]}) + raw_result = self.client.post(url, post_params, content_type="application/json") + + # Check there *still* doesn't exist links + self.assertNotEqual(ps_1.propertyview_set.first().property_id, ps_2.propertyview_set.first().property_id) + + self.assertEqual(200, raw_result.status_code) + + raw_content = json.loads(raw_result.content) + + identifier = ProgressData.from_key(raw_content["progress_key"]).data["unique_id"] + result_key = f"org_match_merge_link_result__{identifier}" + raw_summary = get_cache_raw(result_key) + summary = {str(k): v for k, v in raw_summary.items() if v} # ignore empty cycles + + # Check format of summary + self.assertCountEqual([str(self.cycle_1.id), str(self.cycle_2.id)], summary.keys()) + + # Check that preview shows links would be created + self.assertEqual(summary[str(self.cycle_1.id)][0]["id"], summary[str(self.cycle_2.id)][0]["id"]) + + # try to get result using results endpoint + get_result_url = ( + reverse("api:v3:organizations-match-merge-link-result", args=[self.org.id]) + "?match_merge_link_id=" + str(identifier) + ) + + get_result_raw_response = self.client.get(get_result_url) + raw_summary = json.loads(get_result_raw_response.content) + + summary = {str(k): v for k, v in raw_summary.items() if v} # ignore empty cycles + + # Check format of summary + self.assertCountEqual([str(self.cycle_1.id), str(self.cycle_2.id)], summary.keys()) + + # Check that preview shows links would be created + self.assertEqual(summary[str(self.cycle_1.id)][0]["id"], summary[str(self.cycle_2.id)][0]["id"]) + + def test_whole_org_match_merge_link_preview_endpoint_taxlots(self): + # Cycle 1 / ImportFile 1 - Create 1 taxlot + base_taxlot_details = { + "jurisdiction_tax_lot_id": "1st Non-Match Set", + "city": "City 1", + "district": "Match Set", + "import_file_id": self.import_file_1.id, + "data_state": DATA_STATE_MAPPING, + "no_default_data": False, + "raw_access_level_instance_id": self.org.root.id, + } + + tls_1 = self.taxlot_state_factory.get_taxlot_state(**base_taxlot_details) + + self.import_file_1.mapping_done = True + self.import_file_1.save() + geocode_and_match_buildings_task(self.import_file_1.id) + + # Cycle 2 / ImportFile 2 - Create 1 unlinked taxlot + base_taxlot_details["jurisdiction_tax_lot_id"] = "2nd Non-Match Set" + base_taxlot_details["district"] = "Match Set" + base_taxlot_details["import_file_id"] = self.import_file_2.id + tls_2 = self.taxlot_state_factory.get_taxlot_state(**base_taxlot_details) + + self.import_file_2.mapping_done = True + self.import_file_2.save() + geocode_and_match_buildings_task(self.import_file_2.id) + + # Check there doesn't exist links + self.assertNotEqual(tls_1.taxlotview_set.first().taxlot_id, tls_2.taxlotview_set.first().taxlot_id) + + url = reverse("api:v3:organizations-match-merge-link-preview", args=[self.org.id]) + post_params = json.dumps({"inventory_type": "taxlots", "add": ["district"], "remove": ["jurisdiction_tax_lot_id"]}) + raw_result = self.client.post(url, post_params, content_type="application/json") + + # Check there *still* doesn't exist links + self.assertNotEqual(tls_1.taxlotview_set.first().taxlot_id, tls_2.taxlotview_set.first().taxlot_id) + + self.assertEqual(200, raw_result.status_code) + + raw_content = json.loads(raw_result.content) + + identifier = ProgressData.from_key(raw_content["progress_key"]).data["unique_id"] + result_key = f"org_match_merge_link_result__{identifier}" + raw_summary = get_cache_raw(result_key) + + summary = {str(k): v for k, v in raw_summary.items() if v} # ignore empty cycles + + # Check format of summary + self.assertCountEqual([str(self.cycle_1.id), str(self.cycle_2.id)], summary.keys()) + + # Check that preview shows links would be created + self.assertEqual(summary[str(self.cycle_1.id)][0]["id"], summary[str(self.cycle_2.id)][0]["id"]) + + # try to get result using results endpoint + get_result_url = ( + reverse("api:v3:organizations-match-merge-link-result", args=[self.org.id]) + "?match_merge_link_id=" + str(identifier) + ) + + get_result_raw_response = self.client.get(get_result_url) + raw_summary = json.loads(get_result_raw_response.content) + + summary = {str(k): v for k, v in raw_summary.items() if v} # ignore empty cycles + + # Check format of summary + self.assertCountEqual([str(self.cycle_1.id), str(self.cycle_2.id)], summary.keys()) + + # Check that preview shows links would be created + self.assertEqual(summary[str(self.cycle_1.id)][0]["id"], summary[str(self.cycle_2.id)][0]["id"]) diff --git a/seed/views/v3/organizations.py b/seed/views/v3/organizations.py index 4a13c75530..8e28ade3ae 100644 --- a/seed/views/v3/organizations.py +++ b/seed/views/v3/organizations.py @@ -12,8 +12,10 @@ from io import BytesIO from numbers import Number from pathlib import Path +from random import randint import numpy as np +from celery import shared_task from django.conf import settings from django.contrib.auth.forms import PasswordResetForm from django.contrib.postgres.aggregates.general import ArrayAgg @@ -35,6 +37,7 @@ from seed.data_importer.tasks import save_raw_data from seed.decorators import ajax_request from seed.landing.models import SEEDUser as User +from seed.lib.progress_data.progress_data import ProgressData from seed.lib.superperms.orgs.decorators import has_hierarchy_access, has_perm from seed.lib.superperms.orgs.models import ( ROLE_MEMBER, @@ -70,9 +73,10 @@ from seed.serializers.report_configurations import ReportConfigurationSerializer from seed.utils.api import api_endpoint from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.cache import get_cache_raw, set_cache_raw from seed.utils.encrypt import decrypt, encrypt from seed.utils.geocode import geocode_buildings -from seed.utils.match import match_merge_link +from seed.utils.match import get_matching_criteria_column_names, match_merge_link, whole_org_match_merge_link from seed.utils.merge import merge_properties from seed.utils.organizations import create_organization, create_suborganization, set_default_2fa_method from seed.utils.properties import pair_unpair_property_taxlot @@ -237,6 +241,19 @@ def _dict_org_brief(request, organizations): return orgs +def _get_match_merge_link_key(identifier): + return f"org_match_merge_link_result__{identifier}" + + +@shared_task(serializer="pickle", ignore_result=True) +def cache_match_merge_link_result(summary, identifier, progress_key): + result_key = _get_match_merge_link_key(identifier) + set_cache_raw(result_key, summary) + + progress_data = ProgressData.from_key(progress_key) + progress_data.finish_with_success() + + class OrganizationViewSet(viewsets.ViewSet): # allow using `pk` in url path for authorization (i.e., for has_perm) authz_org_id_kwarg = "pk" @@ -335,6 +352,20 @@ def column_mappings(self, request, pk=None): else: return JsonResponse({"status": "success"}) + def _start_whole_org_match_merge_link(self, org_id, state_class_name, proposed_columns=[]): + identifier = randint(100, 100000) + result_key = _get_match_merge_link_key(identifier) + set_cache_raw(result_key, {}) + + progress_data = ProgressData(func_name="org_match_merge_link", unique_id=identifier) + progress_data.delete() + + whole_org_match_merge_link.apply_async( + args=(org_id, state_class_name, proposed_columns), link=cache_match_merge_link_result.s(identifier, progress_data.key) + ) + + return progress_data.key + @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_boolean_field( @@ -923,6 +954,127 @@ def matching_criteria_columns(self, request, pk=None): return JsonResponse(matching_criteria_column_names) + @swagger_auto_schema( + request_body=AutoSchemaHelper.schema_factory( + {"inventory_type": "string"}, + required=["inventory_type"], + description='Properties:\n- inventory_type: either "properties" or "taxlots"', + ) + ) + @method_decorator( + [ + api_endpoint, + ajax_request, + has_perm("requires_member"), + ] + ) + @action(detail=True, methods=["POST"]) + def match_merge_link(self, request, pk=None): + """ + Run match_merge_link for an org. + """ + inventory_type = request.data.get("inventory_type", None) + if inventory_type not in ["properties", "taxlots"]: + return JsonResponse( + {"status": "error", "message": 'Provided inventory type should either be "properties" or "taxlots".'}, + status=status.HTTP_404_NOT_FOUND, + ) + + try: + org = Organization.objects.get(pk=pk) + except ObjectDoesNotExist: + return JsonResponse( + {"status": "error", "message": "Could not retrieve organization at pk = " + str(pk)}, status=status.HTTP_404_NOT_FOUND + ) + + state_class_name = "PropertyState" if inventory_type == "properties" else "TaxLotState" + + progress_key = self._start_whole_org_match_merge_link(org.id, state_class_name) + + return JsonResponse({"progress_key": progress_key}) + + @swagger_auto_schema( + request_body=AutoSchemaHelper.schema_factory( + { + "inventory_type": "string", + "add": ["string"], + "remove": ["string"], + }, + required=["inventory_type"], + description="Properties:\n" + '- inventory_type: either "properties" or "taxlots"\n' + "- add: list of column names\n" + "- remove: list of column names", + ) + ) + @method_decorator( + [ + api_endpoint, + ajax_request, + has_perm("requires_member"), + ] + ) + @action(detail=True, methods=["POST"]) + def match_merge_link_preview(self, request, pk=None): + """ + Run match_merge_link preview for an org and record type. + """ + inventory_type = request.data.get("inventory_type", None) + if inventory_type not in ["properties", "taxlots"]: + return JsonResponse( + {"status": "error", "message": 'Provided inventory type should either be "properties" or "taxlots".'}, + status=status.HTTP_404_NOT_FOUND, + ) + + try: + org = Organization.objects.get(pk=pk) + except ObjectDoesNotExist: + return JsonResponse( + {"status": "error", "message": "Could not retrieve organization at pk = " + str(pk)}, status=status.HTTP_404_NOT_FOUND + ) + + state_class_name = "PropertyState" if inventory_type == "properties" else "TaxLotState" + + current_columns = get_matching_criteria_column_names(org.id, state_class_name) + + add = set(request.data.get("add", [])) + remove = set(request.data.get("remove", [])) + + provided_columns = Column.objects.filter(column_name__in=add.union(remove), organization_id=org.id, table_name=state_class_name) + if provided_columns.count() != (len(add) + len(remove)): + return JsonResponse({"status": "error", "message": "Invalid column names provided."}, status=status.HTTP_404_NOT_FOUND) + + proposed_columns = current_columns.union(add).difference(remove) + + progress_key = self._start_whole_org_match_merge_link(org.id, state_class_name, list(proposed_columns)) + + return JsonResponse({"progress_key": progress_key}) + + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_integer_field("match_merge_link_id", required=True, description="ID of match merge link")] + ) + @method_decorator( + [ + api_endpoint, + ajax_request, + has_perm("requires_member"), + ] + ) + @action(detail=True, methods=["GET"]) + def match_merge_link_result(self, request, pk=None): + try: + Organization.objects.get(pk=pk) + except ObjectDoesNotExist: + return JsonResponse( + {"status": "error", "message": "Could not retrieve organization at pk = " + str(pk)}, status=status.HTTP_404_NOT_FOUND + ) + + identifier = request.query_params["match_merge_link_id"] + result_key = _get_match_merge_link_key(identifier) + + # using unsafe serialization b/c the result might not be a dict + return JsonResponse(get_cache_raw(result_key), safe=False) + @method_decorator( [ api_endpoint,