From 0815334ec707ce5804a60f4b3985e7b7f6caa304 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Wed, 11 Oct 2023 15:14:02 -0600 Subject: [PATCH 01/49] draft enketoHelper tests --- www/__tests__/enketoHelper.test.ts | 70 ++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 www/__tests__/enketoHelper.test.ts diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts new file mode 100644 index 000000000..537accb79 --- /dev/null +++ b/www/__tests__/enketoHelper.test.ts @@ -0,0 +1,70 @@ +import { getInstanceStr, filterByNameAndVersion } from '../js/survey/enketo/enketoHelper'; +import { mockBEMUserCache } from '../__mocks__/cordovaMocks'; + +mockBEMUserCache(); + + +/** + * @param xmlModel the blank XML model response for the survey + * @param opts object with options like 'prefilledSurveyResponse' or 'prefillFields' + * @returns XML string of an existing or prefilled model response, or null if no response is available + */ +it('gets the model response, if avaliable, or returns null', ()=> { + const xmlModel = '\n \n \n \n \n \n \n ;'; + const filled = '\n \n car\n \n \n \n \n ;'; + const opts = {"prefilledSurveyResponse": filled}; + const opts2 = {"prefillFields": {"travel_mode" : "car"}}; + + //if no xmlModel, returns null + expect(getInstanceStr(null, opts)).toBe(null); + + //if there is a prefilled survey, return it + expect(getInstanceStr(xmlModel, opts)).toBe(filled); + + //if there is a model and fields, return prefilled + // expect(getInstanceStr(xmlModel, opts2)).toBe(filled); + //TODO - figure out how to use the helper function with JEST -- getElementsByTagName is empty? should it be? + + //if none of those things, also return null + expect(getInstanceStr(xmlModel, {})).toBe(null); +}); + +/** + * @param surveyName the name of the survey (e.g. "TimeUseSurvey") + * @param enketoForm the Form object from enketo-core that contains this survey + * @param appConfig the dynamic config file for the app + * @param opts object with SurveyOptions like 'timelineEntry' or 'dataKey' + * @returns Promise of the saved result, or an Error if there was a problem + */ +// export function saveResponse(surveyName: string, enketoForm: Form, appConfig, opts: SurveyOptions) { +it('gets the saved result or throws an error', () => { + +}); + +/* +* We retrieve all the records every time instead of caching because of the +* usage pattern. We assume that the demographic survey is edited fairly +* rarely, so loading it every time will likely do a bunch of unnecessary work. +* Loading it on demand seems like the way to go. If we choose to experiment +* with incremental updates, we may want to revisit this. +*/ +// export function loadPreviousResponseForSurvey(dataKey: string) { +it('loads the previous response to a given survey', () => { + +}); + +/** + * filterByNameAndVersion filter the survey answers by survey name and their version. + * The version for filtering is specified in enketo survey `compatibleWith` config. + * The stored survey answer version must be greater than or equal to `compatibleWith` to be included. + * @param {string} name survey name (defined in enketo survey config) + * @param {EnketoAnswer[]} answers survey answers + * (usually retrieved by calling UnifiedDataLoader.getUnifiedMessagesForInterval('manual/survey_response', tq)) method. + * @return {Promise} filtered survey answers + */ +it('filters the survey answers by their name and version', () => { + const surveyName = "TimeUseSurvey"; + const answers = []; + expect(filterByNameAndVersion(surveyName, answers)).resolves.toBe([]); + +}); From 43b8386acf595c1a9296d032c691feafb7e5255f Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Wed, 11 Oct 2023 15:15:38 -0600 Subject: [PATCH 02/49] convert answer.js into enketoHelper moving the methods form answer.js into enketoHelper as a part of the services migration --- www/js/survey/enketo/enketoHelper.ts | 192 +++++++++++++++++++++++++-- 1 file changed, 184 insertions(+), 8 deletions(-) diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index 6e9147cf8..84c057658 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -2,7 +2,10 @@ import { getAngularService } from "../../angular-react-helper"; import { Form } from 'enketo-core'; import { XMLParser } from 'fast-xml-parser'; import i18next from 'i18next'; -import { logDebug } from "../../plugin/logger"; +import MessageFormat from 'messageformat'; +import { logDebug, logInfo } from "../../plugin/logger"; +import { getConfig } from '../../config/dynamicConfig'; +import { DateTime } from "luxon"; export type PrefillFields = {[key: string]: string}; @@ -14,6 +17,129 @@ export type SurveyOptions = { dataKey?: string; }; +type EnketoAnswerData = { + label: string; //display label (this value is use for displaying on the button) + ts: string; //the timestamp at which the survey was filled out (in seconds) + fmt_time: string; //the formatted timestamp at which the survey was filled out + name: string; //survey name + version: string; //survey version + xmlResponse: string; //survey answer XML string + jsonDocResponse: string; //survey answer JSON object +} + +type EnketoAnswer = { + data: EnketoAnswerData; //answer data + labels: [{[labelField:string]: string}]; //virtual labels (populated by populateLabels method) +} + +type EnketoSurveyConfig = { + [surveyName:string]: { + formPath: string + labelFields: string[]; + version: number; + compatibleWith: number; + } +} + +/** @type {EnketoSurveyConfig} _config */ +//TODO find a more appropriate way to store this +let _config: EnketoSurveyConfig; + +const LABEL_FUNCTIONS = { + UseLabelTemplate : async (xmlDoc: XMLDocument, name: string) => { + let configSurveys = await _lazyLoadConfig(); + + const config = configSurveys[name]; // config for this survey + const lang = i18next.resolvedLanguage; + const labelTemplate = config.labelTemplate?.[lang]; + + if (!labelTemplate) return "Answered"; // no template given in config + if (!config.labelVars) return labelTemplate; // if no vars given, nothing to interpolate, + // so we return the unaltered template + + // gather vars that will be interpolated into the template according to the survey config + const labelVars = {} + for (let lblVar in config.labelVars) { + const fieldName = config.labelVars[lblVar].key; + let fieldStr = _getAnswerByTagName(xmlDoc, fieldName); + if (fieldStr == '') fieldStr = null; + if (config.labelVars[lblVar].type == 'length') { + const fieldMatches = fieldStr?.split(' '); + labelVars[lblVar] = fieldMatches?.length || 0; + } else { + throw new Error(`labelVar type ${config.labelVars[lblVar].type } is not supported!`) + } + } + + // use MessageFormat interpolate the label template with the label vars + const mf = new MessageFormat(lang); + const label = mf.compile(labelTemplate)(labelVars); + return label.replace(/^[ ,]+|[ ,]+$/g, ''); // trim leading and trailing spaces and commas + } +} + +/** + * _getAnswerByTagName lookup for the survey answer by tag name form the given XML document. + * @param {XMLDocument} xmlDoc survey answer object + * @param {string} tagName tag name + * @returns {string} answer string. If not found, return "\" + */ + function _getAnswerByTagName(xmlDoc: XMLDocument, tagName: string) { + const vals = xmlDoc.getElementsByTagName(tagName); + const val = vals.length ? vals[0].innerHTML : null; + if (!val) return ''; + return val; +} + +/** + * _lazyLoadConfig load enketo survey config. If already loaded, return the cached config + * @returns {Promise} enketo survey config + */ +function _lazyLoadConfig() { + if (_config !== undefined) { + return Promise.resolve(_config); + } + return getConfig().then((newConfig) => { + logInfo("Resolved UI_CONFIG_READY promise in enketoHelper, filling in templates"); + _config = newConfig.survey_info.surveys; + return _config; + }) +} + +/** + * filterByNameAndVersion filter the survey answers by survey name and their version. + * The version for filtering is specified in enketo survey `compatibleWith` config. + * The stored survey answer version must be greater than or equal to `compatibleWith` to be included. + * @param {string} name survey name (defined in enketo survey config) + * @param {EnketoAnswer[]} answers survey answers + * (usually retrieved by calling UnifiedDataLoader.getUnifiedMessagesForInterval('manual/survey_response', tq)) method. + * @return {Promise} filtered survey answers + */ + export function filterByNameAndVersion(name: string, answers: EnketoAnswer[]) { + return _lazyLoadConfig().then(config => { + console.log("filtering by name and version,", name, config, answers); + answers.filter(answer => + answer.data.name === name && + answer.data.version >= config[name].compatibleWith + )} + ); +} + +/** + * resolve answer label for the survey + * @param {string} name survey name + * @param {XMLDocument} xmlDoc survey answer object + * @returns {Promise} label string Promise + */ +function resolveLabel(name: string, xmlDoc: XMLDocument) { + // Some studies may want a custom label function for their survey. + // Those can be added in LABEL_FUNCTIONS with the survey name as the key. + // Otherwise, UseLabelTemplate will create a label using the template in the config + if (LABEL_FUNCTIONS[name]) + return LABEL_FUNCTIONS[name](xmlDoc); + return LABEL_FUNCTIONS.UseLabelTemplate(xmlDoc, name); +} + /** * @param xmlModel the blank XML model to be prefilled * @param prefillFields an object with keys that are the XML tag names and values that are the values to be prefilled @@ -21,7 +147,7 @@ export type SurveyOptions = { */ function getXmlWithPrefills(xmlModel: string, prefillFields: PrefillFields) { if (!prefillFields) return null; - const xmlParser = new window.DOMParser(); + const xmlParser = new DOMParser(); const xmlDoc = xmlParser.parseFromString(xmlModel, 'text/xml'); for (const [tagName, value] of Object.entries(prefillFields)) { @@ -46,6 +172,57 @@ export function getInstanceStr(xmlModel: string, opts: SurveyOptions): string|nu return null; } +/** + * resolve timestamps label from the survey response + * @param {XMLDocument} xmlDoc survey answer object + * @param {object} trip trip object + * @returns {object} object with `start_ts` and `end_ts` + * - null if no timestamps are resolved + * - undefined if the timestamps are invalid + */ + function resolveTimestamps(xmlDoc, timelineEntry) { + // check for Date and Time fields + const startDate = xmlDoc.getElementsByTagName('Start_date')?.[0]?.innerHTML; + let startTime = xmlDoc.getElementsByTagName('Start_time')?.[0]?.innerHTML; + const endDate = xmlDoc.getElementsByTagName('End_date')?.[0]?.innerHTML; + let endTime = xmlDoc.getElementsByTagName('End_time')?.[0]?.innerHTML; + + // if any of the fields are missing, return null + if (!startDate || !startTime || !endDate || !endTime) return null; + + const timezone = timelineEntry.start_local_dt?.timezone + || timelineEntry.enter_local_dt?.timezone + || timelineEntry.end_local_dt?.timezone + || timelineEntry.exit_local_dt?.timezone; + // split by + or - to get time without offset + startTime = startTime.split(/\-|\+/)[0]; + endTime = endTime.split(/\-|\+/)[0]; + + let additionStartTs = DateTime.fromISO(startDate + "T" + startTime, {zone: timezone}).valueOf(); + let additionEndTs = DateTime.fromISO(endDate + "T" + endTime, {zone: timezone}).valueOf(); + + if (additionStartTs > additionEndTs) { + return undefined; // if the start time is after the end time, this is an invalid response + } + + /* Enketo survey time inputs are only precise to the minute, while trips/places are precise to + the millisecond. To avoid precision issues, we will check if the start/end timestamps from + the survey response are within the same minute as the start/end or enter/exit timestamps. + If so, we will use the exact trip/place timestamps */ + const entryStartTs = timelineEntry.start_ts || timelineEntry.enter_ts; + const entryEndTs = timelineEntry.end_ts || timelineEntry.exit_ts; + if (additionStartTs - (additionStartTs % 60) == entryStartTs - (entryStartTs % 60)) + additionStartTs = entryStartTs; + if (additionEndTs - (additionEndTs % 60) == entryEndTs - (entryEndTs % 60)) + additionEndTs = entryEndTs; + + // return unix timestamps in seconds + return { + start_ts: additionStartTs, + end_ts: additionEndTs + }; +} + /** * @param surveyName the name of the survey (e.g. "TimeUseSurvey") * @param enketoForm the Form object from enketo-core that contains this survey @@ -54,13 +231,13 @@ export function getInstanceStr(xmlModel: string, opts: SurveyOptions): string|nu * @returns Promise of the saved result, or an Error if there was a problem */ export function saveResponse(surveyName: string, enketoForm: Form, appConfig, opts: SurveyOptions) { - const EnketoSurveyAnswer = getAngularService('EnketoSurveyAnswer'); + // const EnketoSurveyAnswer = getAngularService('EnketoSurveyAnswer'); const xmlParser = new window.DOMParser(); const xmlResponse = enketoForm.getDataStr(); const xmlDoc = xmlParser.parseFromString(xmlResponse, 'text/xml'); const xml2js = new XMLParser({ignoreAttributes: false, attributeNamePrefix: 'attr'}); const jsonDocResponse = xml2js.parse(xmlResponse); - return EnketoSurveyAnswer.resolveLabel(surveyName, xmlDoc).then(rsLabel => { + return resolveLabel(surveyName, xmlDoc).then(rsLabel => { const data: any = { label: rsLabel, name: surveyName, @@ -69,15 +246,14 @@ export function saveResponse(surveyName: string, enketoForm: Form, appConfig, op jsonDocResponse, }; if (opts.timelineEntry) { - let timestamps = EnketoSurveyAnswer.resolveTimestamps(xmlDoc, opts.timelineEntry); + let timestamps = resolveTimestamps(xmlDoc, opts.timelineEntry); if (timestamps === undefined) { // timestamps were resolved, but they are invalid return new Error(i18next.t('survey.enketo-timestamps-invalid')); //"Timestamps are invalid. Please ensure that the start time is before the end time."); } // if timestamps were not resolved from the survey, we will use the trip or place timestamps - timestamps ||= opts.timelineEntry; - data.start_ts = timestamps.start_ts || timestamps.enter_ts; - data.end_ts = timestamps.end_ts || timestamps.exit_ts; + data.start_ts = timestamps.start_ts || opts.timelineEntry.enter_ts; + data.end_ts = timestamps.end_ts || opts.timelineEntry.exit_ts; // UUID generated using this method https://stackoverflow.com/a/66332305 data.match_id = URL.createObjectURL(new Blob([])).slice(-36); } else { From 9840b5a2914fcfc11ae932b47cf4563bded3e96f Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Wed, 11 Oct 2023 15:22:08 -0600 Subject: [PATCH 03/49] exclude platforms When testing, I was getting an error from Jest about duplicate modules, one of which was in platforms. This change resolves that error --- jest.config.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jest.config.json b/jest.config.json index 71bc5f5ca..8d194ffd0 100644 --- a/jest.config.json +++ b/jest.config.json @@ -13,6 +13,9 @@ "transformIgnorePatterns": [ "/node_modules/(?!(@react-native|react-native|react-native-vector-icons))" ], + "modulePathIgnorePatterns": [ + "/platforms/" + ], "moduleNameMapper": { "^react-native$": "react-native-web" } From 1eaf8b8e592d42a980168574452b0b1af98b54a9 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Wed, 11 Oct 2023 15:58:34 -0600 Subject: [PATCH 04/49] add more tests additional test for filterByNameAndVersion fake answers have been constructed to be filtered --- www/__tests__/enketoHelper.test.ts | 52 ++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index 537accb79..bdb55e112 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -63,8 +63,54 @@ it('loads the previous response to a given survey', () => { * @return {Promise} filtered survey answers */ it('filters the survey answers by their name and version', () => { - const surveyName = "TimeUseSurvey"; - const answers = []; - expect(filterByNameAndVersion(surveyName, answers)).resolves.toBe([]); + //no answers -> no filtered answers + expect(filterByNameAndVersion("TimeUseSurvey", [])).resolves.toBe([]); + const answer = [ + { + data: { + label: "Activity", //display label (this value is use for displaying on the button) + ts: "100000000", //the timestamp at which the survey was filled out (in seconds) + fmt_time: "12:36", //the formatted timestamp at which the survey was filled out + name: "TimeUseSurvey", //survey name + version: "1", //survey version + xmlResponse: "", //survey answer XML string + jsonDocResponse: "this is my json object" //survey answer JSON object + }, + labels: {labelField: "goodbye"} //TODO learn more about answer type + } + ]; + + //one answer -> that answer + expect(filterByNameAndVersion("TimeUseSurvey", answer)).resolves.toBe(answer); + + const answers = [ + { + data: { + label: "Activity", //display label (this value is use for displaying on the button) + ts: "100000000", //the timestamp at which the survey was filled out (in seconds) + fmt_time: "12:36", //the formatted timestamp at which the survey was filled out + name: "TimeUseSurvey", //survey name + version: "1", //survey version + xmlResponse: "", //survey answer XML string + jsonDocResponse: "this is my json object" //survey answer JSON object + }, + labels: {labelField: "goodbye"} + }, + { + data: { + label: "Activity", //display label (this value is use for displaying on the button) + ts: "100000000", //the timestamp at which the survey was filled out (in seconds) + fmt_time: "12:36", //the formatted timestamp at which the survey was filled out + name: "OtherSurvey", //survey name + version: "1", //survey version + xmlResponse: "", //survey answer XML string + jsonDocResponse: "this is my json object" //survey answer JSON object + }, + labels: {labelField: "goodbye"} + } + ]; + + //several answers -> only the one that has a name match + expect(filterByNameAndVersion("TimeUseSurvey", answers)).resolves.toBe(answer); }); From 74b847156c420c095ad891b9f15e487e461ec3e8 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Wed, 11 Oct 2023 16:12:56 -0600 Subject: [PATCH 05/49] remove answer.js completely remove answer.js and all references to it, replace references with references to enketoHelper.ts --- www/__tests__/enketoHelper.test.ts | 4 - www/index.js | 1 - www/js/survey/enketo/answer.js | 193 ------------------ .../survey/enketo/enketo-add-note-button.js | 8 +- www/js/survey/enketo/enketo-trip-button.js | 8 +- www/js/survey/enketo/enketoHelper.ts | 6 +- 6 files changed, 10 insertions(+), 210 deletions(-) delete mode 100644 www/js/survey/enketo/answer.js diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index bdb55e112..f601f0f27 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -57,10 +57,6 @@ it('loads the previous response to a given survey', () => { * filterByNameAndVersion filter the survey answers by survey name and their version. * The version for filtering is specified in enketo survey `compatibleWith` config. * The stored survey answer version must be greater than or equal to `compatibleWith` to be included. - * @param {string} name survey name (defined in enketo survey config) - * @param {EnketoAnswer[]} answers survey answers - * (usually retrieved by calling UnifiedDataLoader.getUnifiedMessagesForInterval('manual/survey_response', tq)) method. - * @return {Promise} filtered survey answers */ it('filters the survey answers by their name and version', () => { //no answers -> no filtered answers diff --git a/www/index.js b/www/index.js index 89c3a5e26..30070245f 100644 --- a/www/index.js +++ b/www/index.js @@ -21,7 +21,6 @@ import './js/survey/multilabel/infinite_scroll_filters.js'; import './js/survey/multilabel/multi-label-ui.js'; import './js/diary.js'; import './js/diary/services.js'; -import './js/survey/enketo/answer.js'; import './js/survey/enketo/infinite_scroll_filters.js'; import './js/survey/enketo/enketo-trip-button.js'; import './js/survey/enketo/enketo-add-note-button.js'; diff --git a/www/js/survey/enketo/answer.js b/www/js/survey/enketo/answer.js deleted file mode 100644 index e6077c479..000000000 --- a/www/js/survey/enketo/answer.js +++ /dev/null @@ -1,193 +0,0 @@ -import angular from 'angular'; -import MessageFormat from 'messageformat'; -import { getConfig } from '../../config/dynamicConfig'; - -angular.module('emission.survey.enketo.answer', ['ionic']) -.factory('EnketoSurveyAnswer', function($http) { - /** - * @typedef EnketoAnswerData - * @type {object} - * @property {string} label - display label (this value is use for displaying on the button) - * @property {string} ts - the timestamp at which the survey was filled out (in seconds) - * @property {string} fmt_time - the formatted timestamp at which the survey was filled out - * @property {string} name - survey name - * @property {string} version - survey version - * @property {string} xmlResponse - survey answer XML string - * @property {string} jsonDocResponse - survey answer JSON object - */ - - /** - * @typedef EnketoAnswer - * @type {object} - * @property {EnketoAnswerData} data - answer data - * @property {{[labelField:string]: string}} [labels] - virtual labels (populated by populateLabels method) - */ - - /** - * @typedef EnketoSurveyConfig - * @type {{ - * [surveyName:string]: { - * formPath: string; - * labelFields: string[]; - * version: number; - * compatibleWith: number; - * } - * }} - */ - - const LABEL_FUNCTIONS = { - UseLabelTemplate: (xmlDoc, name) => { - - return _lazyLoadConfig().then(configSurveys => { - - const config = configSurveys[name]; // config for this survey - const lang = i18next.resolvedLanguage; - const labelTemplate = config.labelTemplate?.[lang]; - - if (!labelTemplate) return "Answered"; // no template given in config - if (!config.labelVars) return labelTemplate; // if no vars given, nothing to interpolate, - // so we return the unaltered template - - // gather vars that will be interpolated into the template according to the survey config - const labelVars = {} - for (let lblVar in config.labelVars) { - const fieldName = config.labelVars[lblVar].key; - let fieldStr = _getAnswerByTagName(xmlDoc, fieldName); - if (fieldStr == '') fieldStr = null; - if (config.labelVars[lblVar].type == 'length') { - const fieldMatches = fieldStr?.split(' '); - labelVars[lblVar] = fieldMatches?.length || 0; - } else { - throw new Error(`labelVar type ${config.labelVars[lblVar].type } is not supported!`) - } - } - - // use MessageFormat interpolate the label template with the label vars - const mf = new MessageFormat(lang); - const label = mf.compile(labelTemplate)(labelVars); - return label.replace(/^[ ,]+|[ ,]+$/g, ''); // trim leading and trailing spaces and commas - }) - } - }; - - /** @type {EnketoSurveyConfig} _config */ - let _config; - - /** - * _getAnswerByTagName lookup for the survey answer by tag name form the given XML document. - * @param {XMLDocument} xmlDoc survey answer object - * @param {string} tagName tag name - * @returns {string} answer string. If not found, return "\" - */ - function _getAnswerByTagName(xmlDoc, tagName) { - const vals = xmlDoc.getElementsByTagName(tagName); - const val = vals.length ? vals[0].innerHTML : null; - if (!val) return ''; - return val; - } - - /** - * _lazyLoadConfig load enketo survey config. If already loaded, return the cached config - * @returns {Promise} enketo survey config - */ - function _lazyLoadConfig() { - if (_config !== undefined) { - return Promise.resolve(_config); - } - return getConfig().then((newConfig) => { - Logger.log("Resolved UI_CONFIG_READY promise in answer.js, filling in templates"); - _config = newConfig.survey_info.surveys; - return _config; - }) - } - - /** - * filterByNameAndVersion filter the survey answers by survey name and their version. - * The version for filtering is specified in enketo survey `compatibleWith` config. - * The stored survey answer version must be greater than or equal to `compatibleWith` to be included. - * @param {string} name survey name (defined in enketo survey config) - * @param {EnketoAnswer[]} answers survey answers - * (usually retrieved by calling UnifiedDataLoader.getUnifiedMessagesForInterval('manual/survey_response', tq)) method. - * @return {Promise} filtered survey answers - */ - function filterByNameAndVersion(name, answers) { - return _lazyLoadConfig().then(config => - answers.filter(answer => - answer.data.name === name && - answer.data.version >= config[name].compatibleWith - ) - ); - } - - /** - * resolve answer label for the survey - * @param {string} name survey name - * @param {XMLDocument} xmlDoc survey answer object - * @returns {Promise} label string Promise - */ - function resolveLabel(name, xmlDoc) { - // Some studies may want a custom label function for their survey. - // Those can be added in LABEL_FUNCTIONS with the survey name as the key. - // Otherwise, UseLabelTemplate will create a label using the template in the config - if (LABEL_FUNCTIONS[name]) - return LABEL_FUNCTIONS[name](xmlDoc); - return LABEL_FUNCTIONS.UseLabelTemplate(xmlDoc, name); - } - - /** - * resolve timestamps label from the survey response - * @param {XMLDocument} xmlDoc survey answer object - * @param {object} trip trip object - * @returns {object} object with `start_ts` and `end_ts` - * - null if no timestamps are resolved - * - undefined if the timestamps are invalid - */ - function resolveTimestamps(xmlDoc, timelineEntry) { - // check for Date and Time fields - const startDate = xmlDoc.getElementsByTagName('Start_date')?.[0]?.innerHTML; - let startTime = xmlDoc.getElementsByTagName('Start_time')?.[0]?.innerHTML; - const endDate = xmlDoc.getElementsByTagName('End_date')?.[0]?.innerHTML; - let endTime = xmlDoc.getElementsByTagName('End_time')?.[0]?.innerHTML; - - // if any of the fields are missing, return null - if (!startDate || !startTime || !endDate || !endTime) return null; - - const timezone = timelineEntry.start_local_dt?.timezone - || timelineEntry.enter_local_dt?.timezone - || timelineEntry.end_local_dt?.timezone - || timelineEntry.exit_local_dt?.timezone; - // split by + or - to get time without offset - startTime = startTime.split(/\-|\+/)[0]; - endTime = endTime.split(/\-|\+/)[0]; - - let additionStartTs = moment.tz(startDate+'T'+startTime, timezone).unix(); - let additionEndTs = moment.tz(endDate+'T'+endTime, timezone).unix(); - - if (additionStartTs > additionEndTs) { - return undefined; // if the start time is after the end time, this is an invalid response - } - - /* Enketo survey time inputs are only precise to the minute, while trips/places are precise to - the millisecond. To avoid precision issues, we will check if the start/end timestamps from - the survey response are within the same minute as the start/end or enter/exit timestamps. - If so, we will use the exact trip/place timestamps */ - const entryStartTs = timelineEntry.start_ts || timelineEntry.enter_ts; - const entryEndTs = timelineEntry.end_ts || timelineEntry.exit_ts; - if (additionStartTs - (additionStartTs % 60) == entryStartTs - (entryStartTs % 60)) - additionStartTs = entryStartTs; - if (additionEndTs - (additionEndTs % 60) == entryEndTs - (entryEndTs % 60)) - additionEndTs = entryEndTs; - - // return unix timestamps in seconds - return { - start_ts: additionStartTs, - end_ts: additionEndTs - }; - } - - return { - filterByNameAndVersion, - resolveLabel, - resolveTimestamps, - }; -}); diff --git a/www/js/survey/enketo/enketo-add-note-button.js b/www/js/survey/enketo/enketo-add-note-button.js index 49f7747f6..6dc6be7e5 100644 --- a/www/js/survey/enketo/enketo-add-note-button.js +++ b/www/js/survey/enketo/enketo-add-note-button.js @@ -3,12 +3,12 @@ */ import angular from 'angular'; +import { filterByNameAndVersion } from './enketoHelper' angular.module('emission.survey.enketo.add-note-button', ['emission.services', - 'emission.survey.enketo.answer', 'emission.survey.inputmatcher']) -.factory("EnketoNotesButtonService", function(InputMatcher, EnketoSurveyAnswer, Logger, $timeout) { +.factory("EnketoNotesButtonService", function(InputMatcher, Logger, $timeout) { var enbs = {}; console.log("Creating EnketoNotesButtonService"); enbs.SINGLE_KEY="NOTES"; @@ -33,9 +33,9 @@ angular.module('emission.survey.enketo.add-note-button', * Embed 'inputType' to the timelineEntry. */ enbs.extractResult = function(results) { - const resultsPromises = [EnketoSurveyAnswer.filterByNameAndVersion(enbs.timelineEntrySurveyName, results)]; + const resultsPromises = [filterByNameAndVersion(enbs.timelineEntrySurveyName, results)]; if (enbs.timelineEntrySurveyName != enbs.placeSurveyName) { - resultsPromises.push(EnketoSurveyAnswer.filterByNameAndVersion(enbs.placeSurveyName, results)); + resultsPromises.push(filterByNameAndVersion(enbs.placeSurveyName, results)); } return Promise.all(resultsPromises); }; diff --git a/www/js/survey/enketo/enketo-trip-button.js b/www/js/survey/enketo/enketo-trip-button.js index 6e710435f..623137450 100644 --- a/www/js/survey/enketo/enketo-trip-button.js +++ b/www/js/survey/enketo/enketo-trip-button.js @@ -12,11 +12,11 @@ */ import angular from 'angular'; +import { filterByNameAndVersion } from "./enketoHelper"; angular.module('emission.survey.enketo.trip.button', - ['emission.survey.enketo.answer', - 'emission.survey.inputmatcher']) -.factory("EnketoTripButtonService", function(InputMatcher, EnketoSurveyAnswer, Logger, $timeout) { + ['emission.survey.inputmatcher']) +.factory("EnketoTripButtonService", function(InputMatcher, Logger, $timeout) { var etbs = {}; console.log("Creating EnketoTripButtonService"); etbs.key = "manual/trip_user_input"; @@ -26,7 +26,7 @@ angular.module('emission.survey.enketo.trip.button', /** * Embed 'inputType' to the trip. */ - etbs.extractResult = (results) => EnketoSurveyAnswer.filterByNameAndVersion('TripConfirmSurvey', results); + etbs.extractResult = (results) => filterByNameAndVersion('TripConfirmSurvey', results); etbs.processManualInputs = function(manualResults, resultMap) { if (manualResults.length > 1) { diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index 84c057658..83085ebd7 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -116,12 +116,11 @@ function _lazyLoadConfig() { * @return {Promise} filtered survey answers */ export function filterByNameAndVersion(name: string, answers: EnketoAnswer[]) { - return _lazyLoadConfig().then(config => { - console.log("filtering by name and version,", name, config, answers); + return _lazyLoadConfig().then(config => answers.filter(answer => answer.data.name === name && answer.data.version >= config[name].compatibleWith - )} + ) ); } @@ -231,7 +230,6 @@ export function getInstanceStr(xmlModel: string, opts: SurveyOptions): string|nu * @returns Promise of the saved result, or an Error if there was a problem */ export function saveResponse(surveyName: string, enketoForm: Form, appConfig, opts: SurveyOptions) { - // const EnketoSurveyAnswer = getAngularService('EnketoSurveyAnswer'); const xmlParser = new window.DOMParser(); const xmlResponse = enketoForm.getDataStr(); const xmlDoc = xmlParser.parseFromString(xmlResponse, 'text/xml'); From d259799020bd604f852fc19476af6470e965e41c Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Thu, 12 Oct 2023 10:25:44 -0600 Subject: [PATCH 06/49] adding tests for resolveTimestamps resolveTimestamps is a helper function to saveResponse, but still contains a fair amount of its own logic. Testing the edge cases for this function ensures that it will behave as expected within the larger context --- www/__tests__/enketoHelper.test.ts | 21 ++++++++++++++++++++- www/js/survey/enketo/enketoHelper.ts | 2 +- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index f601f0f27..6ba78d170 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -1,4 +1,4 @@ -import { getInstanceStr, filterByNameAndVersion } from '../js/survey/enketo/enketoHelper'; +import { getInstanceStr, filterByNameAndVersion, resolveTimestamps } from '../js/survey/enketo/enketoHelper'; import { mockBEMUserCache } from '../__mocks__/cordovaMocks'; mockBEMUserCache(); @@ -29,6 +29,25 @@ it('gets the model response, if avaliable, or returns null', ()=> { expect(getInstanceStr(xmlModel, {})).toBe(null); }); +//resolve timestamps +it('resolves the timestamps', () => { + const xmlParser = new window.DOMParser(); + const timelineEntry = { end_local_dt: {timezone: "America/Los_Angeles"}, start_ts: 1469492672.928242, end_ts: 1469493031}; + + //missing data returns null + const missingData = ' 2016-08-28 2016-07-25 17:30:31.000-06:00 '; + const missDataDoc = xmlParser.parseFromString(missingData, 'text/html'); + expect(resolveTimestamps(missDataDoc, timelineEntry)).toBeNull(); + //bad time returns undefined + const badTimes = ' 2016-08-28 2016-07-25 17:32:32.928-06:00 17:30:31.000-06:00 '; + const badTimeDoc = xmlParser.parseFromString(badTimes, 'text/xml'); + expect(resolveTimestamps(badTimeDoc, timelineEntry)).toBeUndefined(); + //good info returns unix start and end timestamps -- TODO : address precise vs less precise? + const timeSurvey = ' 2016-07-25 2016-07-25 17:24:32.928-06:00 17:30:31.000-06:00 '; + const xmlDoc = xmlParser.parseFromString(timeSurvey, 'text/xml'); + expect(resolveTimestamps(xmlDoc, timelineEntry)).toMatchObject({start_ts: 1469492672928, end_ts: 1469493031000}); +}); + /** * @param surveyName the name of the survey (e.g. "TimeUseSurvey") * @param enketoForm the Form object from enketo-core that contains this survey diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index 83085ebd7..99045c222 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -179,7 +179,7 @@ export function getInstanceStr(xmlModel: string, opts: SurveyOptions): string|nu * - null if no timestamps are resolved * - undefined if the timestamps are invalid */ - function resolveTimestamps(xmlDoc, timelineEntry) { + export function resolveTimestamps(xmlDoc, timelineEntry) { // check for Date and Time fields const startDate = xmlDoc.getElementsByTagName('Start_date')?.[0]?.innerHTML; let startTime = xmlDoc.getElementsByTagName('Start_time')?.[0]?.innerHTML; From 11a3df66eb4d9d15216800754af3a4c1d2dbdb95 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Thu, 12 Oct 2023 12:32:51 -0600 Subject: [PATCH 07/49] testing for _lazyLoadConfig adding a basic test for _loadLazyConfig, in order to ensure that my mock setup for that works, before moving into testing functions that depend on it --- www/__mocks__/cordovaMocks.ts | 27 +++++++++++++++++++++++ www/__tests__/enketoHelper.test.ts | 33 +++++++++++++++++++--------- www/js/survey/enketo/enketoHelper.ts | 4 ++-- 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/www/__mocks__/cordovaMocks.ts b/www/__mocks__/cordovaMocks.ts index 44c21677c..31e3e7bf4 100644 --- a/www/__mocks__/cordovaMocks.ts +++ b/www/__mocks__/cordovaMocks.ts @@ -85,6 +85,33 @@ export const mockBEMUserCache = () => { rs(messages.filter(m => m.key == key).map(m => m.value)); }, 100) ); + }, + getDocument: (key: string, withMetadata?: boolean) => { + // this was mocked specifically for enketoHelper's use, could be expanded if needed + const fakeSurveyConfig = { + survey_info: { + surveys: { + TimeUseSurvey: { compatibleWith: 1, + formPath: "https://raw.githubusercontent.com/sebastianbarry/nrel-openpath-deploy-configs/surveys-info-and-surveys-data/survey-resources/data-json/time-use-survey-form-v9.json", + labelTemplate: {en: " erea, plural, =0 {} other {# Employment/Education, } }{ da, plural, =0 {} other {# Domestic activities, }", + es: " erea, plural, =0 {} other {# Empleo/Educación, } }{ da, plural, =0 {} other {# Actividades domesticas, }"}, + labelVars: {da: {key: "Domestic_activities", type: "length"}, + erea: {key: "Employment_related_a_Education_activities", type:"length"}}, + version: 9} + } + } + } + + if(key == "config/app_ui_config"){ + return new Promise((rs, rj) => + setTimeout(() => { + rs(fakeSurveyConfig); + }, 100) + ); + } + else { + return null; + } } } window['cordova'] ||= {}; diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index 6ba78d170..54a147904 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -1,14 +1,27 @@ -import { getInstanceStr, filterByNameAndVersion, resolveTimestamps } from '../js/survey/enketo/enketoHelper'; +import { getInstanceStr, filterByNameAndVersion, resolveTimestamps, resolveLabel, _lazyLoadConfig} from '../js/survey/enketo/enketoHelper'; import { mockBEMUserCache } from '../__mocks__/cordovaMocks'; +import { mockLogger } from '../__mocks__/globalMocks'; mockBEMUserCache(); +mockLogger(); + +it('gets the survey config', async () => { + //this is aimed at testing my mock of the config + //mocked getDocument for the case of getting the config + let config = await _lazyLoadConfig(); + let mockSurveys = { + TimeUseSurvey: { compatibleWith: 1, + formPath: "https://raw.githubusercontent.com/sebastianbarry/nrel-openpath-deploy-configs/surveys-info-and-surveys-data/survey-resources/data-json/time-use-survey-form-v9.json", + labelTemplate: {en: " erea, plural, =0 {} other {# Employment/Education, } }{ da, plural, =0 {} other {# Domestic activities, }", + es: " erea, plural, =0 {} other {# Empleo/Educación, } }{ da, plural, =0 {} other {# Actividades domesticas, }"}, + labelVars: {da: {key: "Domestic_activities", type: "length"}, + erea: {key: "Employment_related_a_Education_activities", type:"length"}}, + version: 9} + } + // console.log(config); + expect(config).toMatchObject(mockSurveys); +}) - -/** - * @param xmlModel the blank XML model response for the survey - * @param opts object with options like 'prefilledSurveyResponse' or 'prefillFields' - * @returns XML string of an existing or prefilled model response, or null if no response is available - */ it('gets the model response, if avaliable, or returns null', ()=> { const xmlModel = '\n \n \n \n \n \n \n ;'; const filled = '\n \n car\n \n \n \n \n ;'; @@ -79,7 +92,7 @@ it('loads the previous response to a given survey', () => { */ it('filters the survey answers by their name and version', () => { //no answers -> no filtered answers - expect(filterByNameAndVersion("TimeUseSurvey", [])).resolves.toBe([]); + expect(filterByNameAndVersion("TimeUseSurvey", [])).resolves.toStrictEqual([]); const answer = [ { @@ -97,7 +110,7 @@ it('filters the survey answers by their name and version', () => { ]; //one answer -> that answer - expect(filterByNameAndVersion("TimeUseSurvey", answer)).resolves.toBe(answer); + expect(filterByNameAndVersion("TimeUseSurvey", answer)).resolves.toStrictEqual(answer); const answers = [ { @@ -127,5 +140,5 @@ it('filters the survey answers by their name and version', () => { ]; //several answers -> only the one that has a name match - expect(filterByNameAndVersion("TimeUseSurvey", answers)).resolves.toBe(answer); + expect(filterByNameAndVersion("TimeUseSurvey", answers)).resolves.toStrictEqual(answer); }); diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index 99045c222..c757ac72b 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -95,7 +95,7 @@ const LABEL_FUNCTIONS = { * _lazyLoadConfig load enketo survey config. If already loaded, return the cached config * @returns {Promise} enketo survey config */ -function _lazyLoadConfig() { +export function _lazyLoadConfig() { if (_config !== undefined) { return Promise.resolve(_config); } @@ -179,7 +179,7 @@ export function getInstanceStr(xmlModel: string, opts: SurveyOptions): string|nu * - null if no timestamps are resolved * - undefined if the timestamps are invalid */ - export function resolveTimestamps(xmlDoc, timelineEntry) { +export function resolveTimestamps(xmlDoc, timelineEntry) { // check for Date and Time fields const startDate = xmlDoc.getElementsByTagName('Start_date')?.[0]?.innerHTML; let startTime = xmlDoc.getElementsByTagName('Start_time')?.[0]?.innerHTML; From bc42845ecfb91e16355cce92a8bad3b3435ab0e7 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Thu, 12 Oct 2023 16:27:51 -0600 Subject: [PATCH 08/49] attempting to add more tests currently struggling with i18next and MessageFormat, as I can't get either of those mocked and working --- www/__mocks__/i18nextMocks.ts | 8 +++++++ www/__mocks__/messageFormatMocks.ts | 32 +++++++++++++++++++++++++ www/__tests__/enketoHelper.test.ts | 36 +++++++++++++++++++++++++--- www/js/survey/enketo/enketoHelper.ts | 2 +- 4 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 www/__mocks__/i18nextMocks.ts create mode 100644 www/__mocks__/messageFormatMocks.ts diff --git a/www/__mocks__/i18nextMocks.ts b/www/__mocks__/i18nextMocks.ts new file mode 100644 index 000000000..dc0d3f2b4 --- /dev/null +++ b/www/__mocks__/i18nextMocks.ts @@ -0,0 +1,8 @@ +const i18next = jest.createMockFromModule('i18next'); + +let resolvedLanugage; + +function _setUpLanguage(language) { + console.log("setting resolved language to ", language, " for testing"); + resolvedLanugage = language; +} diff --git a/www/__mocks__/messageFormatMocks.ts b/www/__mocks__/messageFormatMocks.ts new file mode 100644 index 000000000..f32c07ed4 --- /dev/null +++ b/www/__mocks__/messageFormatMocks.ts @@ -0,0 +1,32 @@ +//call signature MessageFormat.compile(templage)(vars); +//in - template an vars -- {... pca: 0, ...} +//out - 1 Personal Care, + +export default class MessageFormat{ + + constructor( locale: string ) { } + + compile(message: string) { + return (vars: {}) => { + let label = ""; + const brokenList = message.split("}{"); + console.log(brokenList); + + for (let key in vars) { + brokenList.forEach((item) => { + let brokenItem = item.split(","); + if(brokenItem[0] == key) { + let getLabel = brokenItem[2].split("#"); + console.log(getLabel); + label = vars[key] + " " + getLabel[1]; + return label; + } + }) + } + + } + } +} + +exports.MessageFormat = MessageFormat; + \ No newline at end of file diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index 54a147904..96475bd58 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -1,9 +1,15 @@ -import { getInstanceStr, filterByNameAndVersion, resolveTimestamps, resolveLabel, _lazyLoadConfig} from '../js/survey/enketo/enketoHelper'; +import { getInstanceStr, filterByNameAndVersion, resolveTimestamps, resolveLabel, _lazyLoadConfig, loadPreviousResponseForSurvey} from '../js/survey/enketo/enketoHelper'; import { mockBEMUserCache } from '../__mocks__/cordovaMocks'; import { mockLogger } from '../__mocks__/globalMocks'; +import i18next from "i18next"; + mockBEMUserCache(); mockLogger(); +// jest.mock('../__mocks__/messageFormatMocks'); +// jest.mock("i18next"); + +// global.i18next = { resolvedLanguage : "en" } it('gets the survey config', async () => { //this is aimed at testing my mock of the config @@ -18,7 +24,6 @@ it('gets the survey config', async () => { erea: {key: "Employment_related_a_Education_activities", type:"length"}}, version: 9} } - // console.log(config); expect(config).toMatchObject(mockSurveys); }) @@ -61,6 +66,30 @@ it('resolves the timestamps', () => { expect(resolveTimestamps(xmlDoc, timelineEntry)).toMatchObject({start_ts: 1469492672928, end_ts: 1469493031000}); }); +//resolve label +// it('resolves the label', async () => { +// i18next.init({ +// fallbackLng: 'en', +// debug: true +// }, (err, t) => { +// if (err) return console.log('something went wrong loading', err); +// t('key'); // -> same as i18next.t +// }); + +// console.log("language in tests", i18next.resolvedLanguage); +// const xmlParser = new window.DOMParser(); +// //have a custom survey label function TODO: we currently don't have custome label functions, but should test when we do + +// //no custom function, fallback to UseLabelTemplate +// const xmlString = ' option_1/Domestic_activities> option_2 '; +// const xmlDoc = xmlParser.parseFromString(xmlString, 'text/html'); + +// //if no template, returns "Answered" +// expect(await resolveLabel("TimeUseSurvey", xmlDoc)).toBe(""); +// //if no labelVars, returns template +// //else interpolates +// }); + /** * @param surveyName the name of the survey (e.g. "TimeUseSurvey") * @param enketoForm the Form object from enketo-core that contains this survey @@ -82,7 +111,8 @@ it('gets the saved result or throws an error', () => { */ // export function loadPreviousResponseForSurvey(dataKey: string) { it('loads the previous response to a given survey', () => { - + //not really sure if I can test this yet given that it relies on an angular service... + loadPreviousResponseForSurvey("manual/demographic_survey"); }); /** diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index c757ac72b..f72921582 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -130,7 +130,7 @@ export function _lazyLoadConfig() { * @param {XMLDocument} xmlDoc survey answer object * @returns {Promise} label string Promise */ -function resolveLabel(name: string, xmlDoc: XMLDocument) { +export function resolveLabel(name: string, xmlDoc: XMLDocument) { // Some studies may want a custom label function for their survey. // Those can be added in LABEL_FUNCTIONS with the survey name as the key. // Otherwise, UseLabelTemplate will create a label using the template in the config From 608d97ddaa9447076259905374a933e61d34146b Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 13 Oct 2023 09:29:04 -0600 Subject: [PATCH 09/49] update messageformat plugin the message format plugin moved! https://github.com/messageformat/messageformat/tree/main/packages/core --- package.cordovabuild.json | 2 +- package.serve.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.cordovabuild.json b/package.cordovabuild.json index b5d69872f..306362726 100644 --- a/package.cordovabuild.json +++ b/package.cordovabuild.json @@ -102,6 +102,7 @@ }, "dependencies": { "@havesource/cordova-plugin-push": "git+https://github.com/havesource/cordova-plugin-push.git#4.0.0-dev.0", + "@messageformat/core": "^3.2.0", "@react-navigation/native": "^6.1.7", "@react-navigation/stack": "^6.3.17", "@shopify/flash-list": "^1.3.1", @@ -151,7 +152,6 @@ "klaw-sync": "^6.0.0", "leaflet": "^1.9.4", "luxon": "^3.3.0", - "messageformat": "^2.3.0", "moment": "^2.29.4", "moment-timezone": "^0.5.43", "ng-i18next": "^1.0.7", diff --git a/package.serve.json b/package.serve.json index 57470bc2d..c5ad26404 100644 --- a/package.serve.json +++ b/package.serve.json @@ -49,6 +49,7 @@ "webpack-cli": "^5.0.1" }, "dependencies": { + "@messageformat/core": "^3.2.0", "@react-navigation/native": "^6.1.7", "@react-navigation/stack": "^6.3.17", "@shopify/flash-list": "^1.3.1", @@ -78,7 +79,6 @@ "klaw-sync": "^6.0.0", "leaflet": "^1.9.4", "luxon": "^3.3.0", - "messageformat": "^2.3.0", "moment": "^2.29.4", "moment-timezone": "^0.5.43", "ng-i18next": "^1.0.7", From 8b901f6294c387bf69b08f6ec3eb3ec3cde5b494 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 13 Oct 2023 10:12:46 -0600 Subject: [PATCH 10/49] start to configure i18next for tests setting __DEV__ to false in globals, so that it can be used throught the testing suit calling the i18n setup in the tests should work once we incorporate the React testing changes --- jest.config.json | 3 +++ www/__tests__/enketoHelper.test.ts | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/jest.config.json b/jest.config.json index 8d194ffd0..21ba92e12 100644 --- a/jest.config.json +++ b/jest.config.json @@ -18,5 +18,8 @@ ], "moduleNameMapper": { "^react-native$": "react-native-web" + }, + "globals" : { + "__DEV__": true } } diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index 96475bd58..367cfd8bc 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -2,7 +2,8 @@ import { getInstanceStr, filterByNameAndVersion, resolveTimestamps, resolveLabel import { mockBEMUserCache } from '../__mocks__/cordovaMocks'; import { mockLogger } from '../__mocks__/globalMocks'; -import i18next from "i18next"; +// import initializedI18next from '../js/i18nextInit'; +// window['i18next'] = initializedI18next; mockBEMUserCache(); mockLogger(); From cea96dd3b3ea9030a98f6b17358be6a83586ddff Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 13 Oct 2023 10:13:14 -0600 Subject: [PATCH 11/49] update the message format import --- www/js/survey/enketo/enketoHelper.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index f72921582..fa7d300e3 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -2,7 +2,7 @@ import { getAngularService } from "../../angular-react-helper"; import { Form } from 'enketo-core'; import { XMLParser } from 'fast-xml-parser'; import i18next from 'i18next'; -import MessageFormat from 'messageformat'; +import MessageFormat from '@messageformat/core'; import { logDebug, logInfo } from "../../plugin/logger"; import { getConfig } from '../../config/dynamicConfig'; import { DateTime } from "luxon"; From b7f6d68d5385f87ce0066aeabfc5fec831afa78c Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 13 Oct 2023 10:16:31 -0600 Subject: [PATCH 12/49] updates to testing --- www/__tests__/enketoHelper.test.ts | 40 ++++++++++++------------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index 367cfd8bc..ab3ef963e 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -68,28 +68,20 @@ it('resolves the timestamps', () => { }); //resolve label -// it('resolves the label', async () => { -// i18next.init({ -// fallbackLng: 'en', -// debug: true -// }, (err, t) => { -// if (err) return console.log('something went wrong loading', err); -// t('key'); // -> same as i18next.t -// }); - -// console.log("language in tests", i18next.resolvedLanguage); -// const xmlParser = new window.DOMParser(); -// //have a custom survey label function TODO: we currently don't have custome label functions, but should test when we do - -// //no custom function, fallback to UseLabelTemplate -// const xmlString = ' option_1/Domestic_activities> option_2 '; -// const xmlDoc = xmlParser.parseFromString(xmlString, 'text/html'); +it('resolves the label', async () => { + const xmlParser = new window.DOMParser(); -// //if no template, returns "Answered" -// expect(await resolveLabel("TimeUseSurvey", xmlDoc)).toBe(""); -// //if no labelVars, returns template -// //else interpolates -// }); + //have a custom survey label function TODO: we currently don't have custome label functions, but should test when we do + + //no custom function, fallback to UseLabelTemplate + const xmlString = ' option_1/Domestic_activities> option_2 '; + const xmlDoc = xmlParser.parseFromString(xmlString, 'text/html'); + + //if no template, returns "Answered" + expect(await resolveLabel("TimeUseSurvey", xmlDoc)).toBe(""); + //if no labelVars, returns template + //else interpolates +}); /** * @param surveyName the name of the survey (e.g. "TimeUseSurvey") @@ -136,7 +128,7 @@ it('filters the survey answers by their name and version', () => { xmlResponse: "", //survey answer XML string jsonDocResponse: "this is my json object" //survey answer JSON object }, - labels: {labelField: "goodbye"} //TODO learn more about answer type + labels: [{labelField: "goodbye"}] //TODO learn more about answer type } ]; @@ -154,7 +146,7 @@ it('filters the survey answers by their name and version', () => { xmlResponse: "", //survey answer XML string jsonDocResponse: "this is my json object" //survey answer JSON object }, - labels: {labelField: "goodbye"} + labels: [{labelField: "goodbye"}] }, { data: { @@ -166,7 +158,7 @@ it('filters the survey answers by their name and version', () => { xmlResponse: "", //survey answer XML string jsonDocResponse: "this is my json object" //survey answer JSON object }, - labels: {labelField: "goodbye"} + labels: [{labelField: "goodbye"}] } ]; From 75a0371ae8de99b5cb80bcbe797c7371b3d4255f Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 13 Oct 2023 10:48:46 -0600 Subject: [PATCH 13/49] updates to tests remove old i18n code, update types, comment out broken tests --- www/__tests__/enketoHelper.test.ts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index ab3ef963e..5603a010e 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -7,10 +7,6 @@ import { mockLogger } from '../__mocks__/globalMocks'; mockBEMUserCache(); mockLogger(); -// jest.mock('../__mocks__/messageFormatMocks'); -// jest.mock("i18next"); - -// global.i18next = { resolvedLanguage : "en" } it('gets the survey config', async () => { //this is aimed at testing my mock of the config @@ -78,7 +74,7 @@ it('resolves the label', async () => { const xmlDoc = xmlParser.parseFromString(xmlString, 'text/html'); //if no template, returns "Answered" - expect(await resolveLabel("TimeUseSurvey", xmlDoc)).toBe(""); + // expect(await resolveLabel("TimeUseSurvey", xmlDoc)).toBe(""); //if no labelVars, returns template //else interpolates }); @@ -105,7 +101,7 @@ it('gets the saved result or throws an error', () => { // export function loadPreviousResponseForSurvey(dataKey: string) { it('loads the previous response to a given survey', () => { //not really sure if I can test this yet given that it relies on an angular service... - loadPreviousResponseForSurvey("manual/demographic_survey"); + // loadPreviousResponseForSurvey("manual/demographic_survey"); }); /** @@ -128,7 +124,7 @@ it('filters the survey answers by their name and version', () => { xmlResponse: "", //survey answer XML string jsonDocResponse: "this is my json object" //survey answer JSON object }, - labels: [{labelField: "goodbye"}] //TODO learn more about answer type + metadata: {} } ]; @@ -146,7 +142,7 @@ it('filters the survey answers by their name and version', () => { xmlResponse: "", //survey answer XML string jsonDocResponse: "this is my json object" //survey answer JSON object }, - labels: [{labelField: "goodbye"}] + metadata: {} }, { data: { @@ -158,7 +154,7 @@ it('filters the survey answers by their name and version', () => { xmlResponse: "", //survey answer XML string jsonDocResponse: "this is my json object" //survey answer JSON object }, - labels: [{labelField: "goodbye"}] + metadata: {} } ]; From 09f21bc92cd1c9bad45986f465a0071a00f7c021 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 13 Oct 2023 10:49:14 -0600 Subject: [PATCH 14/49] update types from log statements, these answers have data and metadata, no labels --- www/js/survey/enketo/enketoHelper.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index fa7d300e3..af578b2d9 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -29,7 +29,7 @@ type EnketoAnswerData = { type EnketoAnswer = { data: EnketoAnswerData; //answer data - labels: [{[labelField:string]: string}]; //virtual labels (populated by populateLabels method) + metadata: any; } type EnketoSurveyConfig = { From e7fe7a8005c039240c6a4bab32141ecee4a612d5 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 13 Oct 2023 11:03:30 -0600 Subject: [PATCH 15/49] don't mock i18n, use the real thing with the changes from #1049, we are now able to test using i18n, no need to mock! --- www/__mocks__/i18nextMocks.ts | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 www/__mocks__/i18nextMocks.ts diff --git a/www/__mocks__/i18nextMocks.ts b/www/__mocks__/i18nextMocks.ts deleted file mode 100644 index dc0d3f2b4..000000000 --- a/www/__mocks__/i18nextMocks.ts +++ /dev/null @@ -1,8 +0,0 @@ -const i18next = jest.createMockFromModule('i18next'); - -let resolvedLanugage; - -function _setUpLanguage(language) { - console.log("setting resolved language to ", language, " for testing"); - resolvedLanugage = language; -} From 70f98cd87adce8ad9408b76ec2d24c40a4c6d60d Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 13 Oct 2023 11:38:17 -0600 Subject: [PATCH 16/49] set up baseline testing for resolveLabels introduced i18next for the tests updated config in mock, and test of loading config to be accurate to what is expected (missing some '{' ) adjust formatting of function indentation --- www/__mocks__/cordovaMocks.ts | 4 +-- www/__tests__/enketoHelper.test.ts | 38 +++++++++++------------ www/js/survey/enketo/enketoHelper.ts | 45 ++++++++++++++-------------- 3 files changed, 44 insertions(+), 43 deletions(-) diff --git a/www/__mocks__/cordovaMocks.ts b/www/__mocks__/cordovaMocks.ts index 31e3e7bf4..7590d0422 100644 --- a/www/__mocks__/cordovaMocks.ts +++ b/www/__mocks__/cordovaMocks.ts @@ -93,8 +93,8 @@ export const mockBEMUserCache = () => { surveys: { TimeUseSurvey: { compatibleWith: 1, formPath: "https://raw.githubusercontent.com/sebastianbarry/nrel-openpath-deploy-configs/surveys-info-and-surveys-data/survey-resources/data-json/time-use-survey-form-v9.json", - labelTemplate: {en: " erea, plural, =0 {} other {# Employment/Education, } }{ da, plural, =0 {} other {# Domestic activities, }", - es: " erea, plural, =0 {} other {# Empleo/Educación, } }{ da, plural, =0 {} other {# Actividades domesticas, }"}, + labelTemplate: {en: "{ erea, plural, =0 {} other {# Employment/Education, } }{ da, plural, =0 {} other {# Domestic, } }", + es: "{ erea, plural, =0 {} other {# Empleo/Educación, } }{ da, plural, =0 {} other {# Actividades domesticas, }}"}, labelVars: {da: {key: "Domestic_activities", type: "length"}, erea: {key: "Employment_related_a_Education_activities", type:"length"}}, version: 9} diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index 5603a010e..7a13303a8 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -2,8 +2,8 @@ import { getInstanceStr, filterByNameAndVersion, resolveTimestamps, resolveLabel import { mockBEMUserCache } from '../__mocks__/cordovaMocks'; import { mockLogger } from '../__mocks__/globalMocks'; -// import initializedI18next from '../js/i18nextInit'; -// window['i18next'] = initializedI18next; +import initializedI18next from '../js/i18nextInit'; +window['i18next'] = initializedI18next; mockBEMUserCache(); mockLogger(); @@ -13,14 +13,14 @@ it('gets the survey config', async () => { //mocked getDocument for the case of getting the config let config = await _lazyLoadConfig(); let mockSurveys = { - TimeUseSurvey: { compatibleWith: 1, - formPath: "https://raw.githubusercontent.com/sebastianbarry/nrel-openpath-deploy-configs/surveys-info-and-surveys-data/survey-resources/data-json/time-use-survey-form-v9.json", - labelTemplate: {en: " erea, plural, =0 {} other {# Employment/Education, } }{ da, plural, =0 {} other {# Domestic activities, }", - es: " erea, plural, =0 {} other {# Empleo/Educación, } }{ da, plural, =0 {} other {# Actividades domesticas, }"}, - labelVars: {da: {key: "Domestic_activities", type: "length"}, - erea: {key: "Employment_related_a_Education_activities", type:"length"}}, - version: 9} - } + TimeUseSurvey: { compatibleWith: 1, + formPath: "https://raw.githubusercontent.com/sebastianbarry/nrel-openpath-deploy-configs/surveys-info-and-surveys-data/survey-resources/data-json/time-use-survey-form-v9.json", + labelTemplate: {en: "{ erea, plural, =0 {} other {# Employment/Education, } }{ da, plural, =0 {} other {# Domestic, } }", + es: "{ erea, plural, =0 {} other {# Empleo/Educación, } }{ da, plural, =0 {} other {# Actividades domesticas, }}"}, + labelVars: {da: {key: "Domestic_activities", type: "length"}, + erea: {key: "Employment_related_a_Education_activities", type:"length"}}, + version: 9} + } expect(config).toMatchObject(mockSurveys); }) @@ -66,17 +66,17 @@ it('resolves the timestamps', () => { //resolve label it('resolves the label', async () => { const xmlParser = new window.DOMParser(); - - //have a custom survey label function TODO: we currently don't have custome label functions, but should test when we do - - //no custom function, fallback to UseLabelTemplate - const xmlString = ' option_1/Domestic_activities> option_2 '; + const xmlString = ' option_1 '; const xmlDoc = xmlParser.parseFromString(xmlString, 'text/html'); + const xmlString2 = ' option_1 option_3 '; + const xmlDoc2 = xmlParser.parseFromString(xmlString2, 'text/xml'); - //if no template, returns "Answered" - // expect(await resolveLabel("TimeUseSurvey", xmlDoc)).toBe(""); - //if no labelVars, returns template - //else interpolates + //if no template, returns "Answered" TODO: find a way to engineer this case + //if no labelVars, returns template TODO: find a way to engineer this case + //have a custom survey label function TODO: we currently don't have custome label functions, but should test when we do + //no custom function, fallback to UseLabelTemplate (standard case) + expect(await resolveLabel("TimeUseSurvey", xmlDoc)).toBe("3 Domestic"); + expect(await resolveLabel("TimeUseSurvey", xmlDoc2)).toBe("3 Employment/Education, 3 Domestic"); }); /** diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index af578b2d9..133789a48 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -50,31 +50,32 @@ const LABEL_FUNCTIONS = { let configSurveys = await _lazyLoadConfig(); const config = configSurveys[name]; // config for this survey - const lang = i18next.resolvedLanguage; - const labelTemplate = config.labelTemplate?.[lang]; + const lang = i18next.resolvedLanguage; + const labelTemplate = config.labelTemplate?.[lang]; - if (!labelTemplate) return "Answered"; // no template given in config - if (!config.labelVars) return labelTemplate; // if no vars given, nothing to interpolate, - // so we return the unaltered template + if (!labelTemplate) return "Answered"; // no template given in config + if (!config.labelVars) return labelTemplate; // if no vars given, nothing to interpolate, + // so we return the unaltered template - // gather vars that will be interpolated into the template according to the survey config - const labelVars = {} - for (let lblVar in config.labelVars) { - const fieldName = config.labelVars[lblVar].key; - let fieldStr = _getAnswerByTagName(xmlDoc, fieldName); - if (fieldStr == '') fieldStr = null; - if (config.labelVars[lblVar].type == 'length') { - const fieldMatches = fieldStr?.split(' '); - labelVars[lblVar] = fieldMatches?.length || 0; - } else { - throw new Error(`labelVar type ${config.labelVars[lblVar].type } is not supported!`) - } - } + // gather vars that will be interpolated into the template according to the survey config + const labelVars = {} + for (let lblVar in config.labelVars) { + const fieldName = config.labelVars[lblVar].key; + let fieldStr = _getAnswerByTagName(xmlDoc, fieldName); + if (fieldStr == '') fieldStr = null; + if (config.labelVars[lblVar].type == 'length') { + const fieldMatches = fieldStr?.split(' '); + labelVars[lblVar] = fieldMatches?.length || 0; + } else { + throw new Error(`labelVar type ${config.labelVars[lblVar].type } is not supported!`) + } + } - // use MessageFormat interpolate the label template with the label vars - const mf = new MessageFormat(lang); - const label = mf.compile(labelTemplate)(labelVars); - return label.replace(/^[ ,]+|[ ,]+$/g, ''); // trim leading and trailing spaces and commas + // use MessageFormat interpolate the label template with the label vars + const mf = new MessageFormat(lang); + console.log(labelTemplate); + const label = mf.compile(labelTemplate)(labelVars); + return label.replace(/^[ ,]+|[ ,]+$/g, ''); // trim leading and trailing spaces and commas } } From 719403a231c50aeca6b694d03faf59bf16e77618 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 13 Oct 2023 14:39:52 -0600 Subject: [PATCH 17/49] rework getInstanceStr tests now that I understand how this function works, I got the xml (filled and unfilled) directly from console.log statements. The tests are now accurate, and cover each of the cases. --- www/__tests__/enketoHelper.test.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index 7a13303a8..55c0aceef 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -25,21 +25,17 @@ it('gets the survey config', async () => { }) it('gets the model response, if avaliable, or returns null', ()=> { - const xmlModel = '\n \n \n \n \n \n \n ;'; - const filled = '\n \n car\n \n \n \n \n ;'; + const xmlModel = ''; + const filled = '2016-07-2517:24:32.928-06:002016-07-2517:30:31.000-06:00'; const opts = {"prefilledSurveyResponse": filled}; - const opts2 = {"prefillFields": {"travel_mode" : "car"}}; + const opts2 = {"prefillFields": {"Start_date":"2016-07-25", "Start_time": "17:24:32.928-06:00", "End_date": "2016-07-25", "End_time": "17:30:31.000-06:00"}}; //if no xmlModel, returns null expect(getInstanceStr(null, opts)).toBe(null); - //if there is a prefilled survey, return it expect(getInstanceStr(xmlModel, opts)).toBe(filled); - //if there is a model and fields, return prefilled - // expect(getInstanceStr(xmlModel, opts2)).toBe(filled); - //TODO - figure out how to use the helper function with JEST -- getElementsByTagName is empty? should it be? - + expect(getInstanceStr(xmlModel, opts2)).toBe(filled); //if none of those things, also return null expect(getInstanceStr(xmlModel, {})).toBe(null); }); From 252e42f45a64a66322ba3363ca7df7c10ce418cf Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 13 Oct 2023 16:25:06 -0600 Subject: [PATCH 18/49] tests for saveResponse testing for saving the response, both when it works and when the timestamps are invalid, resulting in an error --- www/__tests__/enketoHelper.test.ts | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index 55c0aceef..576b77e18 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -1,4 +1,4 @@ -import { getInstanceStr, filterByNameAndVersion, resolveTimestamps, resolveLabel, _lazyLoadConfig, loadPreviousResponseForSurvey} from '../js/survey/enketo/enketoHelper'; +import { getInstanceStr, filterByNameAndVersion, resolveTimestamps, resolveLabel, _lazyLoadConfig, loadPreviousResponseForSurvey, saveResponse} from '../js/survey/enketo/enketoHelper'; import { mockBEMUserCache } from '../__mocks__/cordovaMocks'; import { mockLogger } from '../__mocks__/globalMocks'; @@ -8,6 +8,9 @@ window['i18next'] = initializedI18next; mockBEMUserCache(); mockLogger(); +global.URL = require('url').URL; +global.Blob = require('node:buffer').Blob; + it('gets the survey config', async () => { //this is aimed at testing my mock of the config //mocked getDocument for the case of getting the config @@ -84,7 +87,27 @@ it('resolves the label', async () => { */ // export function saveResponse(surveyName: string, enketoForm: Form, appConfig, opts: SurveyOptions) { it('gets the saved result or throws an error', () => { - + const surveyName = "TimeUseSurvey"; + const form = { getDataStr: () => { return '2023-10-13T15:05:48.890-06:002023-10-13T15:05:48.892-06:002016-07-2517:24:32.928-06:002016-07-2517:30:31.000-06:00personal_care_activitiesdoing_sportuuid:dc16c287-08b2-4435-95aa-e4d7838b4225'}}; + const badForm = { getDataStr: () => { return '2023-10-13T15:05:48.890-06:002023-10-13T15:05:48.892-06:002016-08-2517:24:32.928-06:002016-07-2517:30:31.000-06:00personal_care_activitiesdoing_sportuuid:dc16c287-08b2-4435-95aa-e4d7838b4225'}}; + const config = { + survey_info: { + surveys: { + TimeUseSurvey: { compatibleWith: 1, + formPath: "https://raw.githubusercontent.com/sebastianbarry/nrel-openpath-deploy-configs/surveys-info-and-surveys-data/survey-resources/data-json/time-use-survey-form-v9.json", + labelTemplate: {en: "{ erea, plural, =0 {} other {# Employment/Education, } }{ da, plural, =0 {} other {# Domestic, } }", + es: "{ erea, plural, =0 {} other {# Empleo/Educación, } }{ da, plural, =0 {} other {# Actividades domesticas, }}"}, + labelVars: {da: {key: "Domestic_activities", type: "length"}, + erea: {key: "Employment_related_a_Education_activities", type:"length"}}, + version: 9} + } + } + }; + const opts = { timelineEntry: { end_local_dt: {timezone: "America/Los_Angeles"}, start_ts: 1469492672.928242, end_ts: 1469493031}}; + + console.log(config); + expect(saveResponse(surveyName, form, config, opts)).resolves.toMatchObject({label: "1 Personal Care", name: "TimeUseSurvey"}); + expect(saveResponse(surveyName, badForm, config, opts)).resolves.toMatchObject({message: "The times you entered are invalid. Please ensure that the start time is before the end time."}); }); /* From 9dc93844a9554d391434e7f39471ab5e4714195a Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 13 Oct 2023 16:25:19 -0600 Subject: [PATCH 19/49] take out old console.log --- www/js/survey/enketo/enketoHelper.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index 133789a48..3fa02f60c 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -73,7 +73,6 @@ const LABEL_FUNCTIONS = { // use MessageFormat interpolate the label template with the label vars const mf = new MessageFormat(lang); - console.log(labelTemplate); const label = mf.compile(labelTemplate)(labelVars); return label.replace(/^[ ,]+|[ ,]+$/g, ''); // trim leading and trailing spaces and commas } From 4d24d3e2a4a97492759c1f8c74da96c31fc3954d Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 13 Oct 2023 16:51:03 -0600 Subject: [PATCH 20/49] update types based on looking at these variables in breakpoints, these typings are more accurate --- www/js/survey/enketo/enketoHelper.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index 3fa02f60c..147ffede8 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -35,7 +35,8 @@ type EnketoAnswer = { type EnketoSurveyConfig = { [surveyName:string]: { formPath: string - labelFields: string[]; + labelTemplate: {[lang: string] : string}; + labelVars: {[activity: string]: {[key: string]: string, type:string}}, version: number; compatibleWith: number; } From 1977258a45cab6738fbfd33ac05effa9a08602e4 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Mon, 16 Oct 2023 11:34:03 -0600 Subject: [PATCH 21/49] carry through async nature of the label functions --- www/js/survey/enketo/enketoHelper.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index 147ffede8..773e3b7cb 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -131,13 +131,13 @@ export function _lazyLoadConfig() { * @param {XMLDocument} xmlDoc survey answer object * @returns {Promise} label string Promise */ -export function resolveLabel(name: string, xmlDoc: XMLDocument) { +export async function resolveLabel(name: string, xmlDoc: XMLDocument) { // Some studies may want a custom label function for their survey. // Those can be added in LABEL_FUNCTIONS with the survey name as the key. // Otherwise, UseLabelTemplate will create a label using the template in the config if (LABEL_FUNCTIONS[name]) - return LABEL_FUNCTIONS[name](xmlDoc); - return LABEL_FUNCTIONS.UseLabelTemplate(xmlDoc, name); + return await LABEL_FUNCTIONS[name](xmlDoc); + return await LABEL_FUNCTIONS.UseLabelTemplate(xmlDoc, name); } /** From a269a55ab8b94c3644a1e2788e71efe9801b4056 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Mon, 16 Oct 2023 11:34:47 -0600 Subject: [PATCH 22/49] correct precision of enketo dates the mismatch of precision and expected precision here is what was causing the added time entries to fail --- www/js/survey/enketo/enketoHelper.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index 773e3b7cb..0aab833b6 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -198,8 +198,8 @@ export function resolveTimestamps(xmlDoc, timelineEntry) { startTime = startTime.split(/\-|\+/)[0]; endTime = endTime.split(/\-|\+/)[0]; - let additionStartTs = DateTime.fromISO(startDate + "T" + startTime, {zone: timezone}).valueOf(); - let additionEndTs = DateTime.fromISO(endDate + "T" + endTime, {zone: timezone}).valueOf(); + let additionStartTs = DateTime.fromISO(startDate + "T" + startTime, {zone: timezone}).toSeconds(); + let additionEndTs = DateTime.fromISO(endDate + "T" + endTime, {zone: timezone}).toSeconds(); if (additionStartTs > additionEndTs) { return undefined; // if the start time is after the end time, this is an invalid response From e12ee3d326046ec4d356158d7f2579e2252f6c47 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Mon, 16 Oct 2023 15:24:38 -0600 Subject: [PATCH 23/49] resolve vscode errors there were errors because I was accessing with window.cordova instead of window['cordova'] --- www/js/control/ControlSyncHelper.tsx | 40 ++++++++++++++-------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/www/js/control/ControlSyncHelper.tsx b/www/js/control/ControlSyncHelper.tsx index edc0e7470..7802caaeb 100644 --- a/www/js/control/ControlSyncHelper.tsx +++ b/www/js/control/ControlSyncHelper.tsx @@ -15,7 +15,7 @@ import { updateUser } from "../commHelper"; * BEGIN: Simple read/write wrappers */ export function forcePluginSync() { - return window.cordova.plugins.BEMServerSync.forceSync(); + return window['cordova'].plugins.BEMServerSync.forceSync(); }; const formatConfigForDisplay = (configToFormat) => { @@ -27,11 +27,11 @@ const formatConfigForDisplay = (configToFormat) => { } const setConfig = function(config) { - return window.cordova.plugins.BEMServerSync.setConfig(config); + return window['cordova'].plugins.BEMServerSync.setConfig(config); }; const getConfig = function() { - return window.cordova.plugins.BEMServerSync.getConfig(); + return window['cordova'].plugins.BEMServerSync.getConfig(); }; export async function getHelperSyncSettings() { @@ -40,10 +40,10 @@ export async function getHelperSyncSettings() { } const getEndTransitionKey = function() { - if(window.cordova.platformId == 'android') { + if(window['cordova'].platformId == 'android') { return "local.transition.stopped_moving"; } - else if(window.cordova.platformId == 'ios') { + else if(window['cordova'].platformId == 'ios') { return "T_TRIP_ENDED"; } } @@ -62,7 +62,7 @@ export const ForceSyncRow = ({getState}) => { async function forceSync() { try { - let addedEvent = addStatEvent(statKeys.BUTTON_FORCE_SYNC); + let addedEvent = await addStatEvent(statKeys.BUTTON_FORCE_SYNC); console.log("Added "+statKeys.BUTTON_FORCE_SYNC+" event"); let sync = await forcePluginSync(); @@ -72,7 +72,7 @@ export const ForceSyncRow = ({getState}) => { * See https://github.com/e-mission/e-mission-phone/issues/279 for details */ var sensorKey = "statemachine/transition"; - let sensorDataList = await window.cordova.plugins.BEMUserCache.getAllMessages(sensorKey, true); + let sensorDataList = await window['cordova'].plugins.BEMUserCache.getAllMessages(sensorKey, true); // If everything has been pushed, we should // have no more trip end transitions left @@ -98,28 +98,28 @@ export const ForceSyncRow = ({getState}) => { }; const getStartTransitionKey = function() { - if(window.cordova.platformId == 'android') { + if(window['cordova'].platformId == 'android') { return "local.transition.exited_geofence"; } - else if(window.cordova.platformId == 'ios') { + else if(window['cordova'].platformId == 'ios') { return "T_EXITED_GEOFENCE"; } } const getEndTransitionKey = function() { - if(window.cordova.platformId == 'android') { + if(window['cordova'].platformId == 'android') { return "local.transition.stopped_moving"; } - else if(window.cordova.platformId == 'ios') { + else if(window['cordova'].platformId == 'ios') { return "T_TRIP_ENDED"; } } const getOngoingTransitionState = function() { - if(window.cordova.platformId == 'android') { + if(window['cordova'].platformId == 'android') { return "local.state.ongoing_trip"; } - else if(window.cordova.platformId == 'ios') { + else if(window['cordova'].platformId == 'ios') { return "STATE_ONGOING_TRIP"; } } @@ -127,12 +127,12 @@ export const ForceSyncRow = ({getState}) => { async function getTransition(transKey) { var entry_data = {}; const curr_state = await getState(); - entry_data.curr_state = curr_state; + entry_data['curr_state'] = curr_state; if (transKey == getEndTransitionKey()) { - entry_data.curr_state = getOngoingTransitionState(); + entry_data['curr_state'] = getOngoingTransitionState(); } - entry_data.transition = transKey; - entry_data.ts = moment().unix(); + entry_data['transition'] = transKey; + entry_data['ts'] = moment().unix(); return entry_data; } @@ -141,9 +141,9 @@ export const ForceSyncRow = ({getState}) => { * result for start so that we ensure ordering */ var sensorKey = "statemachine/transition"; let entry_data = await getTransition(getStartTransitionKey()); - let messagePut = await window.cordova.plugins.BEMUserCache.putMessage(sensorKey, entry_data); + let messagePut = await window['cordova'].plugins.BEMUserCache.putMessage(sensorKey, entry_data); entry_data = await getTransition(getEndTransitionKey()); - messagePut = await window.cordova.plugins.BEMUserCache.putMessage(sensorKey, entry_data); + messagePut = await window['cordova'].plugins.BEMUserCache.putMessage(sensorKey, entry_data); forceSync(); }; @@ -246,7 +246,7 @@ const ControlSyncHelper = ({ editVis, setEditVis }) => { * configure the UI */ let toggle; - if(window.cordova.platformId == 'ios'){ + if(window['cordova'].platformId == 'ios'){ toggle = Use Remote Push From ff30d3240e4d5d2b01891789628df1d5e4953cda Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Mon, 16 Oct 2023 15:39:05 -0600 Subject: [PATCH 24/49] update test after the change from miliseconds to seconds, the expected output here changed https://github.com/e-mission/e-mission-phone/pull/1063#issuecomment-1764950924 --- www/__tests__/enketoHelper.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index 576b77e18..2c0abd225 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -59,7 +59,7 @@ it('resolves the timestamps', () => { //good info returns unix start and end timestamps -- TODO : address precise vs less precise? const timeSurvey = ' 2016-07-25 2016-07-25 17:24:32.928-06:00 17:30:31.000-06:00 '; const xmlDoc = xmlParser.parseFromString(timeSurvey, 'text/xml'); - expect(resolveTimestamps(xmlDoc, timelineEntry)).toMatchObject({start_ts: 1469492672928, end_ts: 1469493031000}); + expect(resolveTimestamps(xmlDoc, timelineEntry)).toMatchObject({start_ts: 1469492672.928242, end_ts: 1469493031}); }); //resolve label From 6ef2725b21a5bd613752904b93af346fd543702a Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Tue, 17 Oct 2023 15:00:21 -0600 Subject: [PATCH 25/49] move the declaration of _config https://github.com/e-mission/e-mission-phone/pull/1063#discussion_r1362757509 --- www/js/survey/enketo/enketoHelper.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index 0aab833b6..f9a6ddf7a 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -42,10 +42,6 @@ type EnketoSurveyConfig = { } } -/** @type {EnketoSurveyConfig} _config */ -//TODO find a more appropriate way to store this -let _config: EnketoSurveyConfig; - const LABEL_FUNCTIONS = { UseLabelTemplate : async (xmlDoc: XMLDocument, name: string) => { let configSurveys = await _lazyLoadConfig(); @@ -92,6 +88,9 @@ const LABEL_FUNCTIONS = { return val; } +/** @type {EnketoSurveyConfig} _config */ +let _config: EnketoSurveyConfig; + /** * _lazyLoadConfig load enketo survey config. If already loaded, return the cached config * @returns {Promise} enketo survey config From b48027958eb25a1e2f79f75dec53abd3f93a2ab8 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Tue, 17 Oct 2023 15:13:41 -0600 Subject: [PATCH 26/49] move the fake config, add on Moving the fake config into it's own file so that we can easily add onto it. I kept the survey portion that I set up for the enketoHelper tests, and added the rest of the sections found in a typical config. https://github.com/e-mission/e-mission-phone/pull/1063#discussion_r1362749725 --- www/__mocks__/cordovaMocks.ts | 15 +----- www/__mocks__/fakeConfig.json | 94 +++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 13 deletions(-) create mode 100644 www/__mocks__/fakeConfig.json diff --git a/www/__mocks__/cordovaMocks.ts b/www/__mocks__/cordovaMocks.ts index bae65ad88..db22627f5 100644 --- a/www/__mocks__/cordovaMocks.ts +++ b/www/__mocks__/cordovaMocks.ts @@ -1,4 +1,5 @@ import packageJsonBuild from '../../package.cordovabuild.json'; +import fakeConfig from "./fakeConfig.json"; export const mockCordova = () => { window['cordova'] ||= {}; @@ -90,19 +91,7 @@ export const mockBEMUserCache = () => { }, getDocument: (key: string, withMetadata?: boolean) => { // this was mocked specifically for enketoHelper's use, could be expanded if needed - const fakeSurveyConfig = { - survey_info: { - surveys: { - TimeUseSurvey: { compatibleWith: 1, - formPath: "https://raw.githubusercontent.com/sebastianbarry/nrel-openpath-deploy-configs/surveys-info-and-surveys-data/survey-resources/data-json/time-use-survey-form-v9.json", - labelTemplate: {en: "{ erea, plural, =0 {} other {# Employment/Education, } }{ da, plural, =0 {} other {# Domestic, } }", - es: "{ erea, plural, =0 {} other {# Empleo/Educación, } }{ da, plural, =0 {} other {# Actividades domesticas, }}"}, - labelVars: {da: {key: "Domestic_activities", type: "length"}, - erea: {key: "Employment_related_a_Education_activities", type:"length"}}, - version: 9} - } - } - } + const fakeSurveyConfig = fakeConfig; if(key == "config/app_ui_config"){ return new Promise((rs, rj) => diff --git a/www/__mocks__/fakeConfig.json b/www/__mocks__/fakeConfig.json new file mode 100644 index 000000000..40471718d --- /dev/null +++ b/www/__mocks__/fakeConfig.json @@ -0,0 +1,94 @@ +{ + "version": 1, + "ts": 1655143472, + "server": { + "connectUrl": "https://openpath-test.nrel.gov/api/", + "aggregate_call_auth": "user_only" + }, + "intro": { + "program_or_study": "study", + "start_month": "10", + "start_year": "2023", + "program_admin_contact": "K. Shankari", + "deployment_partner_name": "NREL", + "translated_text": { + "en": { + "deployment_partner_name": "NREL", + "deployment_name": "Testing environment for Jest testing", + "summary_line_1": "", + "summary_line_2": "", + "summary_line_3": "", + "short_textual_description": "", + "why_we_collect": "", + "research_questions": [ + "", + "" + ] + }, + "es": { + "deployment_partner_name": "NREL", + "deployment_name": "Ambiente prueba para las pruebas de Jest", + "summary_line_1": "", + "summary_line_2": "", + "summary_line_3": "", + "short_textual_description": "", + "why_we_collect": "", + "research_questions": [ + "", + "" + ] + } + } + }, + "survey_info": { + "surveys": { + "TimeUseSurvey": { + "compatibleWith": 1, + "formPath": "https://raw.githubusercontent.com/sebastianbarry/nrel-openpath-deploy-configs/surveys-info-and-surveys-data/survey-resources/data-json/time-use-survey-form-v9.json", + "labelTemplate": { + "en": "{ erea, plural, =0 {} other {# Employment/Education, } }{ da, plural, =0 {} other {# Domestic, } }", + "es": "{ erea, plural, =0 {} other {# Empleo/Educación, } }{ da, plural, =0 {} other {# Actividades domesticas, }}" + }, + "labelVars": { + "da": { + "key": "Domestic_activities", + "type": "length" + }, + "erea": { + "key": "Employment_related_a_Education_activities", + "type": "length" + } + }, + "version": 9 + } + }, + "trip-labels": "ENKETO" + }, + "display_config": { + "use_imperial": false + }, + "profile_controls": { + "support_upload": true, + "trip_end_notification": false + }, + "admin_dashboard": { + "overview_users": true, + "overview_active_users": true, + "overview_trips": true, + "overview_signup_trends": true, + "overview_trips_trend": true, + "data_uuids": true, + "data_trips": true, + "data_trips_columns_exclude": [], + "additional_trip_columns": [], + "data_uuids_columns_exclude": [], + "token_generate": true, + "token_prefix": "nrelop", + "map_heatmap": true, + "map_bubble": true, + "map_trip_lines": true, + "push_send": true, + "options_uuids": true, + "options_emails": true + } +} \ No newline at end of file From 159d665295f67ad3f0d59df5bc94967e72c83fe8 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 8 Nov 2023 13:29:37 -0500 Subject: [PATCH 27/49] support client-side transformation for enketo XMLs With this change, the `formPath` specifying the URL of an Enketo survey can be either JSON or XML. If it's JSON, we'll be able parse and use it directly. If it cannot be parsed as JSON, we'll perform XML -> JSON transformation with enketo-transformer/web. --- package.cordovabuild.json | 1 + package.serve.json | 1 + webpack.config.js | 7 ++++++- www/js/survey/enketo/EnketoModal.tsx | 16 +++++----------- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/package.cordovabuild.json b/package.cordovabuild.json index c61fa72c5..2e78f7363 100644 --- a/package.cordovabuild.json +++ b/package.cordovabuild.json @@ -142,6 +142,7 @@ "cordova-plugin-x-socialsharing": "6.0.4", "core-js": "^2.5.7", "enketo-core": "^6.1.7", + "enketo-transformer": "^4.0.0", "fast-xml-parser": "^4.2.2", "fs-extra": "^9.0.1", "i18next": "^22.5.0", diff --git a/package.serve.json b/package.serve.json index 59a803085..c66c32b1e 100644 --- a/package.serve.json +++ b/package.serve.json @@ -73,6 +73,7 @@ "chartjs-plugin-annotation": "^3.0.1", "core-js": "^2.5.7", "enketo-core": "^6.1.7", + "enketo-transformer": "^4.0.0", "fast-xml-parser": "^4.2.2", "fs-extra": "^9.0.1", "i18next": "^22.5.0", diff --git a/webpack.config.js b/webpack.config.js index 1e504ac5f..3e7e6d368 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -79,7 +79,12 @@ module.exports = { /* Enketo expects its per-app configuration to be available as 'enketo-config', so we have to alias it here. https://github.com/enketo/enketo-core#global-configuration */ - 'enketo/config': path.resolve(__dirname, 'www/js/config/enketo-config') + 'enketo/config': path.resolve(__dirname, 'www/js/config/enketo-config'), + /* enketo-transformer has 'libxslt' as an optional peer dependency. + We don't need it since we are only doing client-side transformations via + enketo-transformer/web (https://github.com/enketo/enketo-transformer#web). + So, we can tell webpack it's ok to ignore libxslt by aliasing it to false. */ + 'libxslt': false, }, extensions: ['.web.js', '.jsx', '.tsx', '.ts', '.js'], }, diff --git a/www/js/survey/enketo/EnketoModal.tsx b/www/js/survey/enketo/EnketoModal.tsx index de1f505f3..a0dc667c5 100644 --- a/www/js/survey/enketo/EnketoModal.tsx +++ b/www/js/survey/enketo/EnketoModal.tsx @@ -6,8 +6,8 @@ import useAppConfig from '../../useAppConfig'; import { useTranslation } from 'react-i18next'; import { SurveyOptions, getInstanceStr, saveResponse } from './enketoHelper'; import { fetchUrlCached } from '../../commHelper'; -import { displayError, displayErrorMsg } from '../../plugin/logger'; -// import { transform } from 'enketo-transformer/web'; +import { displayError, displayErrorMsg, logDebug } from '../../plugin/logger'; +import { transform } from 'enketo-transformer/web'; type Props = Omit & { surveyName: string; @@ -26,15 +26,9 @@ const EnketoModal = ({ surveyName, onResponseSaved, opts, ...rest }: Props) => { const responseText = await fetchUrlCached(url); try { return JSON.parse(responseText); - } catch ({ name, message }) { - // not JSON, so it must be XML - return Promise.reject( - 'downloaded survey was not JSON; enketo-transformer is not available yet', - ); - /* uncomment once enketo-transformer is available */ - // if `response` is not JSON, it is an XML string and needs transformation to JSON - // const xmlText = await res.text(); - // return await transform({xform: xmlText}); + } catch (e) { + logDebug(`${e.name}: Survey was not in JSON format. Attempting to transform XML -> JSON...`); + return await transform({ xform: responseText }); } } From 2b72850ce16de27531185bb8e3f0aa39a613ce39 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 8 Nov 2023 14:41:32 -0500 Subject: [PATCH 28/49] rename + move fetchSurvey to enketoHelper.ts --- www/js/survey/enketo/EnketoModal.tsx | 16 ++-------------- www/js/survey/enketo/enketoHelper.ts | 12 ++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/www/js/survey/enketo/EnketoModal.tsx b/www/js/survey/enketo/EnketoModal.tsx index a0dc667c5..9267b9808 100644 --- a/www/js/survey/enketo/EnketoModal.tsx +++ b/www/js/survey/enketo/EnketoModal.tsx @@ -4,10 +4,8 @@ import { StyleSheet, Modal, ScrollView, SafeAreaView, Pressable } from 'react-na import { ModalProps } from 'react-native-paper'; import useAppConfig from '../../useAppConfig'; import { useTranslation } from 'react-i18next'; -import { SurveyOptions, getInstanceStr, saveResponse } from './enketoHelper'; -import { fetchUrlCached } from '../../commHelper'; +import { SurveyOptions, fetchSurvey, getInstanceStr, saveResponse } from './enketoHelper'; import { displayError, displayErrorMsg, logDebug } from '../../plugin/logger'; -import { transform } from 'enketo-transformer/web'; type Props = Omit & { surveyName: string; @@ -22,16 +20,6 @@ const EnketoModal = ({ surveyName, onResponseSaved, opts, ...rest }: Props) => { const enketoForm = useRef
(null); const appConfig = useAppConfig(); - async function fetchSurveyJson(url) { - const responseText = await fetchUrlCached(url); - try { - return JSON.parse(responseText); - } catch (e) { - logDebug(`${e.name}: Survey was not in JSON format. Attempting to transform XML -> JSON...`); - return await transform({ xform: responseText }); - } - } - async function validateAndSave() { const valid = await enketoForm.current.validate(); if (!valid) return false; @@ -56,7 +44,7 @@ const EnketoModal = ({ surveyName, onResponseSaved, opts, ...rest }: Props) => { const formPath = appConfig.survey_info?.surveys?.[surveyName]?.formPath; if (!formPath) return console.error('No form path found for survey', surveyName); - fetchSurveyJson(formPath).then(({ form, model }) => { + fetchSurvey(formPath).then(({ form, model }) => { surveyJson.current = { form, model }; headerEl?.current.insertAdjacentHTML('afterend', form); // inject form into DOM const formEl = document.querySelector('form.or'); diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index 9b96c8463..e4a1fcf45 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -1,11 +1,13 @@ import { getAngularService } from '../../angular-react-helper'; import { Form } from 'enketo-core'; +import { transform } from 'enketo-transformer/web'; import { XMLParser } from 'fast-xml-parser'; import i18next from 'i18next'; import MessageFormat from '@messageformat/core'; import { logDebug, logInfo } from '../../plugin/logger'; import { getConfig } from '../../config/dynamicConfig'; import { DateTime } from 'luxon'; +import { fetchUrlCached } from '../../commHelper'; export type PrefillFields = { [key: string]: string }; @@ -287,3 +289,13 @@ export function loadPreviousResponseForSurvey(dataKey: string) { _getMostRecent(answers), ); } + +export async function fetchSurvey(url: string) { + const responseText = await fetchUrlCached(url); + try { + return JSON.parse(responseText); + } catch (e) { + logDebug(`${e.name}: Survey was not in JSON format. Attempting to transform XML -> JSON...`); + return await transform({ xform: responseText }); + } +} From e230420fa5c816a4eb7da359946312a93fc82a3c Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 10 Nov 2023 10:41:11 -0700 Subject: [PATCH 29/49] prettify merge conflicted file --- www/js/survey/enketo/enketo-add-note-button.js | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/www/js/survey/enketo/enketo-add-note-button.js b/www/js/survey/enketo/enketo-add-note-button.js index 2d8730f6f..8dc2e26e4 100644 --- a/www/js/survey/enketo/enketo-add-note-button.js +++ b/www/js/survey/enketo/enketo-add-note-button.js @@ -7,8 +7,8 @@ import { filterByNameAndVersion } from './enketoHelper'; import { getAdditionsForTimelineEntry, getUniqueEntries } from '../inputMatcher'; angular - .module('emission.survey.enketo.add-note-button', ['emission.services',]) - .factory('EnketoNotesButtonService', function ( Logger, $timeout) { + .module('emission.survey.enketo.add-note-button', ['emission.services']) + .factory('EnketoNotesButtonService', function (Logger, $timeout) { var enbs = {}; console.log('Creating EnketoNotesButtonService'); enbs.SINGLE_KEY = 'NOTES'; @@ -33,13 +33,9 @@ angular * Embed 'inputType' to the timelineEntry. */ enbs.extractResult = function (results) { - const resultsPromises = [ - filterByNameAndVersion(enbs.timelineEntrySurveyName, results), - ]; + const resultsPromises = [filterByNameAndVersion(enbs.timelineEntrySurveyName, results)]; if (enbs.timelineEntrySurveyName != enbs.placeSurveyName) { - resultsPromises.push( - filterByNameAndVersion(enbs.placeSurveyName, results), - ); + resultsPromises.push(filterByNameAndVersion(enbs.placeSurveyName, results)); } return Promise.all(resultsPromises); }; From da4947d7c303f6dfeef865410f901ae524e0aa75 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 10 Nov 2023 10:57:47 -0700 Subject: [PATCH 30/49] add exception to transform the enketoHelper tests were failing because one of the files in the added directory was not transpiled ,so it was picking up as bad syntax --- jest.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jest.config.js b/jest.config.js index 743ab5a00..ef3503294 100644 --- a/jest.config.js +++ b/jest.config.js @@ -12,7 +12,7 @@ module.exports = { "^.+\\.(ts|tsx|js|jsx)$": "babel-jest" }, transformIgnorePatterns: [ - "node_modules/(?!((jest-)?react-native(-.*)?|@react-native(-community)?)/)" + "node_modules/(?!((enketo-transformer/dist/enketo-transformer/web)|(jest-)?react-native(-.*)?|@react-native(-community)?)/)", ], moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx'], moduleDirectories: ["node_modules", "src"], From 245982159acb2ee4f6410d027b8acfe2a73ff9c2 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 10 Nov 2023 14:19:56 -0700 Subject: [PATCH 31/49] lingering merge conflict this file does not exist in this branch! --- www/js/diary.js | 1 - 1 file changed, 1 deletion(-) diff --git a/www/js/diary.js b/www/js/diary.js index c580ad8f2..7e64e555f 100644 --- a/www/js/diary.js +++ b/www/js/diary.js @@ -5,7 +5,6 @@ angular .module('emission.main.diary', [ 'emission.main.diary.services', 'emission.plugin.logger', - 'emission.survey.enketo.answer', ]) .config(function ($stateProvider) { From 0890e23f6d82e8fd878fef9d74633889f5650fc0 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 10 Nov 2023 14:25:41 -0700 Subject: [PATCH 32/49] re-run prettier after merging --- www/js/diary.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/www/js/diary.js b/www/js/diary.js index 7e64e555f..08909886d 100644 --- a/www/js/diary.js +++ b/www/js/diary.js @@ -2,10 +2,7 @@ import angular from 'angular'; import LabelTab from './diary/LabelTab'; angular - .module('emission.main.diary', [ - 'emission.main.diary.services', - 'emission.plugin.logger', - ]) + .module('emission.main.diary', ['emission.main.diary.services', 'emission.plugin.logger']) .config(function ($stateProvider) { $stateProvider.state('root.main.inf_scroll', { From 7df44e9c3f6892c572635f386ce32c7e87246703 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Sun, 12 Nov 2023 22:19:04 -0800 Subject: [PATCH 33/49] cleanup enketo after merge - Restore the use of 'filterByNameAndVersion' for trip confim surveys (not currently used for anything else). If a survey response is no longer compatible according to the appConfig, it should be filtered out here. - Make the terminology unambiguous such that "answer" means how the user answered a particular question within the survey, and "response" means the overall survey response - Clean up / clarify comments - Remove unused import statements --- www/__tests__/enketoHelper.test.ts | 32 ++++++++-------- www/js/diary/timelineHelper.ts | 7 ++-- www/js/survey/enketo/EnketoModal.tsx | 2 +- www/js/survey/enketo/enketoHelper.ts | 55 ++++++++++++++-------------- 4 files changed, 47 insertions(+), 49 deletions(-) diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index a2e5514b1..113e7f995 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -186,15 +186,15 @@ it('loads the previous response to a given survey', () => { }); /** - * filterByNameAndVersion filter the survey answers by survey name and their version. + * filterByNameAndVersion filter the survey responses by survey name and their version. * The version for filtering is specified in enketo survey `compatibleWith` config. - * The stored survey answer version must be greater than or equal to `compatibleWith` to be included. + * The stored survey response version must be greater than or equal to `compatibleWith` to be included. */ -it('filters the survey answers by their name and version', () => { - //no answers -> no filtered answers +it('filters the survey responses by their name and version', () => { + //no response -> no filtered responses expect(filterByNameAndVersion('TimeUseSurvey', [])).resolves.toStrictEqual([]); - const answer = [ + const response = [ { data: { label: 'Activity', //display label (this value is use for displaying on the button) @@ -202,17 +202,17 @@ it('filters the survey answers by their name and version', () => { fmt_time: '12:36', //the formatted timestamp at which the survey was filled out name: 'TimeUseSurvey', //survey name version: '1', //survey version - xmlResponse: '', //survey answer XML string - jsonDocResponse: 'this is my json object', //survey answer JSON object + xmlResponse: '', //survey response XML string + jsonDocResponse: 'this is my json object', //survey response JSON object }, metadata: {}, }, ]; - //one answer -> that answer - expect(filterByNameAndVersion('TimeUseSurvey', answer)).resolves.toStrictEqual(answer); + //one response -> that response + expect(filterByNameAndVersion('TimeUseSurvey', response)).resolves.toStrictEqual(response); - const answers = [ + const responses = [ { data: { label: 'Activity', //display label (this value is use for displaying on the button) @@ -220,8 +220,8 @@ it('filters the survey answers by their name and version', () => { fmt_time: '12:36', //the formatted timestamp at which the survey was filled out name: 'TimeUseSurvey', //survey name version: '1', //survey version - xmlResponse: '', //survey answer XML string - jsonDocResponse: 'this is my json object', //survey answer JSON object + xmlResponse: '', //survey response XML string + jsonDocResponse: 'this is my json object', //survey response JSON object }, metadata: {}, }, @@ -232,13 +232,13 @@ it('filters the survey answers by their name and version', () => { fmt_time: '12:36', //the formatted timestamp at which the survey was filled out name: 'OtherSurvey', //survey name version: '1', //survey version - xmlResponse: '', //survey answer XML string - jsonDocResponse: 'this is my json object', //survey answer JSON object + xmlResponse: '', //survey response XML string + jsonDocResponse: 'this is my json object', //survey response JSON object }, metadata: {}, }, ]; - //several answers -> only the one that has a name match - expect(filterByNameAndVersion('TimeUseSurvey', answers)).resolves.toStrictEqual(answer); + //several responses -> only the one that has a name match + expect(filterByNameAndVersion('TimeUseSurvey', responses)).resolves.toStrictEqual(response); }); diff --git a/www/js/diary/timelineHelper.ts b/www/js/diary/timelineHelper.ts index 174974a9d..19c885cc1 100644 --- a/www/js/diary/timelineHelper.ts +++ b/www/js/diary/timelineHelper.ts @@ -1,11 +1,10 @@ import moment from 'moment'; -import { displayError, logDebug } from '../plugin/logger'; +import { logDebug } from '../plugin/logger'; import { getBaseModeByKey, getBaseModeByValue } from './diaryHelper'; import { getUnifiedDataForInterval } from '../services/unifiedDataLoader'; -import i18next from 'i18next'; import { UserInputEntry } from '../types/diaryTypes'; import { getLabelInputDetails, getLabelInputs } from '../survey/multilabel/confirmHelper'; -import { getNotDeletedCandidates, getUniqueEntries } from '../survey/inputMatcher'; +import { filterByNameAndVersion } from '../survey/enketo/enketoHelper'; const cachedGeojsons = new Map(); /** @@ -92,7 +91,7 @@ function updateUnprocessedInputs(labelsPromises, notesPromises, appConfig) { // fill in the unprocessedLabels object with the labels we just read labelResults.forEach((r, i) => { if (appConfig.survey_info?.['trip-labels'] == 'ENKETO') { - unprocessedLabels['SURVEY'] = r; + unprocessedLabels['SURVEY'] = filterByNameAndVersion('TripConfirmSurvey', r); } else { unprocessedLabels[getLabelInputs()[i]] = r; } diff --git a/www/js/survey/enketo/EnketoModal.tsx b/www/js/survey/enketo/EnketoModal.tsx index 9267b9808..1d169ee9b 100644 --- a/www/js/survey/enketo/EnketoModal.tsx +++ b/www/js/survey/enketo/EnketoModal.tsx @@ -5,7 +5,7 @@ import { ModalProps } from 'react-native-paper'; import useAppConfig from '../../useAppConfig'; import { useTranslation } from 'react-i18next'; import { SurveyOptions, fetchSurvey, getInstanceStr, saveResponse } from './enketoHelper'; -import { displayError, displayErrorMsg, logDebug } from '../../plugin/logger'; +import { displayError, displayErrorMsg } from '../../plugin/logger'; type Props = Omit & { surveyName: string; diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index 9defa77f5..379120373 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -1,4 +1,3 @@ -import { getAngularService } from '../../angular-react-helper'; import { Form } from 'enketo-core'; import { transform } from 'enketo-transformer/web'; import { XMLParser } from 'fast-xml-parser'; @@ -7,7 +6,7 @@ import MessageFormat from '@messageformat/core'; import { logDebug, logInfo } from '../../plugin/logger'; import { getConfig } from '../../config/dynamicConfig'; import { DateTime } from 'luxon'; -import { fetchUrlCached } from '../../commHelper'; +import { fetchUrlCached } from '../../services/commHelper'; import { getUnifiedDataForInterval } from '../../services/unifiedDataLoader'; export type PrefillFields = { [key: string]: string }; @@ -20,18 +19,18 @@ export type SurveyOptions = { dataKey?: string; }; -type EnketoAnswerData = { +type EnketoResponseData = { label: string; //display label (this value is use for displaying on the button) ts: string; //the timestamp at which the survey was filled out (in seconds) fmt_time: string; //the formatted timestamp at which the survey was filled out name: string; //survey name version: string; //survey version - xmlResponse: string; //survey answer XML string - jsonDocResponse: string; //survey answer JSON object + xmlResponse: string; //survey response as XML string + jsonDocResponse: string; //survey response as JSON object }; -type EnketoAnswer = { - data: EnketoAnswerData; //answer data +type EnketoResponse = { + data: EnketoResponseData; //survey response data metadata: any; }; @@ -79,9 +78,10 @@ const LABEL_FUNCTIONS = { }; /** - * _getAnswerByTagName lookup for the survey answer by tag name form the given XML document. - * @param {XMLDocument} xmlDoc survey answer object - * @param {string} tagName tag name + * _getAnswerByTagName look up how a question was answered, given the survey response + * and the tag name of the question + * @param {XMLDocument} xmlDoc survey response as XML object + * @param {string} tagName tag name of the question * @returns {string} answer string. If not found, return "\" */ function _getAnswerByTagName(xmlDoc: XMLDocument, tagName: string) { @@ -110,26 +110,24 @@ export function _lazyLoadConfig() { } /** - * filterByNameAndVersion filter the survey answers by survey name and their version. + * filterByNameAndVersion filter the survey responses by survey name and their version. * The version for filtering is specified in enketo survey `compatibleWith` config. - * The stored survey answer version must be greater than or equal to `compatibleWith` to be included. + * The survey version of the response must be greater than or equal to `compatibleWith` to be included. * @param {string} name survey name (defined in enketo survey config) - * @param {EnketoAnswer[]} answers survey answers - * (usually retrieved by calling UnifiedDataLoader.getUnifiedMessagesForInterval('manual/survey_response', tq)) method. - * @return {Promise} filtered survey answers + * @param {EnketoResponse[]} responses An array of previously recorded responses to Enketo surveys + * (presumably having been retrieved from unifiedDataLoader) + * @return {Promise} filtered survey responses */ -export function filterByNameAndVersion(name: string, answers: EnketoAnswer[]) { +export function filterByNameAndVersion(name: string, responses: EnketoResponse[]) { return _lazyLoadConfig().then((config) => - answers.filter( - (answer) => answer.data.name === name && answer.data.version >= config[name].compatibleWith, - ), + responses.filter((r) => r.data.name === name && r.data.version >= config[name].compatibleWith), ); } /** - * resolve answer label for the survey + * resolve a label for the survey response * @param {string} name survey name - * @param {XMLDocument} xmlDoc survey answer object + * @param {XMLDocument} xmlDoc survey response as XML object * @returns {Promise} label string Promise */ export async function resolveLabel(name: string, xmlDoc: XMLDocument) { @@ -172,7 +170,7 @@ export function getInstanceStr(xmlModel: string, opts: SurveyOptions): string | /** * resolve timestamps label from the survey response - * @param {XMLDocument} xmlDoc survey answer object + * @param {XMLDocument} xmlDoc survey response as XML object * @param {object} trip trip object * @returns {object} object with `start_ts` and `end_ts` * - null if no timestamps are resolved @@ -269,10 +267,11 @@ export function saveResponse(surveyName: string, enketoForm: Form, appConfig, op .then((data) => data); } -const _getMostRecent = (answers) => { - answers.sort((a, b) => a.metadata.write_ts < b.metadata.write_ts); - console.log('first answer is ', answers[0], ' last answer is ', answers[answers.length - 1]); - return answers[0]; +const _getMostRecent = (responses) => { + responses.sort((a, b) => a.metadata.write_ts < b.metadata.write_ts); + logDebug(`_getMostRecent: first response is ${responses[0]}; + last response is ${responses.slice(-1)[0]}`); + return responses[0]; }; /* @@ -286,8 +285,8 @@ export function loadPreviousResponseForSurvey(dataKey: string) { const tq = window['cordova'].plugins.BEMUserCache.getAllTimeQuery(); logDebug('loadPreviousResponseForSurvey: dataKey = ' + dataKey + '; tq = ' + tq); const getMethod = window['cordova'].plugins.BEMUserCache.getSensorDataForInterval; - return getUnifiedDataForInterval(dataKey, tq, getMethod).then((answers) => - _getMostRecent(answers), + return getUnifiedDataForInterval(dataKey, tq, getMethod).then((responses) => + _getMostRecent(responses), ); } From 4499e24097e42bac9d096a690437cca7f90256da Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Mon, 13 Nov 2023 10:58:18 -0700 Subject: [PATCH 34/49] test remaining function now that Unified Data Loader has been merged, we can test this function, may benefit from more end-to-end testing later, as it relies on the usercache plugin heavily --- www/__mocks__/cordovaMocks.ts | 10 ++++++++++ www/__tests__/enketoHelper.test.ts | 4 +--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/www/__mocks__/cordovaMocks.ts b/www/__mocks__/cordovaMocks.ts index 0684fa06d..a74df6b18 100644 --- a/www/__mocks__/cordovaMocks.ts +++ b/www/__mocks__/cordovaMocks.ts @@ -129,6 +129,16 @@ export const mockBEMUserCache = () => { return false; } }, + getAllTimeQuery: () => { + return {key: "write_ts", startTs: 0, endTs: Date.now()/1000}; + }, + getSensorDataForInterval: (key, tq, withMetadata) => { + return new Promise((rs, rj) => + setTimeout(() => { + rs({metadata: {write_ts: "1699897723"}, data: "completed", time: "01/01/2001"}); + }, 100), + ); + }, }; window['cordova'] ||= {}; window['cordova'].plugins ||= {}; diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index 113e7f995..ae43b7bb4 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -179,10 +179,8 @@ it('gets the saved result or throws an error', () => { * Loading it on demand seems like the way to go. If we choose to experiment * with incremental updates, we may want to revisit this. */ -// export function loadPreviousResponseForSurvey(dataKey: string) { it('loads the previous response to a given survey', () => { - //not really sure if I can test this yet given that it relies on an angular service... - // loadPreviousResponseForSurvey("manual/demographic_survey"); + expect(loadPreviousResponseForSurvey("manual/demographic_survey")).resolves.toMatchObject({data: "completed", time: "01/01/2001"}); }); /** From 4a000fc4e82be569afe27e87066f5aaac172b4a2 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Mon, 13 Nov 2023 11:01:54 -0700 Subject: [PATCH 35/49] re-run prettier --- www/__mocks__/cordovaMocks.ts | 10 +++++----- www/__tests__/enketoHelper.test.ts | 5 ++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/www/__mocks__/cordovaMocks.ts b/www/__mocks__/cordovaMocks.ts index a74df6b18..f08293a85 100644 --- a/www/__mocks__/cordovaMocks.ts +++ b/www/__mocks__/cordovaMocks.ts @@ -130,14 +130,14 @@ export const mockBEMUserCache = () => { } }, getAllTimeQuery: () => { - return {key: "write_ts", startTs: 0, endTs: Date.now()/1000}; + return { key: 'write_ts', startTs: 0, endTs: Date.now() / 1000 }; }, getSensorDataForInterval: (key, tq, withMetadata) => { return new Promise((rs, rj) => - setTimeout(() => { - rs({metadata: {write_ts: "1699897723"}, data: "completed", time: "01/01/2001"}); - }, 100), - ); + setTimeout(() => { + rs({ metadata: { write_ts: '1699897723' }, data: 'completed', time: '01/01/2001' }); + }, 100), + ); }, }; window['cordova'] ||= {}; diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index ae43b7bb4..a8f49b29c 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -180,7 +180,10 @@ it('gets the saved result or throws an error', () => { * with incremental updates, we may want to revisit this. */ it('loads the previous response to a given survey', () => { - expect(loadPreviousResponseForSurvey("manual/demographic_survey")).resolves.toMatchObject({data: "completed", time: "01/01/2001"}); + expect(loadPreviousResponseForSurvey('manual/demographic_survey')).resolves.toMatchObject({ + data: 'completed', + time: '01/01/2001', + }); }); /** From ebaa1e6e5cd19b76ef7b6f23c5509f99b0c845f2 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Mon, 13 Nov 2023 15:41:41 -0700 Subject: [PATCH 36/49] fix issue with filter The root of the issue I was having is that filterByNameAndVersion returns a promise, but it was being assigned to the unprocessedLabels var before the promise was fulfilled, adding the fiter.then() flow resolved this --- www/js/diary/timelineHelper.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/www/js/diary/timelineHelper.ts b/www/js/diary/timelineHelper.ts index 19c885cc1..1a2c87462 100644 --- a/www/js/diary/timelineHelper.ts +++ b/www/js/diary/timelineHelper.ts @@ -91,7 +91,9 @@ function updateUnprocessedInputs(labelsPromises, notesPromises, appConfig) { // fill in the unprocessedLabels object with the labels we just read labelResults.forEach((r, i) => { if (appConfig.survey_info?.['trip-labels'] == 'ENKETO') { - unprocessedLabels['SURVEY'] = filterByNameAndVersion('TripConfirmSurvey', r); + filterByNameAndVersion('TripConfirmSurvey', r).then((filtered) => { + unprocessedLabels['SURVEY'] = filtered; + }); } else { unprocessedLabels[getLabelInputs()[i]] = r; } From 8f7c6748ed161e248e247cc04514dacc46908317 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 17 Nov 2023 16:45:29 -0700 Subject: [PATCH 37/49] add a test case we also want to test that the timestamps follow the "minute" accuracy convention we implemented, so that if the survey and timelineEntry match within the minute, we use the timelineEntry timestamps, else use the timestamps from the survey https://github.com/e-mission/e-mission-phone/pull/1063#discussion_r1397710929 --- www/__tests__/enketoHelper.test.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index a8f49b29c..f4cfeb8ee 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -86,14 +86,22 @@ it('resolves the timestamps', () => { ' 2016-08-28 2016-07-25 17:32:32.928-06:00 17:30:31.000-06:00 '; const badTimeDoc = xmlParser.parseFromString(badTimes, 'text/xml'); expect(resolveTimestamps(badTimeDoc, timelineEntry)).toBeUndefined(); - //good info returns unix start and end timestamps -- TODO : address precise vs less precise? - const timeSurvey = + //if within a minute, timelineEntry timestamps + const timeEntry = ' 2016-07-25 2016-07-25 17:24:32.928-06:00 17:30:31.000-06:00 '; - const xmlDoc = xmlParser.parseFromString(timeSurvey, 'text/xml'); - expect(resolveTimestamps(xmlDoc, timelineEntry)).toMatchObject({ + const xmlDoc1 = xmlParser.parseFromString(timeEntry, 'text/xml'); + expect(resolveTimestamps(xmlDoc1, timelineEntry)).toMatchObject({ start_ts: 1469492672.928242, end_ts: 1469493031, }); + // else survey timestamps + const timeSurvey = + ' 2016-07-25 2016-07-25 17:22:33.928-06:00 17:33:33.000-06:00 '; + const xmlDoc2 = xmlParser.parseFromString(timeSurvey, 'text/xml'); + expect(resolveTimestamps(xmlDoc2, timelineEntry)).toMatchObject({ + start_ts: 1469492553.928, + end_ts: 1469493213, + }); }); //resolve label From 7ec575e41701725723609c9eb40654ef8d05385b Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 17 Nov 2023 17:02:39 -0700 Subject: [PATCH 38/49] note the issue this form is invalide because of the start and end times mismatching --- www/__tests__/enketoHelper.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index f4cfeb8ee..e06dc94a6 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -136,6 +136,7 @@ it('gets the saved result or throws an error', () => { return '2023-10-13T15:05:48.890-06:002023-10-13T15:05:48.892-06:002016-07-2517:24:32.928-06:002016-07-2517:30:31.000-06:00personal_care_activitiesdoing_sportuuid:dc16c287-08b2-4435-95aa-e4d7838b4225'; }, }; + //the start time listed is after the end time listed const badForm = { getDataStr: () => { return '2023-10-13T15:05:48.890-06:002023-10-13T15:05:48.892-06:002016-08-2517:24:32.928-06:002016-07-2517:30:31.000-06:00personal_care_activitiesdoing_sportuuid:dc16c287-08b2-4435-95aa-e4d7838b4225'; From 8b3ab76327d93011e2a31cfa6409bcb23f710e1f Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 17 Nov 2023 17:07:22 -0700 Subject: [PATCH 39/49] check for key before resolving with this information --- www/__mocks__/cordovaMocks.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/www/__mocks__/cordovaMocks.ts b/www/__mocks__/cordovaMocks.ts index f08293a85..4911b3ebe 100644 --- a/www/__mocks__/cordovaMocks.ts +++ b/www/__mocks__/cordovaMocks.ts @@ -133,11 +133,15 @@ export const mockBEMUserCache = () => { return { key: 'write_ts', startTs: 0, endTs: Date.now() / 1000 }; }, getSensorDataForInterval: (key, tq, withMetadata) => { - return new Promise((rs, rj) => - setTimeout(() => { - rs({ metadata: { write_ts: '1699897723' }, data: 'completed', time: '01/01/2001' }); - }, 100), - ); + if (key == `manual/demographic_survey`) { + return new Promise((rs, rj) => + setTimeout(() => { + rs({ metadata: { write_ts: '1699897723' }, data: 'completed', time: '01/01/2001' }); + }, 100), + ); + } else { + return undefined; + } }, }; window['cordova'] ||= {}; From d123b4f23f90537671e13e59ff672f7156e655c2 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 17 Nov 2023 17:43:34 -0700 Subject: [PATCH 40/49] remove _lazyLoadConfig, move types the function was not all that necessary, so I removed it Also moved my survey config type into the appropriate place https://github.com/e-mission/e-mission-phone/pull/1063#discussion_r1397747847 --- www/__tests__/enketoHelper.test.ts | 6 ++--- www/js/survey/enketo/enketoHelper.ts | 36 +++++++--------------------- www/js/types/appConfigTypes.ts | 22 +++++++++++++++++ 3 files changed, 33 insertions(+), 31 deletions(-) create mode 100644 www/js/types/appConfigTypes.ts diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index e06dc94a6..0f2982318 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -3,12 +3,12 @@ import { filterByNameAndVersion, resolveTimestamps, resolveLabel, - _lazyLoadConfig, loadPreviousResponseForSurvey, saveResponse, } from '../js/survey/enketo/enketoHelper'; import { mockBEMUserCache } from '../__mocks__/cordovaMocks'; import { mockLogger } from '../__mocks__/globalMocks'; +import { getConfig } from '../../www/js/config/dynamicConfig'; import initializedI18next from '../js/i18nextInit'; window['i18next'] = initializedI18next; @@ -22,7 +22,7 @@ global.Blob = require('node:buffer').Blob; it('gets the survey config', async () => { //this is aimed at testing my mock of the config //mocked getDocument for the case of getting the config - let config = await _lazyLoadConfig(); + let config = await getConfig(); let mockSurveys = { TimeUseSurvey: { compatibleWith: 1, @@ -39,7 +39,7 @@ it('gets the survey config', async () => { version: 9, }, }; - expect(config).toMatchObject(mockSurveys); + expect(config.survey_info.surveys).toMatchObject(mockSurveys); }); it('gets the model response, if avaliable, or returns null', () => { diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index 379120373..cb5e0bac0 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -8,6 +8,7 @@ import { getConfig } from '../../config/dynamicConfig'; import { DateTime } from 'luxon'; import { fetchUrlCached } from '../../services/commHelper'; import { getUnifiedDataForInterval } from '../../services/unifiedDataLoader'; +import { EnketoSurveyConfig } from '../../types/appConfigTypes'; export type PrefillFields = { [key: string]: string }; @@ -34,19 +35,10 @@ type EnketoResponse = { metadata: any; }; -type EnketoSurveyConfig = { - [surveyName: string]: { - formPath: string; - labelTemplate: { [lang: string]: string }; - labelVars: { [activity: string]: { [key: string]: string; type: string } }; - version: number; - compatibleWith: number; - }; -}; - const LABEL_FUNCTIONS = { UseLabelTemplate: async (xmlDoc: XMLDocument, name: string) => { - let configSurveys = await _lazyLoadConfig(); + let appConfig = await getConfig(); + const configSurveys = appConfig.survey_info.surveys; const config = configSurveys[name]; // config for this survey const lang = i18next.resolvedLanguage; @@ -94,21 +86,6 @@ function _getAnswerByTagName(xmlDoc: XMLDocument, tagName: string) { /** @type {EnketoSurveyConfig} _config */ let _config: EnketoSurveyConfig; -/** - * _lazyLoadConfig load enketo survey config. If already loaded, return the cached config - * @returns {Promise} enketo survey config - */ -export function _lazyLoadConfig() { - if (_config !== undefined) { - return Promise.resolve(_config); - } - return getConfig().then((newConfig) => { - logInfo('Resolved UI_CONFIG_READY promise in enketoHelper, filling in templates'); - _config = newConfig.survey_info.surveys; - return _config; - }); -} - /** * filterByNameAndVersion filter the survey responses by survey name and their version. * The version for filtering is specified in enketo survey `compatibleWith` config. @@ -119,8 +96,11 @@ export function _lazyLoadConfig() { * @return {Promise} filtered survey responses */ export function filterByNameAndVersion(name: string, responses: EnketoResponse[]) { - return _lazyLoadConfig().then((config) => - responses.filter((r) => r.data.name === name && r.data.version >= config[name].compatibleWith), + return getConfig().then((config) => + responses.filter( + (r) => + r.data.name === name && r.data.version >= config.survey_info.surveys[name].compatibleWith, + ), ); } diff --git a/www/js/types/appConfigTypes.ts b/www/js/types/appConfigTypes.ts new file mode 100644 index 000000000..f55b27bc0 --- /dev/null +++ b/www/js/types/appConfigTypes.ts @@ -0,0 +1,22 @@ +// WIP: type definitions for the 'dynamic config' spec +// examples of configs: https://github.com/e-mission/nrel-openpath-deploy-configs/tree/main/configs + +export type AppConfig = { + server: ServerConnConfig; + [k: string]: any; // TODO fill in all the other fields +}; + +export type ServerConnConfig = { + connectUrl: `https://${string}`; + aggregate_call_auth: 'no_auth' | 'user_only' | 'never'; +}; + +export type EnketoSurveyConfig = { + [surveyName: string]: { + formPath: string; + labelTemplate: { [lang: string]: string }; + labelVars: { [activity: string]: { [key: string]: string; type: string } }; + version: number; + compatibleWith: number; + }; +}; From 100eefb76dd7bd118be1f78b8dd003a11c924ac2 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 17 Nov 2023 17:54:37 -0700 Subject: [PATCH 41/49] test the version adding a response that should get filtered out because the version is too low https://github.com/e-mission/e-mission-phone/pull/1063#discussion_r1397717837 --- www/__tests__/enketoHelper.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index 0f2982318..e81e1e15b 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -247,6 +247,18 @@ it('filters the survey responses by their name and version', () => { }, metadata: {}, }, + { + data: { + label: 'Activity', //display label (this value is use for displaying on the button) + ts: '100000000', //the timestamp at which the survey was filled out (in seconds) + fmt_time: '12:39', //the formatted timestamp at which the survey was filled out + name: 'TimeUseSurvey', //survey name + version: '0.5', //survey version + xmlResponse: '', //survey response XML string + jsonDocResponse: 'this is my json object', //survey response JSON object + }, + metadata: {}, + }, ]; //several responses -> only the one that has a name match From 22decbfd0b58776450737ba536dc9bf42e375aa5 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 17 Nov 2023 18:15:34 -0700 Subject: [PATCH 42/49] make config a parameter to mockBEMUserCache --- www/__mocks__/cordovaMocks.ts | 8 +++----- www/__tests__/clientStats.test.ts | 3 ++- www/__tests__/enketoHelper.test.ts | 3 ++- www/__tests__/pushNotifySettings.test.ts | 3 ++- www/__tests__/remoteNotifyHandler.test.ts | 3 ++- www/__tests__/startprefs.test.ts | 3 ++- www/__tests__/storage.test.ts | 3 ++- www/__tests__/storeDeviceSettings.test.ts | 3 ++- 8 files changed, 17 insertions(+), 12 deletions(-) diff --git a/www/__mocks__/cordovaMocks.ts b/www/__mocks__/cordovaMocks.ts index 4911b3ebe..e8b680965 100644 --- a/www/__mocks__/cordovaMocks.ts +++ b/www/__mocks__/cordovaMocks.ts @@ -35,7 +35,7 @@ export const mockFile = () => { //for consent document const _storage = {}; -export const mockBEMUserCache = () => { +export const mockBEMUserCache = (config) => { const _cache = {}; const messages = []; const mockBEMUserCache = { @@ -101,13 +101,11 @@ export const mockBEMUserCache = () => { ); }, getDocument: (key: string, withMetadata?: boolean) => { - // this was mocked specifically for enketoHelper's use, could be expanded if needed - const fakeSurveyConfig = fakeConfig; - + //returns the config provided as a paramenter to this mock! if (key == 'config/app_ui_config') { return new Promise((rs, rj) => setTimeout(() => { - rs(fakeSurveyConfig); + rs(config); }, 100), ); } else { diff --git a/www/__tests__/clientStats.test.ts b/www/__tests__/clientStats.test.ts index a3a953582..a508550e5 100644 --- a/www/__tests__/clientStats.test.ts +++ b/www/__tests__/clientStats.test.ts @@ -6,12 +6,13 @@ import { getAppVersion, statKeys, } from '../js/plugin/clientStats'; +import fakeConfig from '../__mocks__/fakeConfig.json'; mockDevice(); // this mocks cordova-plugin-app-version, generating a "Mock App", version "1.2.3" mockGetAppVersion(); // clientStats.ts uses BEMUserCache to store the stats, so we need to mock that too -mockBEMUserCache(); +mockBEMUserCache(fakeConfig); const db = window['cordova']?.plugins?.BEMUserCache; it('gets the app version', async () => { diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index e81e1e15b..3b5a95f3d 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -9,11 +9,12 @@ import { import { mockBEMUserCache } from '../__mocks__/cordovaMocks'; import { mockLogger } from '../__mocks__/globalMocks'; import { getConfig } from '../../www/js/config/dynamicConfig'; +import fakeConfig from '../__mocks__/fakeConfig.json'; import initializedI18next from '../js/i18nextInit'; window['i18next'] = initializedI18next; -mockBEMUserCache(); +mockBEMUserCache(fakeConfig); mockLogger(); global.URL = require('url').URL; diff --git a/www/__tests__/pushNotifySettings.test.ts b/www/__tests__/pushNotifySettings.test.ts index d452aa819..9e6e25bb5 100644 --- a/www/__tests__/pushNotifySettings.test.ts +++ b/www/__tests__/pushNotifySettings.test.ts @@ -11,11 +11,12 @@ import { mockPushNotification, getCalled, } from '../__mocks__/pushNotificationMocks'; +import fakeConfig from '../__mocks__/fakeConfig.json'; mockCordova(); mockLogger(); mockPushNotification(); -mockBEMUserCache(); +mockBEMUserCache(fakeConfig); mockBEMDataCollection(); global.fetch = (url: string) => diff --git a/www/__tests__/remoteNotifyHandler.test.ts b/www/__tests__/remoteNotifyHandler.test.ts index 320877c6b..6fe0a73fe 100644 --- a/www/__tests__/remoteNotifyHandler.test.ts +++ b/www/__tests__/remoteNotifyHandler.test.ts @@ -9,10 +9,11 @@ import { mockInAppBrowser, } from '../__mocks__/cordovaMocks'; import { clearAlerts, getAlerts, mockAlert, mockLogger } from '../__mocks__/globalMocks'; +import fakeConfig from '../__mocks__/fakeConfig.json'; mockLogger(); mockDevice(); -mockBEMUserCache(); +mockBEMUserCache(fakeConfig); mockGetAppVersion(); mockInAppBrowser(); mockAlert(); diff --git a/www/__tests__/startprefs.test.ts b/www/__tests__/startprefs.test.ts index 75ed707dc..17b44a4be 100644 --- a/www/__tests__/startprefs.test.ts +++ b/www/__tests__/startprefs.test.ts @@ -7,8 +7,9 @@ import { import { mockBEMUserCache, mockBEMDataCollection } from '../__mocks__/cordovaMocks'; import { mockLogger } from '../__mocks__/globalMocks'; +import fakeConfig from '../__mocks__/fakeConfig.json'; -mockBEMUserCache(); +mockBEMUserCache(fakeConfig); mockBEMDataCollection(); mockLogger(); diff --git a/www/__tests__/storage.test.ts b/www/__tests__/storage.test.ts index ca6d71dec..bbfa9c410 100644 --- a/www/__tests__/storage.test.ts +++ b/www/__tests__/storage.test.ts @@ -1,11 +1,12 @@ import { mockBEMUserCache } from '../__mocks__/cordovaMocks'; import { mockLogger } from '../__mocks__/globalMocks'; import { storageClear, storageGet, storageRemove, storageSet } from '../js/plugin/storage'; +import fakeConfig from '../__mocks__/fakeConfig.json'; // mocks used - storage.ts uses BEMUserCache and logging. // localStorage is already mocked for us by Jest :) mockLogger(); -mockBEMUserCache(); +mockBEMUserCache(fakeConfig); it('stores a value and retrieves it back', async () => { await storageSet('test1', 'test value 1'); diff --git a/www/__tests__/storeDeviceSettings.test.ts b/www/__tests__/storeDeviceSettings.test.ts index 4bccbc0af..41c36eb16 100644 --- a/www/__tests__/storeDeviceSettings.test.ts +++ b/www/__tests__/storeDeviceSettings.test.ts @@ -13,8 +13,9 @@ import { import { mockLogger } from '../__mocks__/globalMocks'; import { EVENTS, publish } from '../js/customEventHandler'; import { markIntroDone } from '../js/onboarding/onboardingHelper'; +import fakeConfig from '../__mocks__/fakeConfig.json'; -mockBEMUserCache(); +mockBEMUserCache(fakeConfig); mockDevice(); mockCordova(); mockLogger(); From 80aa5ea18081849819236d65dc5d07ea583aa8e8 Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Fri, 17 Nov 2023 18:42:33 -0700 Subject: [PATCH 43/49] more test cases!! Because of the new parameter, I was able to add more test cases to resolve Labels I needed to also clear the locally stored config out of dynamicConfig.ts --- www/__tests__/enketoHelper.test.ts | 81 ++++++++++++++++++++++++++++-- www/js/config/dynamicConfig.ts | 5 ++ 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/www/__tests__/enketoHelper.test.ts b/www/__tests__/enketoHelper.test.ts index 3b5a95f3d..c4fda7dc4 100644 --- a/www/__tests__/enketoHelper.test.ts +++ b/www/__tests__/enketoHelper.test.ts @@ -8,7 +8,7 @@ import { } from '../js/survey/enketo/enketoHelper'; import { mockBEMUserCache } from '../__mocks__/cordovaMocks'; import { mockLogger } from '../__mocks__/globalMocks'; -import { getConfig } from '../../www/js/config/dynamicConfig'; +import { getConfig, resetStoredConfig } from '../../www/js/config/dynamicConfig'; import fakeConfig from '../__mocks__/fakeConfig.json'; import initializedI18next from '../js/i18nextInit'; @@ -20,6 +20,10 @@ mockLogger(); global.URL = require('url').URL; global.Blob = require('node:buffer').Blob; +beforeEach(() => { + resetStoredConfig(); +}); + it('gets the survey config', async () => { //this is aimed at testing my mock of the config //mocked getDocument for the case of getting the config @@ -106,7 +110,7 @@ it('resolves the timestamps', () => { }); //resolve label -it('resolves the label', async () => { +it('resolves the label, normal case', async () => { const xmlParser = new window.DOMParser(); const xmlString = ' option_1 '; const xmlDoc = xmlParser.parseFromString(xmlString, 'text/html'); @@ -114,14 +118,83 @@ it('resolves the label', async () => { ' option_1 option_3 '; const xmlDoc2 = xmlParser.parseFromString(xmlString2, 'text/xml'); - //if no template, returns "Answered" TODO: find a way to engineer this case - //if no labelVars, returns template TODO: find a way to engineer this case //have a custom survey label function TODO: we currently don't have custome label functions, but should test when we do //no custom function, fallback to UseLabelTemplate (standard case) + mockBEMUserCache(fakeConfig); expect(await resolveLabel('TimeUseSurvey', xmlDoc)).toBe('3 Domestic'); expect(await resolveLabel('TimeUseSurvey', xmlDoc2)).toBe('3 Employment/Education, 3 Domestic'); }); +it('resolves the label, if no template, returns "Answered"', async () => { + const xmlParser = new window.DOMParser(); + const xmlString = ' option_1 '; + const xmlDoc = xmlParser.parseFromString(xmlString, 'text/html'); + const xmlString2 = + ' option_1 option_3 '; + const xmlDoc2 = xmlParser.parseFromString(xmlString2, 'text/xml'); + + const noTemplate = { + survey_info: { + surveys: { + TimeUseSurvey: { + compatibleWith: 1, + formPath: + 'https://raw.githubusercontent.com/sebastianbarry/nrel-openpath-deploy-configs/surveys-info-and-surveys-data/survey-resources/data-json/time-use-survey-form-v9.json', + labelVars: { + da: { + key: 'Domestic_activities', + type: 'length', + }, + erea: { + key: 'Employment_related_a_Education_activities', + type: 'length', + }, + }, + version: 9, + }, + }, + 'trip-labels': 'ENKETO', + }, + }; + mockBEMUserCache(noTemplate); + expect(await resolveLabel('TimeUseSurvey', xmlDoc)).toBe('Answered'); + expect(await resolveLabel('TimeUseSurvey', xmlDoc2)).toBe('Answered'); +}); + +it('resolves the label, if no labelVars, returns template', async () => { + const xmlParser = new window.DOMParser(); + const xmlString = ' option_1 '; + const xmlDoc = xmlParser.parseFromString(xmlString, 'text/html'); + const xmlString2 = + ' option_1 option_3 '; + const xmlDoc2 = xmlParser.parseFromString(xmlString2, 'text/xml'); + + const noLabels = { + survey_info: { + surveys: { + TimeUseSurvey: { + compatibleWith: 1, + formPath: + 'https://raw.githubusercontent.com/sebastianbarry/nrel-openpath-deploy-configs/surveys-info-and-surveys-data/survey-resources/data-json/time-use-survey-form-v9.json', + labelTemplate: { + en: '{ erea, plural, =0 {} other {# Employment/Education, } }{ da, plural, =0 {} other {# Domestic, } }', + es: '{ erea, plural, =0 {} other {# Empleo/Educación, } }{ da, plural, =0 {} other {# Actividades domesticas, }}', + }, + version: 9, + }, + }, + 'trip-labels': 'ENKETO', + }, + }; + mockBEMUserCache(noLabels); + expect(await resolveLabel('TimeUseSurvey', xmlDoc)).toBe( + '{ erea, plural, =0 {} other {# Employment/Education, } }{ da, plural, =0 {} other {# Domestic, } }', + ); + expect(await resolveLabel('TimeUseSurvey', xmlDoc2)).toBe( + '{ erea, plural, =0 {} other {# Employment/Education, } }{ da, plural, =0 {} other {# Domestic, } }', + ); +}); + /** * @param surveyName the name of the survey (e.g. "TimeUseSurvey") * @param enketoForm the Form object from enketo-core that contains this survey diff --git a/www/js/config/dynamicConfig.ts b/www/js/config/dynamicConfig.ts index eb709c16c..801e24b07 100644 --- a/www/js/config/dynamicConfig.ts +++ b/www/js/config/dynamicConfig.ts @@ -10,6 +10,11 @@ export let storedConfig = null; export let configChanged = false; export const setConfigChanged = (b) => (configChanged = b); +//used test multiple configs, not used outside of test +export const resetStoredConfig = function () { + storedConfig = null; +}; + const _getStudyName = function (connectUrl) { const orig_host = new URL(connectUrl).hostname; const first_domain = orig_host.split('.')[0]; From 0ef1db81e574d44f9fe09858a9c24ffc6175904a Mon Sep 17 00:00:00 2001 From: Abby Wheelis <54848919+Abby-Wheelis@users.noreply.github.com> Date: Sun, 19 Nov 2023 08:33:47 -0700 Subject: [PATCH 44/49] add survey_info to the AppConfig Type from @JGreenlee's suggestion in review Co-authored-by: Jack Greenlee --- www/js/types/appConfigTypes.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/www/js/types/appConfigTypes.ts b/www/js/types/appConfigTypes.ts index f55b27bc0..07e8ccb5f 100644 --- a/www/js/types/appConfigTypes.ts +++ b/www/js/types/appConfigTypes.ts @@ -3,6 +3,10 @@ export type AppConfig = { server: ServerConnConfig; + survey_info: { + 'trip-labels': 'MULTILABEL' | 'ENKETO'; + surveys: EnketoSurveyConfig; + } [k: string]: any; // TODO fill in all the other fields }; From 10b954669a277210ac129892aa1ac61682aa8fa4 Mon Sep 17 00:00:00 2001 From: Abby Wheelis <54848919+Abby-Wheelis@users.noreply.github.com> Date: Sun, 19 Nov 2023 08:36:24 -0700 Subject: [PATCH 45/49] make config optional param not all (in fact many) of the tests don't need this config at all, so the parameter should be optional to clean things up Co-authored-by: Jack Greenlee --- www/__mocks__/cordovaMocks.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/__mocks__/cordovaMocks.ts b/www/__mocks__/cordovaMocks.ts index e8b680965..08293e73f 100644 --- a/www/__mocks__/cordovaMocks.ts +++ b/www/__mocks__/cordovaMocks.ts @@ -35,7 +35,7 @@ export const mockFile = () => { //for consent document const _storage = {}; -export const mockBEMUserCache = (config) => { +export const mockBEMUserCache = (config?) => { const _cache = {}; const messages = []; const mockBEMUserCache = { From fd0103427627dab2a38d7cbb46ddaaabc8df313c Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Sun, 19 Nov 2023 08:43:44 -0700 Subject: [PATCH 46/49] fallback to fakeConfig follow-on to making the config an optional parameter to mockBEMUserCache Instead of passing it in EVERY TIME, it is now a fallback, and the config only needs to be specified and re-specified in the enketoHelper tests added fallback and removed specification from tests that didn't need it --- www/__mocks__/cordovaMocks.ts | 2 +- www/__tests__/clientStats.test.ts | 3 +-- www/__tests__/pushNotifySettings.test.ts | 3 +-- www/__tests__/remoteNotifyHandler.test.ts | 3 +-- www/__tests__/startprefs.test.ts | 3 +-- www/__tests__/storage.test.ts | 3 +-- www/__tests__/storeDeviceSettings.test.ts | 3 +-- 7 files changed, 7 insertions(+), 13 deletions(-) diff --git a/www/__mocks__/cordovaMocks.ts b/www/__mocks__/cordovaMocks.ts index 08293e73f..1d3934ea4 100644 --- a/www/__mocks__/cordovaMocks.ts +++ b/www/__mocks__/cordovaMocks.ts @@ -105,7 +105,7 @@ export const mockBEMUserCache = (config?) => { if (key == 'config/app_ui_config') { return new Promise((rs, rj) => setTimeout(() => { - rs(config); + rs(config || fakeConfig); }, 100), ); } else { diff --git a/www/__tests__/clientStats.test.ts b/www/__tests__/clientStats.test.ts index a508550e5..a3a953582 100644 --- a/www/__tests__/clientStats.test.ts +++ b/www/__tests__/clientStats.test.ts @@ -6,13 +6,12 @@ import { getAppVersion, statKeys, } from '../js/plugin/clientStats'; -import fakeConfig from '../__mocks__/fakeConfig.json'; mockDevice(); // this mocks cordova-plugin-app-version, generating a "Mock App", version "1.2.3" mockGetAppVersion(); // clientStats.ts uses BEMUserCache to store the stats, so we need to mock that too -mockBEMUserCache(fakeConfig); +mockBEMUserCache(); const db = window['cordova']?.plugins?.BEMUserCache; it('gets the app version', async () => { diff --git a/www/__tests__/pushNotifySettings.test.ts b/www/__tests__/pushNotifySettings.test.ts index 9e6e25bb5..d452aa819 100644 --- a/www/__tests__/pushNotifySettings.test.ts +++ b/www/__tests__/pushNotifySettings.test.ts @@ -11,12 +11,11 @@ import { mockPushNotification, getCalled, } from '../__mocks__/pushNotificationMocks'; -import fakeConfig from '../__mocks__/fakeConfig.json'; mockCordova(); mockLogger(); mockPushNotification(); -mockBEMUserCache(fakeConfig); +mockBEMUserCache(); mockBEMDataCollection(); global.fetch = (url: string) => diff --git a/www/__tests__/remoteNotifyHandler.test.ts b/www/__tests__/remoteNotifyHandler.test.ts index 6fe0a73fe..320877c6b 100644 --- a/www/__tests__/remoteNotifyHandler.test.ts +++ b/www/__tests__/remoteNotifyHandler.test.ts @@ -9,11 +9,10 @@ import { mockInAppBrowser, } from '../__mocks__/cordovaMocks'; import { clearAlerts, getAlerts, mockAlert, mockLogger } from '../__mocks__/globalMocks'; -import fakeConfig from '../__mocks__/fakeConfig.json'; mockLogger(); mockDevice(); -mockBEMUserCache(fakeConfig); +mockBEMUserCache(); mockGetAppVersion(); mockInAppBrowser(); mockAlert(); diff --git a/www/__tests__/startprefs.test.ts b/www/__tests__/startprefs.test.ts index 17b44a4be..75ed707dc 100644 --- a/www/__tests__/startprefs.test.ts +++ b/www/__tests__/startprefs.test.ts @@ -7,9 +7,8 @@ import { import { mockBEMUserCache, mockBEMDataCollection } from '../__mocks__/cordovaMocks'; import { mockLogger } from '../__mocks__/globalMocks'; -import fakeConfig from '../__mocks__/fakeConfig.json'; -mockBEMUserCache(fakeConfig); +mockBEMUserCache(); mockBEMDataCollection(); mockLogger(); diff --git a/www/__tests__/storage.test.ts b/www/__tests__/storage.test.ts index bbfa9c410..ca6d71dec 100644 --- a/www/__tests__/storage.test.ts +++ b/www/__tests__/storage.test.ts @@ -1,12 +1,11 @@ import { mockBEMUserCache } from '../__mocks__/cordovaMocks'; import { mockLogger } from '../__mocks__/globalMocks'; import { storageClear, storageGet, storageRemove, storageSet } from '../js/plugin/storage'; -import fakeConfig from '../__mocks__/fakeConfig.json'; // mocks used - storage.ts uses BEMUserCache and logging. // localStorage is already mocked for us by Jest :) mockLogger(); -mockBEMUserCache(fakeConfig); +mockBEMUserCache(); it('stores a value and retrieves it back', async () => { await storageSet('test1', 'test value 1'); diff --git a/www/__tests__/storeDeviceSettings.test.ts b/www/__tests__/storeDeviceSettings.test.ts index 41c36eb16..4bccbc0af 100644 --- a/www/__tests__/storeDeviceSettings.test.ts +++ b/www/__tests__/storeDeviceSettings.test.ts @@ -13,9 +13,8 @@ import { import { mockLogger } from '../__mocks__/globalMocks'; import { EVENTS, publish } from '../js/customEventHandler'; import { markIntroDone } from '../js/onboarding/onboardingHelper'; -import fakeConfig from '../__mocks__/fakeConfig.json'; -mockBEMUserCache(fakeConfig); +mockBEMUserCache(); mockDevice(); mockCordova(); mockLogger(); From 83a773ec79096cd90fce36e18d5bb1a8f5e8610d Mon Sep 17 00:00:00 2001 From: Abby Wheelis Date: Sun, 19 Nov 2023 08:45:25 -0700 Subject: [PATCH 47/49] prettier types --- www/js/types/appConfigTypes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/js/types/appConfigTypes.ts b/www/js/types/appConfigTypes.ts index 07e8ccb5f..aa2e3f312 100644 --- a/www/js/types/appConfigTypes.ts +++ b/www/js/types/appConfigTypes.ts @@ -6,7 +6,7 @@ export type AppConfig = { survey_info: { 'trip-labels': 'MULTILABEL' | 'ENKETO'; surveys: EnketoSurveyConfig; - } + }; [k: string]: any; // TODO fill in all the other fields }; From becc4af0d724e676bedde73f3fb145fcdc211485 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Mon, 20 Nov 2023 10:41:35 -0500 Subject: [PATCH 48/49] enketoHelper: expand typings for resolveTimestamps -add types for the parameters of resolveTimestamps -update doc of this function -cast 'timelineEntry' to trip or place where needed -add a few fields that were missing from type defs of ConfirmedPlace and EnketoSurveyConfig --- www/js/survey/enketo/enketoHelper.ts | 28 ++++++++++++++++++---------- www/js/types/appConfigTypes.ts | 1 + www/js/types/diaryTypes.ts | 12 ++++++++++-- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index cb5e0bac0..933ca3aed 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -8,7 +8,8 @@ import { getConfig } from '../../config/dynamicConfig'; import { DateTime } from 'luxon'; import { fetchUrlCached } from '../../services/commHelper'; import { getUnifiedDataForInterval } from '../../services/unifiedDataLoader'; -import { EnketoSurveyConfig } from '../../types/appConfigTypes'; +import { AppConfig, EnketoSurveyConfig } from '../../types/appConfigTypes'; +import { CompositeTrip, ConfirmedPlace, TimelineEntry } from '../../types/diaryTypes'; export type PrefillFields = { [key: string]: string }; @@ -151,12 +152,12 @@ export function getInstanceStr(xmlModel: string, opts: SurveyOptions): string | /** * resolve timestamps label from the survey response * @param {XMLDocument} xmlDoc survey response as XML object - * @param {object} trip trip object + * @param {object} timelineEntry trip or place object * @returns {object} object with `start_ts` and `end_ts` * - null if no timestamps are resolved * - undefined if the timestamps are invalid */ -export function resolveTimestamps(xmlDoc, timelineEntry) { +export function resolveTimestamps(xmlDoc: XMLDocument, timelineEntry: TimelineEntry) { // check for Date and Time fields const startDate = xmlDoc.getElementsByTagName('Start_date')?.[0]?.innerHTML; let startTime = xmlDoc.getElementsByTagName('Start_time')?.[0]?.innerHTML; @@ -167,10 +168,10 @@ export function resolveTimestamps(xmlDoc, timelineEntry) { if (!startDate || !startTime || !endDate || !endTime) return null; const timezone = - timelineEntry.start_local_dt?.timezone || - timelineEntry.enter_local_dt?.timezone || - timelineEntry.end_local_dt?.timezone || - timelineEntry.exit_local_dt?.timezone; + (timelineEntry as CompositeTrip).start_local_dt?.timezone || + (timelineEntry as ConfirmedPlace).enter_local_dt?.timezone || + (timelineEntry as CompositeTrip).end_local_dt?.timezone || + (timelineEntry as ConfirmedPlace).exit_local_dt?.timezone; // split by + or - to get time without offset startTime = startTime.split(/\-|\+/)[0]; endTime = endTime.split(/\-|\+/)[0]; @@ -188,8 +189,10 @@ export function resolveTimestamps(xmlDoc, timelineEntry) { the millisecond. To avoid precision issues, we will check if the start/end timestamps from the survey response are within the same minute as the start/end or enter/exit timestamps. If so, we will use the exact trip/place timestamps */ - const entryStartTs = timelineEntry.start_ts || timelineEntry.enter_ts; - const entryEndTs = timelineEntry.end_ts || timelineEntry.exit_ts; + const entryStartTs = + (timelineEntry as CompositeTrip).start_ts || (timelineEntry as ConfirmedPlace).enter_ts; + const entryEndTs = + (timelineEntry as CompositeTrip).end_ts || (timelineEntry as ConfirmedPlace).exit_ts; if (additionStartTs - (additionStartTs % 60) == entryStartTs - (entryStartTs % 60)) additionStartTs = entryStartTs; if (additionEndTs - (additionEndTs % 60) == entryEndTs - (entryEndTs % 60)) @@ -209,7 +212,12 @@ export function resolveTimestamps(xmlDoc, timelineEntry) { * @param opts object with SurveyOptions like 'timelineEntry' or 'dataKey' * @returns Promise of the saved result, or an Error if there was a problem */ -export function saveResponse(surveyName: string, enketoForm: Form, appConfig, opts: SurveyOptions) { +export function saveResponse( + surveyName: string, + enketoForm: Form, + appConfig: AppConfig, + opts: SurveyOptions, +) { const xmlParser = new window.DOMParser(); const xmlResponse = enketoForm.getDataStr(); const xmlDoc = xmlParser.parseFromString(xmlResponse, 'text/xml'); diff --git a/www/js/types/appConfigTypes.ts b/www/js/types/appConfigTypes.ts index aa2e3f312..1a2e50722 100644 --- a/www/js/types/appConfigTypes.ts +++ b/www/js/types/appConfigTypes.ts @@ -22,5 +22,6 @@ export type EnketoSurveyConfig = { labelVars: { [activity: string]: { [key: string]: string; type: string } }; version: number; compatibleWith: number; + dataKey?: string; }; }; diff --git a/www/js/types/diaryTypes.ts b/www/js/types/diaryTypes.ts index 743d75b15..7cce67923 100644 --- a/www/js/types/diaryTypes.ts +++ b/www/js/types/diaryTypes.ts @@ -5,14 +5,17 @@ import { BaseModeKey, MotionTypeKey } from '../diary/diaryHelper'; type ObjectId = { $oid: string }; -type ConfirmedPlace = { +export type ConfirmedPlace = { _id: ObjectId; additions: UserInputEntry[]; cleaned_place: ObjectId; ending_trip: ObjectId; - enter_fmt_time: string; // ISO string 2023-10-31T12:00:00.000-04:00 + enter_fmt_time: string; // ISO string e.g. 2023-10-31T12:00:00.000-04:00 enter_local_dt: LocalDt; enter_ts: number; // Unix timestamp + exit_fmt_time: string; // ISO string e.g. 2023-10-31T12:00:00.000-04:00 + exit_local_dt: LocalDt; + exit_ts: number; // Unix timestamp key: string; location: { type: string; coordinates: number[] }; origin_key: string; @@ -76,6 +79,11 @@ export type CompositeTrip = { so a 'timeline entry' is either a trip or a place. */ export type TimelineEntry = ConfirmedPlace | CompositeTrip; +/* Type guard to disambiguate timeline entries as either trips or places + If it has a 'start_ts' and 'end_ts', it's a trip. Else, it's a place. */ +export const isTrip = (entry: TimelineEntry): entry is CompositeTrip => + entry.hasOwnProperty('start_ts') && entry.hasOwnProperty('end_ts'); + /* These properties aren't received from the server, but are derived from the above properties. They are used in the UI to display trip/place details and are computed by the useDerivedProperties hook. */ export type DerivedProperties = { From 828ee1279fa2ad866e3753d4894c8f95888eac65 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Tue, 21 Nov 2023 11:55:54 -0500 Subject: [PATCH 49/49] enketoHelper: fix error if timestamps are null if the survey doesn't use start and end times, timestamps will be null. This is fine and we can use optional chaining to prevent throwing an error here --- www/js/survey/enketo/enketoHelper.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/www/js/survey/enketo/enketoHelper.ts b/www/js/survey/enketo/enketoHelper.ts index 9b96c8463..d3ad13c98 100644 --- a/www/js/survey/enketo/enketoHelper.ts +++ b/www/js/survey/enketo/enketoHelper.ts @@ -250,8 +250,8 @@ export function saveResponse(surveyName: string, enketoForm: Form, appConfig, op return new Error(i18next.t('survey.enketo-timestamps-invalid')); //"Timestamps are invalid. Please ensure that the start time is before the end time."); } // if timestamps were not resolved from the survey, we will use the trip or place timestamps - data.start_ts = timestamps.start_ts || opts.timelineEntry.enter_ts; - data.end_ts = timestamps.end_ts || opts.timelineEntry.exit_ts; + data.start_ts = timestamps?.start_ts || opts.timelineEntry.enter_ts; + data.end_ts = timestamps?.end_ts || opts.timelineEntry.exit_ts; // UUID generated using this method https://stackoverflow.com/a/66332305 data.match_id = URL.createObjectURL(new Blob([])).slice(-36); } else {