Skip to content

[FLAG-943] Tropical Tree Cover analysis malfunctioning #4707

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Oct 20, 2023
Merged

[FLAG-943] Tropical Tree Cover analysis malfunctioning #4707

Changes from 2 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
601c39a
fix(ttc): add tree cover density validation to getWHEREQuery method
willian-viana Oct 16, 2023
abc47eb
fix(ttc): refactor getWHEREQuery
willian-viana Oct 17, 2023
d471332
chore(analysis-cached): move getWhereQuery and dependencies to a diff…
willian-viana Oct 20, 2023
93ab2b5
chore(jest): add absolute path config
willian-viana Oct 20, 2023
3d29ded
chore(jest): add getWhereQuery test for tree cover density widget ins…
willian-viana Oct 20, 2023
fcb2b14
chore(jest): add getWhereQuery general test
willian-viana Oct 20, 2023
c860d12
chore(jest): add a non expected example
willian-viana Oct 20, 2023
84b776d
chore(jest): improve readability
willian-viana Oct 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
205 changes: 120 additions & 85 deletions services/analysis-cached.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,14 @@ const SQL_QUERIES = {

const ALLOWED_PARAMS = {
annual: ['adm0', 'adm1', 'adm2', 'threshold', 'forestType', 'landCategory'],
treeCoverDensity: [
'adm0',
'adm1',
'adm2',
'threshold',
'forestType',
'landCategory',
],
integrated_alerts: [
'adm0',
'adm1',
Expand Down Expand Up @@ -305,99 +313,123 @@ const getLocationSelect = ({
);
};

const isNumber = (value) => !!(typeof value === 'number' || !isNaN(value));

const translateParameterKey = (parameterKey, { type, dataset }) => {
let paramKey = '';

if (parameterKey === 'confidence') paramKey = 'confidence__cat';
if (parameterKey === 'adm0' && type === 'country') paramKey = 'iso';
if (parameterKey === 'adm1' && type === 'country') paramKey = 'adm1';
if (parameterKey === 'adm2' && type === 'country') paramKey = 'adm2';
if (parameterKey === 'adm0' && type === 'geostore') paramKey = 'geostore__id';
if (parameterKey === 'adm0' && type === 'wdpa')
paramKey = 'wdpa_protected_area__id';

if (parameterKey === 'threshold') {
if (dataset === 'tropicalTreeCover') {
paramKey = 'wri_tropical_tree_cover__decile';
} else {
paramKey = 'umd_tree_cover_density_2000__threshold';
}
}

return paramKey;
};

// build {where} statement for query
export const getWHEREQuery = (params) => {
const allPolynames = forestTypes.concat(landCategories);
const paramKeys = params && Object.keys(params);
const allowedParams = ALLOWED_PARAMS[params.dataset || 'annual'];
const paramKeysFiltered = paramKeys.filter(
(p) => (params[p] || p === 'threshold') && allowedParams.includes(p)
);
export const getWHEREQuery = (params = {}) => {
const { type, dataset } = params || {};
let comparisonString = ' = ';
if (paramKeysFiltered && paramKeysFiltered.length) {
let paramString = 'WHERE ';
paramKeysFiltered.forEach((p, i) => {
const isLast = paramKeysFiltered.length - 1 === i;
const isPolyname = ['forestType', 'landCategory'].includes(p);
const value = isPolyname ? 1 : params[p];
const polynameMeta = allPolynames.find(
(pname) => pname.value === params[p]
);
const tableKey =
polynameMeta &&
(polynameMeta.tableKey || polynameMeta.tableKeys[dataset || 'annual']);

/* TODO
perform better casting / allow to configure types:
AS for example wdpa_protected_area__id needs to be a string,
even that it evaluates AS a number.
Note that the postgres tables will allow us to cast at the query level.
*/
// const zeroString = polynameMeta?.dataType === 'keyword' ? "'0'" : '0';
let isNumericValue = !!(
typeof value === 'number' ||
(!isNaN(value) && !['adm0', 'confidence'].includes(p))
);

let paramKey = p;
if (p === 'confidence') paramKey = 'confidence__cat';
if (p === 'threshold') {
// paramKey = 'umd_tree_cover_density__threshold';
comparisonString = ' = ';

if (dataset === 'tropicalTreeCover') {
paramKey = 'wri_tropical_tree_cover__decile';
} else {
paramKey = 'umd_tree_cover_density_2000__threshold';
}

// }
}
if (p === 'adm0' && type === 'country') paramKey = 'iso';
if (p === 'adm1' && type === 'country') paramKey = 'adm1';
if (p === 'adm2' && type === 'country') paramKey = 'adm2';
if (p === 'adm0' && type === 'geostore') paramKey = 'geostore__id';
if (p === 'adm0' && type === 'wdpa') {
paramKey = 'wdpa_protected_area__id';
isNumericValue = false;

const allFilterOptions = forestTypes.concat(landCategories);
const allowedParams = ALLOWED_PARAMS[params.dataset || 'annual'];
const isTreeCoverDensity = dataset === 'treeCoverDensity';
const comparisonString = ' = ';

let paramString = 'WHERE ';
let paramKeys = Object.keys(params).filter((parameterName) => {
return (
(params[parameterName] || parameterName === 'threshold') &&
allowedParams.includes(parameterName)
);
});

if (!paramKeys?.length) {
return '';
}

/*
* Removing threshold from Tree Cover Density request
* Tree Cover Density has a default threshold >=10
*/
if (isTreeCoverDensity && paramKeys.includes('threshold')) {
paramKeys = paramKeys.filter((item) => item !== 'threshold');
}

paramKeys.forEach((parameter, index) => {
const isLastParameter = paramKeys.length - 1 === index;
const hasFilterOption = ['forestType', 'landCategory'].includes(parameter);
const value = hasFilterOption ? 1 : params[parameter];
const polynameMeta = allFilterOptions.find(
(pname) => pname.value === params[parameter]
);
const tableKey =
polynameMeta &&
(polynameMeta.tableKey || polynameMeta.tableKeys[dataset || 'annual']);
let isNumericValue = isNumber(value);

const paramKey = translateParameterKey(parameter, params);

if (parameter === 'adm0' && type === 'wdpa') {
isNumericValue = false;
}

if (dataset === 'net_change') {
isNumericValue = false;
}

const hasPrefixIs__ = hasFilterOption && tableKey.includes('is__');
let WHERE = '';

if (hasFilterOption) {
if (hasPrefixIs__) {
WHERE = `${WHERE}${tableKey} = 'true'`;
}
if (dataset === 'net_change') {
isNumericValue = false;

if (!hasPrefixIs__) {
WHERE = `${WHERE}${tableKey} IS NOT NULL`;
}

const polynameString = `
${
isPolyname && tableKey.includes('is__') ? `${tableKey} = 'true'` : ''
}${
isPolyname && !tableKey.includes('is__')
? `${tableKey} IS NOT NULL`
: ''
}${
isPolyname &&
if (
polynameMeta &&
!tableKey.includes('is__') &&
!hasPrefixIs__ &&
polynameMeta.default &&
polynameMeta.categories
? ` AND ${tableKey} ${polynameMeta.comparison || '='} ${
polynameMeta?.dataType === 'keyword'
? `'${polynameMeta?.default}'`
: `${polynameMeta?.default}`
}`
: ''
}${
!isPolyname
? `${paramKey}${comparisonString}${
isNumericValue ? value : `'${value}'`
}`
: ''
}${isLast ? '' : ' AND '}`;
) {
WHERE = `${WHERE} AND ${tableKey} ${polynameMeta.comparison || '='} ${
polynameMeta?.dataType === 'keyword'
? `'${polynameMeta?.default}'`
: `${polynameMeta?.default}`
}`;
}
}

paramString = paramString.concat(polynameString);
});
return paramString;
}
return '';
if (!hasFilterOption) {
WHERE = `${WHERE}${paramKey}${comparisonString}${
isNumericValue ? value : `'${value}'`
}`;
}

if (isLastParameter) {
WHERE = `${WHERE} `;
} else {
WHERE = `${WHERE} AND `;
}

paramString = paramString.concat(WHERE);
});

return paramString;
};

export const getDatesFilter = ({ startDate }) => {
Expand Down Expand Up @@ -2065,7 +2097,10 @@ export const getTreeCoverDensity = (params) => {
getLocationSelect({ ...params, cast: false })
)
.replace(/{location}/g, getLocationSelect({ ...params }))
.replace('{WHERE}', getWHEREQuery({ ...params, dataset: 'annual' }))
.replace(
'{WHERE}',
getWHEREQuery({ ...params, dataset: 'treeCoverDensity' })
)
);

return dataRequest.get(url).then((response) => response.data);
Expand Down