FLAG-1263: Data mart api client

components/widgets/forest-change/tree-loss-drivers/index.js

@@ -1,5 +1,5 @@
 import { all, spread } from 'axios';
 import { getYearsRangeFromMinMax } from 'components/widgets/utils/data';
 
 import {
   POLITICAL_BOUNDARIES_DATASET,
@@ -14,10 +14,12 @@
 import treeLoss from 'components/widgets/forest-change/tree-loss';
 import { getExtent, getLoss } from 'services/analysis-cached';
 
+import { fetchDataMart } from 'services/datamart';
 import getWidgetProps from './selectors';
 
+
 const MIN_YEAR = 2001;
 const MAX_YEAR = 2023;
 
 export default {
   ...treeLoss,
@@ -121,8 +123,36 @@
   whitelists: {
     checkStatus: true,
   },
-  getData: (params) =>
-    all([
+  getData: async (params) => {
+    const {
+      adm0,
+      adm1,
+      adm2,
+      analysis,
+      dashboard,
+      geostore,
+      threshold,
+      type, // country, global etc
+    } = params;
+    const dataset = 'tree_cover_loss_by_driver';
+
+    // TODO: depending on type, send either geostore or adm0, adm1 etc
+    const response = await fetchDataMart({
+      dataset,
+      geostoreId: 'c3833748f6815d31bad47d47f147c0f0',
+      isGlobal: false,
+      adm0: '',
+      adm1: '',
+      adm2: '',
+      isAnalyis: true,
+      threshold: 27,
+      isDownload: false,
+    });
+
+    return response;
+
+    /*
+    return all([
       getLoss({ ...params, landCategory: 'tsc', lossTsc: true }),
       getExtent({ ...params }),
     ]).then(
@@ -154,7 +184,9 @@
           },
         };
       })
-    ),
+    );
+    */
+  },
   getDataURL: (params) => [
     getLoss({
       ...params,

pages/api/datamart/v0/land/[...slug].js

@@ -0,0 +1,117 @@
+// eslint-disable-next-line no-unused-vars
+import { NextApiRequest, NextApiResponse } from 'next';
+import {
+  createRequestByGeostoryId,
+  getDataByGeostoreId,
+  getDataFromLink,
+} from 'services/datamart';
+import { GFW_DATA_API, GFW_STAGING_DATA_API } from 'utils/apis';
+
+// types
+/**
+ * @typedef {object} DataLinkObject
+ * @property {string} link - The URL to POST the content.
+ */
+
+/**
+ * @typedef {object} GetResponseObject
+ * @property {string} status - status.
+ * @property {DataLinkObject} data - data link object.
+ */
+
+/**
+ * @typedef {object} NotFoundObject
+ * @property {string} status - status.
+ * @property {string} message - message.
+ */
+// END types
+
+const ENVIRONMENT = process.env.NEXT_PUBLIC_FEATURE_ENV;
+
+export const DATA_API_URL =
+  ENVIRONMENT === 'staging' ? GFW_STAGING_DATA_API : GFW_DATA_API;
+
+/**
+ * @param {NextApiRequest} req
+ * @param {NextApiResponse} res
+ */
+const fetchDataByDatasetAndGeostore = async (req, res) => {
+  const { query } = req;
+  // TODO: add more parameters to the query like, global, adm9, adm1, etc etc etc
+  const { slug: slugs, geostore_id, canopy_cover } = query;
+
+  if (slugs.length === 0) {
+    res.status(400).send();
+    return;
+  }
+
+  if (slugs.length === 1) {
+    const dataByGeostore = await getDataByGeostoreId({
+      dataset: slugs[0],
+      geostoreId: geostore_id,
+      canopy: canopy_cover,
+    });
+
+    res.status(200).send(dataByGeostore);
+    return;
+  }
+
+  const url = `${DATA_API_URL}/${slugs.join('/')}`;
+  try {
+    const dataByUrl = await getDataFromLink({ url });
+
+    res.send(dataByUrl);
+  } catch (error) {
+    res.status(error.response?.status).send({
+      status: error.response?.status,
+      message: error?.message,
+    });
+  }
+};
+
+/**
+ * @param {NextApiRequest} req
+ * @param {NextApiResponse} res
+ */
+const postData = async (req, res) => {
+  const { query } = req;
+  // TODO: add more parameters to the query like, global, adm9, adm1, etc etc etc
+  const { slug: slugs, geostore_id, canopy_cover } = query;
+
+  if (slugs.length === 0) {
+    res.status(400).send();
+    return;
+  }
+
+  try {
+    const submitted = await createRequestByGeostoryId({
+      dataset: slugs[0],
+      geostoreId: geostore_id,
+      canopy: canopy_cover,
+    });
+
+    res.status(201).send(submitted);
+  } catch (error) {
+    res.status(error.response?.status).send({
+      status: error.response?.status,
+      message: error?.message,
+    });
+  }
+};
+
+/**
+ * @param {NextApiRequest} req
+ * @param {NextApiResponse} res
+ */
+export default async (req, res) => {
+  switch (req.method) {
+    case 'POST':
+      postData(req, res);
+      break;
+    case 'GET':
+      fetchDataByDatasetAndGeostore(req, res);
+      break;
+    default:
+      res.send(405);
+  }
+};

services/datamart.js

@@ -0,0 +1,170 @@
+import qs from 'qs';
+import { dataRequest } from 'utils/request';
+import { GFW_DATA_API, GFW_STAGING_DATA_API } from 'utils/apis';
+
+const ENVIRONMENT = process.env.NEXT_PUBLIC_FEATURE_ENV;
+const DATA_API_URL =
+  ENVIRONMENT === 'staging' ? GFW_STAGING_DATA_API : GFW_DATA_API;
+
+/**
+ * @typedef {object} DataLinkObject
+ * @property {string} link - The URL to POST the content.
+ */
+
+/**
+ * @typedef {object} GetResponseObject
+ * @property {string} status - status.
+ * @property {DataLinkObject} data - data link object.
+ */
+
+/**
+ * @typedef {object} NotFoundObject
+ * @property {string} status - status.
+ * @property {string} message - message.
+ */
+
+/**
+ * 1
+ * @param {Object} request - request
+ * @param {string} request.dataset - dataset.
+ * @param {string} request.geostoreId - a geostore id.
+ * @param {number} request.canopy - canopy filter.
+ * @returns {Promise<GetResponseObject | NotFoundObject>} response.
+ */
+const getDataByGeostoreId = async ({ dataset, geostoreId, canopy }) => {
+  const url = `/v0/land/${dataset}`;
+  const params = {
+    geostore_id: geostoreId,
+    canopy_cover: canopy,
+  };
+
+  const requestUrl = `${url}/?${qs.stringify(params)}`;
+
+  let response;
+
+  try {
+    response = await dataRequest.get(requestUrl);
+  } catch (error) {
+    if (error.response?.status === 404) {
+      return new Promise((resolve) => {
+        // eslint-disable-next-line prefer-promise-reject-errors
+        resolve({
+          status: error.response?.status,
+          message: error.response?.statusText,
+        });
+      });
+    }
+  }
+
+  return response.data;
+};
+
+/**
+ * 2
+ * @param {Object} request - request
+ * @param {string} request.dataset - dataset.
+ * @param {string} request.geostoreId - a geostore id.
+ * @param {number} request.canopy - canopy filter.
+ * @returns {Promise<GetResponseObject>} response.
+ */
+const createRequestByGeostoryId = async ({
+  dataset,
+  geostoreId,
+  canopy,
+}) => {
+  const url = `/v0/land/${dataset}`;
+  const params = {
+    geostore_id: geostoreId,
+    canopy_cover: canopy,
+  };
+
+  const response = await dataRequest.post(url, params);
+
+  return response;
+};
+
+/**
+ * 3
+ * @param {Object} request - request
+ * @param {string} request.url - url
+ * @returns {Promise<GetResponseObject>} response.
+ */
+const getDataFromLink = async ({ url }) => {
+    return dataRequest.get(url.replace(DATA_API_URL, ''));
+};
+
+
+const wait = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+
+const retryRequest = async (fn, params, retries = 3, interval = 1000, finalErr = 'Retry failed') => {
+  try {
+    console.log(`retryRequest retries ${retries} with fn ${fn}`);
+    const res = await fn(params);
+
+    if (res.data?.status === 'pending') {
+      console.log(`is pending, waiting ${interval} sec and retrying`);
+      await wait(interval);
+      return retryRequest(fn, params, retries - 1, interval, finalErr);
+    }
+
+    return res;
+
+  } catch (err) {
+    console.log('caught err: ', err);
+    if (retries <= 0) {
+      console.log('no more retries, rejecting');
+      return Promise.reject(finalErr);
+    }
+    await wait(interval);
+    return retryRequest(fn, params, retries - 1, interval, finalErr);
+  }
+};
+
+/**
+ * 
+ * @param {Object} request
+ * @param {string} request.dataset - dataset
+ * @param {string} request.geostoreId - geostore id
+ * @param {boolean} request.isGlobal - whether the query is global or not
+ * @param {string} request.adm0 - adm0
+ * @param {string} request.adm1 - adm1
+ * @param {string} request.adm2 - adm2
+ * @param {boolean} request.isAnalyis - is analysis
+ * @param {number} request.threshold - canopy threshold
+ * @param {boolean} request.isDownload - whether the query is is download
+ * @param {number} request.retries - this parameter is to manage retries (only for recursion)
+ * @param 
+ */
+export const fetchDataMart = async ({
+  dataset,
+  geostoreId,
+  isGlobal,
+  adm0,
+  adm1,
+  adm2,
+  isAnalyis,
+  threshold,
+  isDownload,
+  retries,
+}) => {
+  const response = await getDataByGeostoreId({ dataset, geostoreId, canopy: threshold });
+
+  if (response.status !== 404) {
+    console.log('link exists, need to fetch: ', response.link);
+    const existing = await retryRequest(getDataFromLink, { url: response.link });
+    console.log('existing: ', existing);
+  } else {
+    // make post to create the data in back end
+    console.log('make post to create the data in back end');
+    const submitted = await createRequestByGeostoryId({ dataset, geostoreId, canopy: threshold });
+    console.log('> submitted: ', submitted);
+
+    // get link and fetch
+
+    // retry based on secondTry.headers['retry-after]
+    const secondTry = await retryRequest(getDataFromLink, { url: submitted.data.link });
+
+    console.log('secondTry: ', secondTry);
+  }
+
+};

utils/request.js

@@ -19,7 +19,7 @@ const ENVIRONMENT = process.env.NEXT_PUBLIC_FEATURE_ENV;
 const GFW_API_URL = GFW_API;
 const GFW_METADATA_API_URL =
   ENVIRONMENT === 'staging' ? GFW_STAGING_METADATA_API : GFW_METADATA_API;
-const DATA_API_URL =
+export const DATA_API_URL =
   ENVIRONMENT === 'staging' ? GFW_STAGING_DATA_API : GFW_DATA_API;
 
 // We never use the `staging-api.resourcewatch.org`.