Add test for reference API

.github/ci.env

@@ -10,13 +10,19 @@ TYPEORM_SYNCHRONIZE=true
 TYPEORM_LOGGING=false
 TYPEORM_ENTITIES=src/entity/*.ts
 TYPEORM_MIGRATIONS=src/migration/*.ts
-DP_PID_SERVICE_URL=http://localhost:5801/pid
 DP_SS_URL=http://localhost:5920
 DP_SS_USER=test
 DP_SS_PASSWORD=test
-DP_PID_SERVICE_TIMEOUT_MS=1000
 DP_BACKEND_URL=http://backend:3000/api
 DP_FRONTEND_URL=http://frontend-prod:8080
 DP_SS_TEST_URL=http://backend:5920
 GEOLITE2_COUNTRY_PATH=tests/data/GeoLite2-Country-Test.mmdb
 CITATION_SERVICE_URL=http://localhost:1234
+DATACITE_API_URL=http://localhost:5802
+DATACITE_API_USERNAME=XXX
+DATACITE_API_PASSWORD=XXX
+DATACITE_API_TIMEOUT_MS=2000
+DATACITE_DOI_SERVER=http://handle.datacite.test
+DATACITE_DOI_PREFIX=XXX
+LABELLING_URL=http://localhost:5803
+HANDLE_API_URL=http://localhost:5804

.pre-commit-config.yaml

@@ -5,6 +5,7 @@ repos:
     rev: v4.3.0
     hooks:
       - id: trailing-whitespace
+        exclude: __snapshots__
       - id: end-of-file-fixer
       - id: check-yaml
       - id: check-added-large-files

backend/dev.env.template

@@ -25,3 +25,5 @@ DATACITE_API_PASSWORD=XXX
 DATACITE_API_TIMEOUT_MS=2000
 DATACITE_DOI_SERVER=https://handle.stage.datacite.org
 DATACITE_DOI_PREFIX=XXX
+LABELLING_URL=https://actris-nf-labelling.out.ocp.fmi.fi
+HANDLE_API_URL=https://hdl.handle.net/api/

backend/fixtures/1-site.json

@@ -8,6 +8,7 @@
     "altitude": 93,
     "gaw": null,
     "dvasId": "INO",
+    "actrisId": 1337,
     "country": "Romania",
     "countryCode": "RO",
     "countrySubdivisionCode": null,

backend/fixtures/2-model_file.json

@@ -88,5 +88,23 @@
     "site": "bucharest",
     "version": "123",
     "errorLevel": "pass"
+  },
+  {
+    "uuid": "a45a2e9a-e39d-4af2-9798-5ea0fadf041e",
+    "pid": "www.pid.com",
+    "volatile": false,
+    "measurementDate": "2019-07-16",
+    "product": "model",
+    "model": "ecmwf",
+    "cloudnetpyVersion": "1.0.4",
+    "createdAt": "2020-02-20T10:56:19.382Z",
+    "updatedAt": "2020-02-20T10:56:19.382Z",
+    "s3key": "20190716_bucharest_ecmwf.nc",
+    "checksum": "e7712f5a1a01fac7e0fa23d47f1879d5107de0c7e9c4b530d5ec98cf4091ddfd",
+    "size": 12200657,
+    "format": "HDF5 (NetCDF4)",
+    "site": "bucharest",
+    "version": "123",
+    "errorLevel": "pass"
   }
 ]

backend/fixtures/2-person.json

@@ -1 +1,7 @@
-[]
+[
+  {
+    "firstname": "Benjamin",
+    "surname": "Bucharinen",
+    "citations": [{ "id": "bucharest_test" }]
+  }
+]

backend/fixtures/2-regular_file.json

@@ -115,6 +115,11 @@
     "size": 7127282,
     "format": "HDF5 (NetCDF4)",
     "site": "bucharest",
+    "sourceFileIds": [
+      "b6de8cf4-8825-47b0-aaa9-4fd413bbb0d7",
+      "f036da43-c19c-4832-99f9-6cc88f3255c5",
+      "a45a2e9a-e39d-4af2-9798-5ea0fadf041e"
+    ],
     "version": "123",
     "errorLevel": null
   },
@@ -244,5 +249,41 @@
     "version": "123",
     "quality": "qc",
     "errorLevel": "error"
+  },
+  {
+    "uuid": "b6de8cf4-8825-47b0-aaa9-4fd413bbb0d7",
+    "pid": "",
+    "volatile": false,
+    "measurementDate": "2019-07-16",
+    "product": "lidar",
+    "cloudnetpyVersion": "1.0.4",
+    "createdAt": "2020-02-19T10:59:58.449Z",
+    "updatedAt": "2020-02-19T10:59:58.449Z",
+    "s3key": "20190715_bucharest_lidar.nc",
+    "checksum": "f5e059df0c0dccecc5a5d07d4306e17b9b2b4272ea176688e0ee5c1f651010df",
+    "size": 7127282,
+    "format": "HDF5 (NetCDF4)",
+    "site": "bucharest",
+    "version": "123",
+    "errorLevel": null,
+    "instrumentPid": "https://hdl.handle.net/123/bucharest_lidar"
+  },
+  {
+    "uuid": "f036da43-c19c-4832-99f9-6cc88f3255c5",
+    "pid": "",
+    "volatile": false,
+    "measurementDate": "2019-07-16",
+    "product": "radar",
+    "cloudnetpyVersion": "1.0.4",
+    "createdAt": "2020-02-19T10:59:58.449Z",
+    "updatedAt": "2020-02-19T10:59:58.449Z",
+    "s3key": "20190715_bucharest_lidar.nc",
+    "checksum": "e09b61366e9c1a4b2676d07dc9bfccff1436a73e46f1a3a8ea51429218d5cb50",
+    "size": 7127282,
+    "format": "HDF5 (NetCDF4)",
+    "site": "bucharest",
+    "version": "123",
+    "errorLevel": null,
+    "instrumentPid": "https://hdl.handle.net/123/bucharest_radar"
   }
 ]

backend/fixtures/3-collection.json

@@ -40,5 +40,12 @@
     ],
     "createdAt": "2020-09-28T12:45:21.916Z",
     "updatedAt": "2020-09-28T12:45:21.916Z"
+  },
+  {
+    "uuid": "7a6c6675-af65-4650-a20c-e061344393d1",
+    "regularFiles": [{ "uuid": "2bb32746-faf0-4057-9076-ed2e698dcf36" }],
+    "modelFiles": [],
+    "createdAt": "2020-09-28T12:45:21.916Z",
+    "updatedAt": "2020-09-28T12:45:21.916Z"
   }
 ]

backend/src/lib/cite.ts

@@ -1,13 +1,11 @@
-import { Connection, Repository } from "typeorm";
+import { Connection } from "typeorm";
 
 import { RegularFile, ModelFile, File } from "../entity/File";
 import { Collection } from "../entity/Collection";
 import axios from "axios";
-import { getCollectionLandingPage, getFileLandingPage } from ".";
+import { formatList, getCollectionLandingPage, getFileLandingPage, truncateList } from ".";
 import env from "../lib/env";
-import { Model } from "../entity/Model";
 
-const LABELLING_URL = "https://actris-nf-labelling.out.ocp.fmi.fi/api/facilities";
 const MODEL_AUTHOR: Person = { firstName: "Ewan", lastName: "O'Connor", orcid: "0000-0001-9834-5100", role: "modelPi" };
 const PUBLISHER = "ACTRIS Cloud remote sensing data centre unit (CLU)";
 const MONTHS_FULL = [
@@ -86,9 +84,10 @@ export class CitationService {
         productNames.map((x) => x.toLowerCase()),
         5,
         "products"
-      )
+      ),
+      ", and "
     );
-    const sites = formatList(truncateList(siteNames, 5, "sites"));
+    const sites = formatList(truncateList(siteNames, 5, "sites"), ", and ");
     const date = formatDateRange(dateRange.startDate, dateRange.endDate);
     const title = `Custom collection of ${products} data from ${sites} ${date}`;
     return {
@@ -103,9 +102,25 @@ export class CitationService {
   async getFileCitation(file: RegularFile | ModelFile): Promise<Citation> {
     let people: Person[];
     if (file instanceof RegularFile) {
-      const instrumentPids = await this.queryInstrumentPids(file);
-      people = await fetchInstrumentPis(instrumentPids);
-      if (await this.usesModelData(file)) {
+      const [instrumentPis, nfPi, usesModelData] = await Promise.all([
+        this.queryInstrumentPids(file).then((pids) => fetchInstrumentPis(pids)),
+        file.site.actrisId
+          ? fetchNfPi(file.site.actrisId, file.measurementDate as unknown as string)
+          : Promise.resolve([]),
+        this.usesModelData(file),
+      ]);
+      people = [
+        ...instrumentPis,
+        ...nfPi.map(
+          (pi): Person => ({
+            firstName: pi.first_name,
+            lastName: pi.last_name,
+            orcid: pi.orcid_id ? normalizeOrcid(pi.orcid_id) : null,
+            role: "nfPi",
+          })
+        ),
+      ];
+      if (usesModelData) {
         people.push(MODEL_AUTHOR);
       }
       people = people.concat(
@@ -142,7 +157,10 @@ export class CitationService {
     }
     if (modelAcks.length > 0) {
       output += " We acknowledge ";
-      output += formatList(modelAcks.map((a) => a.replace(/\.$/, "")));
+      output += formatList(
+        modelAcks.map((a) => a.replace(/\.$/, "")),
+        ", and "
+      );
       output += ".";
     }
     return output;
@@ -325,7 +343,7 @@ async function fetchInstrumentPi(pid: string, measurementDate?: Date): Promise<I
   if (!match) {
     throw new Error("Invalid PID format");
   }
-  const url = "https://hdl.handle.net/api/handles/" + match[1];
+  const url = `${env.HANDLE_API_URL}/handles/${match[1]}`;
   const response = await axios.get(url);
   const values = response.data.values;
   if (!Array.isArray(values)) {
@@ -374,7 +392,7 @@ async function fetchInstrumentPis(data: { instrumentPid: string; dates: string[]
 }
 
 async function fetchNfPi(actrisId: number, measurementDate?: string): Promise<NfContact[]> {
-  const response = await axios.get(`${LABELLING_URL}/${actrisId}/contacts`, {
+  const response = await axios.get(`${env.LABELLING_URL}/api/facilities/${actrisId}/contacts`, {
     params: {
       date: measurementDate,
       role: "pi",
@@ -442,20 +460,6 @@ function removeDuplicateNames(pis: Person[]): Person[] {
   return out;
 }
 
-function truncateList(list: string[], limit: number, placeholder: string) {
-  if (list.length <= limit) {
-    return list;
-  }
-  return [...list.slice(0, limit), `${list.length - limit} other ${placeholder}`];
-}
-
-function formatList(parts: string[]): string {
-  if (parts.length <= 2) {
-    return parts.join(", and ");
-  }
-  return parts.slice(0, -1).join(", ") + ", and " + parts[parts.length - 1];
-}
-
 function formatDateRange(startDate: Date, endDate: Date): string {
   const sameDate = startDate.getDate() == endDate.getDate();
   const sameMonth = startDate.getMonth() == endDate.getMonth();

backend/src/lib/env.ts

@@ -15,6 +15,8 @@ const requiredVars = [
   "DATACITE_API_TIMEOUT_MS",
   "DATACITE_DOI_SERVER",
   "DATACITE_DOI_PREFIX",
+  "LABELLING_URL",
+  "HANDLE_API_URL",
 ] as const;
 
 type RequiredVar = typeof requiredVars[number];
@@ -44,6 +46,8 @@ interface Env {
   DATACITE_API_TIMEOUT_MS: number;
   DATACITE_DOI_SERVER: string;
   DATACITE_DOI_PREFIX: string;
+  LABELLING_URL: string;
+  HANDLE_API_URL: string;
 }
 
 const env: Env = {
@@ -53,6 +57,8 @@ const env: Env = {
   DATACITE_API_URL: rawEnv.DATACITE_API_URL.replace(/\/$/, ""),
   DATACITE_API_TIMEOUT_MS: parseInt(rawEnv.DATACITE_API_TIMEOUT_MS),
   DATACITE_DOI_SERVER: rawEnv.DATACITE_DOI_SERVER.replace(/\/$/, ""),
+  LABELLING_URL: rawEnv.LABELLING_URL.replace(/\/$/, ""),
+  HANDLE_API_URL: rawEnv.HANDLE_API_URL.replace(/\/$/, ""),
 };
 
 export default env;

backend/src/lib/index.ts

@@ -213,3 +213,17 @@ export function getCollectionLandingPage(collection: Collection): string {
 export function getObjectLandingPage(object: RegularFile | ModelFile | Collection): string {
   return object instanceof Collection ? getCollectionLandingPage(object) : getFileLandingPage(object);
 }
+
+export function truncateList(list: string[], limit: number, placeholder: string) {
+  if (list.length < limit + 2) {
+    return list;
+  }
+  return [...list.slice(0, limit), `${list.length - limit} other ${placeholder}`];
+}
+
+export function formatList(parts: string[], conjunction: string): string {
+  if (parts.length <= 2) {
+    return parts.join(conjunction);
+  }
+  return parts.slice(0, -1).join(", ") + conjunction + parts[parts.length - 1];
+}

backend/src/routes/reference.ts

@@ -3,7 +3,7 @@ import { Request, RequestHandler, Response } from "express";
 
 import { RegularFile, ModelFile } from "../entity/File";
 import { Collection } from "../entity/Collection";
-import { getObjectLandingPage } from "../lib";
+import { formatList, getObjectLandingPage } from "../lib";
 import { Citation, CitationService } from "../lib/cite";
 
 export class ReferenceRoutes {
@@ -42,7 +42,6 @@ export class ReferenceRoutes {
         await this.getCitation(req, res, object);
       }
     } catch (err) {
-      console.error(err);
       next(err);
     }
   };
@@ -102,16 +101,19 @@ function getInitials(name: string): string {
   return name.replace(/\p{L}+/gu, (part) => part.slice(0, 1) + ".");
 }
 
-function formatList(parts: string[]): string {
-  if (parts.length <= 2) {
-    return parts.join(" & ");
-  }
-  return parts.slice(0, -1).join(", ") + ", & " + parts[parts.length - 1];
-}
-
 function citation2html(c: Citation) {
-  const authors = formatList(c.authors.map((a) => `${a.lastName}, ${getInitials(a.firstName)}`));
-  return `${authors} (${c.year}). ${c.title}. ${c.publisher}. <a href="${c.url}">${c.url}</a>`;
+  const authors = formatList(
+    c.authors.map((a) => `${a.lastName}, ${getInitials(a.firstName)}`),
+    ", & "
+  );
+  const year = `(${c.year})`;
+  const title = c.title;
+  const publisher = c.publisher;
+  const link = `<a href="${c.url}">${c.url}</a>`;
+  if (!authors) {
+    return `${title}. ${year}. ${publisher}. ${link}`;
+  }
+  return `${authors} ${year}. ${title}. ${publisher}. ${link}`;
 }
 
 function citation2ris(c: Citation) {

backend/test.env

@@ -9,13 +9,19 @@ TYPEORM_SYNCHRONIZE=true
 TYPEORM_LOGGING=false
 TYPEORM_ENTITIES=src/entity/*.ts
 TYPEORM_MIGRATIONS=src/migration/*.ts
-DP_PID_SERVICE_URL=http://localhost:5801/pid
 DP_SS_URL=http://localhost:5920
 DP_SS_USER=test
 DP_SS_PASSWORD=test
-DP_PID_SERVICE_TIMEOUT_MS=200
 DP_BACKEND_URL=http://localhost:3000/api
 DP_FRONTEND_URL=http://localhost:8080
 DP_SS_TEST_URL=http://dataportal-backend-test:5920
 GEOLITE2_COUNTRY_PATH=tests/data/GeoLite2-Country-Test.mmdb
 CITATION_SERVICE_URL=http://citation-service
+DATACITE_API_URL=http://localhost:5802
+DATACITE_API_USERNAME=XXX
+DATACITE_API_PASSWORD=XXX
+DATACITE_API_TIMEOUT_MS=2000
+DATACITE_DOI_SERVER=http://handle.datacite.test
+DATACITE_DOI_PREFIX=XXX
+LABELLING_URL=http://localhost:5803
+HANDLE_API_URL=http://localhost:5804