Fix: Implement pagination for article file list retrieval in Figshare Service

support/integration-testing/docker-compose.bruno.yml

@@ -27,7 +27,7 @@ services:
       --output /opt/redbox-portal/.tmp/junit/backend-bruno/backend-bruno.xml 
       --bail"
   redboxportal:
-    image: qcifengineering/redbox-portal:develop
+    image: qcifengineering/redbox-portal:master
     pull_policy: 'always'
     ports:
        - "1500:1500"
@@ -70,9 +70,8 @@ services:
     entrypoint: >-
       /bin/bash -c "cd /opt/redbox-portal &&
       ./support/integration-testing/prepare-guest.sh /opt/redbox-portal &&
-      npm install -g nyc &&
       npm install &&
-      nyc
+      ./node_modules/.bin/nyc
       --no-clean
       --report-dir coverage/bruno
       --reporter=lcov

support/integration-testing/docker-compose.mocha.yml

@@ -4,8 +4,9 @@ networks:
 
 services:
   redboxportal:
-    image: qcifengineering/redbox-portal:develop
+    image: qcifengineering/redbox-portal:master
     pull_policy: 'always'
+    user: "root"
     ports:
        - "1500:1500"
       # Debugging port
@@ -46,9 +47,8 @@ services:
     entrypoint: >-
       /bin/bash -c "cd /opt/redbox-portal &&
       ./support/integration-testing/prepare-guest.sh /opt/redbox-portal &&
-      npm install -g nyc &&
       npm install &&
-      nyc
+      ./node_modules/.bin/nyc
       --no-clean
       --report-dir coverage/mocha 
       --reporter=lcov

typescript/api/services/FigshareService.ts

@@ -695,15 +695,38 @@ export module Services {
     }
 
     private async getArticleFileList(articleId:string, logEnabled:boolean = true) {
-      let articleFileListConfig = this.getAxiosConfig('get', `/account/articles/${articleId}/files`, null);
-      if(logEnabled) {
-        sails.log[this.createUpdateFigshareArticleLogLevel](`FigService - getArticleFileList - ${articleFileListConfig.method} - ${articleFileListConfig.url}`);
+      const defaultPageSize = 20;
+      const pageSizeConfig = _.get(sails.config, 'figshareAPI.mapping.upload.fileListPageSize', defaultPageSize);
+      const pageSize = _.isNumber(pageSizeConfig) && pageSizeConfig > 0 ? pageSizeConfig : defaultPageSize;
+
+      let page = 1;
+      let articleFileList = [];
+      let hasMorePages = true;
+
+      while(hasMorePages) {
+        let articleFileListConfig = this.getAxiosConfig('get', `/account/articles/${articleId}/files?page_size=${pageSize}&page=${page}`, null);
+        if(logEnabled) {
+          sails.log[this.createUpdateFigshareArticleLogLevel](`FigService - getArticleFileList - page ${page} - ${articleFileListConfig.method} - ${articleFileListConfig.url}`);
+        }
+        let responseArticleList = await axios(articleFileListConfig);
+        if(logEnabled) {
+          sails.log[this.createUpdateFigshareArticleLogLevel](`FigService - getArticleFileList - page ${page} status: ${responseArticleList.status} statusText: ${responseArticleList.statusText}`);
+        }
+
+        let currentPage = responseArticleList.data;
+        if(_.isArray(currentPage) && currentPage.length > 0) {
+          articleFileList.push(...currentPage);
+          hasMorePages = currentPage.length >= pageSize;
+        } else {
+          hasMorePages = false;
+        }
+        page++;
       }
-      let responseArticleList = await axios(articleFileListConfig);
+
       if(logEnabled) {
-        sails.log[this.createUpdateFigshareArticleLogLevel](`FigService - getArticleFileList - status: ${responseArticleList.status} statusText: ${responseArticleList.statusText}`);
+        sails.log[this.createUpdateFigshareArticleLogLevel](`FigService - getArticleFileList - total files fetched ${articleFileList.length}`);
       }
-      let articleFileList = responseArticleList.data;
+
       return articleFileList;
     }
 
@@ -1401,11 +1424,7 @@ export module Services {
 
             //Try to upload files to article
             let that = this;
-            let articleFileListConfig = this.getAxiosConfig('get', `/account/articles/${articleId}/files`, null);
-            sails.log[this.createUpdateFigshareArticleLogLevel](`FigService - checkUploadFilesPending - ${articleFileListConfig.method} - ${articleFileListConfig.url}`);
-            let responseArticleList = await axios(articleFileListConfig);
-            sails.log[this.createUpdateFigshareArticleLogLevel](`FigService - checkUploadFilesPending - status: ${responseArticleList.status} statusText: ${responseArticleList.statusText}`);
-            let articleFileList = responseArticleList.data;
+            let articleFileList = await this.getArticleFileList(articleId);
             let filePath = sails.config.figshareAPI.attachmentsFigshareTempDir;
             sails.log[this.createUpdateFigshareArticleLogLevel]('FigService - checkUploadFilesPending - attachmentsFigshareTempDir '+filePath);
 
@@ -1440,9 +1459,8 @@ export module Services {
                   let attachId = attachmentFile.fileId;
                   let fileName = attachmentFile.name;
                   let fileSize = attachmentFile.size;
-                  //check if the file has been uploaded already or not to the figshare article 
-                  responseArticleList = await axios(articleFileListConfig);
-                  articleFileList = responseArticleList.data;
+                  //check if the file has been uploaded already or not to the figshare article
+                  articleFileList = await this.getArticleFileList(articleId, false);
                   sails.log[this.createUpdateFigshareArticleLogLevel]('FigService - checkUploadFilesPending - article file list: '+JSON.stringify(articleFileList));
                   let filePendingToBeUploaded = _.find(articleFileList, ['name', fileName]);
                   let fileFullPath = filePath + '/' +fileName;
@@ -1899,11 +1917,7 @@ export module Services {
 
         if(articleId > 0) {
 
-          let articleFileListConfig = this.getAxiosConfig('get', `/account/articles/${articleId}/files`, null);
-          sails.log[this.createUpdateFigshareArticleLogLevel](`FigService - deleteFilesAndUpdateDataLocationEntries - ${articleFileListConfig.method} - ${articleFileListConfig.url}`);
-          let responseArticleList = await axios(articleFileListConfig);
-          sails.log[this.createUpdateFigshareArticleLogLevel](`FigService - deleteFilesAndUpdateDataLocationEntries - status: ${responseArticleList.status} statusText: ${responseArticleList.statusText}`);
-          let articleFileList = responseArticleList.data;
+          let articleFileList = await this.getArticleFileList(articleId);
 
           let dataLocations = _.get(record,this.dataLocationsPathInRecord);
           let urlList = [];
@@ -2247,3 +2261,4 @@ export module Services {
   }
 }
 module.exports = new Services.FigshareService().exports();
+module.exports.Services = Services;

typescript/api/test/unit/services/FigshareService.test.ts

@@ -0,0 +1,160 @@
+const { describe, it, beforeEach, before } = require('mocha');
+
+// Minimal sails stub required by FigshareService
+(global as any).sails = {
+  config: {
+    figshareAPIEnv: {
+      overrideArtifacts: {
+        mapping: {
+          artifacts: {}
+        }
+      }
+    },
+    figshareAPI: {
+      APIToken: 'test-token',
+      baseURL: 'https://api.figshare.test.localhost',
+      frontEndURL: 'https://figshare.test.localhost',
+      mapping: {
+        upload: {
+          fileListPageSize: 2,
+          override: {}
+        },
+        figshareOnlyPublishSelectedAttachmentFiles: false,
+        recordAllFilesUploaded: false,
+        targetState: {},
+        response: {
+          article: {}
+        }
+      },
+      diskSpaceThreshold: 0
+    },
+    figshareReDBoxFORMapping: {
+      FORMapping: []
+    },
+    record: {
+      createUpdateFigshareArticleLogLevel: 'verbose'
+    },
+    queue: {
+      serviceName: ''
+    }
+  },
+  log: {
+    verbose: () => {},
+    info: () => {},
+    error: () => {},
+    warn: () => {},
+    debug: () => {}
+  },
+  services: {},
+  on: () => {}
+};
+
+// axios is required at module load in FigshareService, so we must set the stub before importing it
+const axiosCalls: any[] = [];
+let axiosResponses: any[] = [];
+const axiosStub = (config) => {
+  axiosCalls.push(config);
+  if (!axiosResponses.length) {
+    return Promise.reject(new Error('No axios mock response available'));
+  }
+  return Promise.resolve(axiosResponses.shift());
+};
+(require as any).cache[require.resolve('axios')] = { exports: axiosStub as any };
+
+const { Services } = require('../../../services/FigshareService');
+const FigshareService = Services.FigshareService;
+
+describe('FigshareService - getArticleFileList pagination', () => {
+  let service;
+  let expect;
+
+  before(async () => {
+    // @ts-ignore
+    const chai = await import('chai');
+    expect = chai.expect;
+  });
+
+  beforeEach(() => {
+    axiosCalls.length = 0;
+    axiosResponses = [];
+    service = new FigshareService();
+    // Set required properties that are private in TS but accessible at runtime
+    (service as any).baseURL = 'https://api.figshare.test.localhost';
+    (service as any).APIToken = 'test-token';
+    (global as any).sails.config.figshareAPI.mapping.upload.fileListPageSize = 2;
+  });
+
+  it('aggregates multiple pages until a partial page is returned', async () => {
+    axiosResponses = [
+      { status: 200, statusText: 'OK', data: [{ id: 1 }, { id: 2 }] },
+      { status: 200, statusText: 'OK', data: [{ id: 3 }] }
+    ];
+
+    const files = await (service as any).getArticleFileList('123');
+
+    expect(files.map((f) => f.id)).to.deep.equal([1, 2, 3]);
+    expect(axiosCalls).to.have.length(2);
+    expect(axiosCalls[0].url).to.contain('/account/articles/123/files?page_size=2&page=1');
+    expect(axiosCalls[1].url).to.contain('/account/articles/123/files?page_size=2&page=2');
+  });
+
+  it('falls back to the default page size when the config value is invalid', async () => {
+    (global as any).sails.config.figshareAPI.mapping.upload.fileListPageSize = 'invalid';
+    const expectedDefault = 20;
+
+    axiosResponses = [
+      { status: 200, statusText: 'OK', data: [{ id: 'a' }] }
+    ];
+
+    const files = await (service as any).getArticleFileList('abc');
+
+    expect(files.map((f) => f.id)).to.deep.equal(['a']);
+    expect(axiosCalls).to.have.length(1);
+    expect(axiosCalls[0].url).to.contain(`/account/articles/abc/files?page_size=${expectedDefault}&page=1`);
+  });
+
+  it('returns an empty list when no files are found', async () => {
+    axiosResponses = [
+      { status: 200, statusText: 'OK', data: [] }
+    ];
+
+    const files = await (service as any).getArticleFileList('empty');
+
+    expect(files).to.deep.equal([]);
+    expect(axiosCalls).to.have.length(1);
+    expect(axiosCalls[0].url).to.contain('/account/articles/empty/files?page_size=2&page=1');
+  });
+});
+
+describe('FigshareService - isFileUploadInProgress', () => {
+  let service;
+  let expect;
+
+  before(async () => {
+    // @ts-ignore
+    const chai = await import('chai');
+    expect = chai.expect;
+  });
+
+  beforeEach(() => {
+    service = new FigshareService();
+    (service as any).baseURL = 'https://api.figshare.test.localhost';
+    (service as any).APIToken = 'test-token';
+  });
+
+  it('returns true when any file has status "created"', async () => {
+    const mockFileList = [{ id: 1, status: 'available' }, { id: 2, status: 'created' }];
+
+    const inProgress = await (service as any).isFileUploadInProgress('article-1', mockFileList);
+
+    expect(inProgress).to.equal(true);
+  });
+
+  it('returns false when no files are in progress', async () => {
+    const mockFileList = [{ id: 1, status: 'available' }];
+
+    const inProgress = await (service as any).isFileUploadInProgress('article-2', mockFileList);
+
+    expect(inProgress).to.equal(false);
+  });
+});