Merge pull request #3 from sonnenkern/develop

New option --resolve-references
sonnenkern · Feb 18, 2020 · 9036020 · 9036020
2 parents 175333f + 5fabd53
commit 9036020
Show file tree

Hide file tree

Showing 9 changed files with 295 additions and 42 deletions.
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -1,7 +1,7 @@
 {
   "extends": ["eslint:recommended"],
   "parserOptions": {
-    "ecmaVersion": 2017
+    "ecmaVersion": 9
   },
   "rules": {
     "require-atomic-updates": "warn"

diff --git a/README.md b/README.md
@@ -70,6 +70,7 @@ npm install quip-export
   -z, --zip                  Zip export files
   --embedded-styles          Embedded in each document stylesheet
   --embedded-images          Embedded images
+  --resolve-references       Resolves references to other Quip documents and folders to a proper relative path
   --debug                    Extended logging
 ```
 

diff --git a/__tests__/app.test.js b/__tests__/app.test.js
@@ -50,7 +50,8 @@ function initApp() {
         destination: 'c:/temp',
         token: 'TOKEN',
         ['embedded-styles']: true,
-        ['embedded-images']: true
+        ['embedded-images']: true,
+        ['resolve-references']: true
     });
     QuipService.mockImplementation(() => {
         return {
@@ -163,7 +164,8 @@ describe('main() tests', () => {
             {
                 documentTemplate,
                 documentCSS: documentCSS,
-                embeddedImages: true
+                embeddedImages: true,
+                resolveReferences: true
             }
         );
         expect(app.quipProcessor.setLogger).toHaveBeenCalledWith(app.Logger);

diff --git a/app.js b/app.js
@@ -160,7 +160,8 @@ class App {
             {
                 documentTemplate,
                 documentCSS: this.cliArguments['embedded-styles']? documentCSS : '',
-                embeddedImages: this.cliArguments['embedded-images']
+                embeddedImages: this.cliArguments['embedded-images'],
+                resolveReferences: this.cliArguments['resolve-references']
             });
         this.quipProcessor.setLogger(this.Logger);
 

diff --git a/lib/QuipProcessor.js b/lib/QuipProcessor.js
@@ -22,6 +22,8 @@ class QuipProcessor {
         this.threadsTotal = 0;
         this.foldersTotal = 0;
 
+        this.referencesMap = new Map();
+
         this.phase = 'STOP'; //START, STOP, ANALYSIS, EXPORT
 
         this.quipService = new QuipService(quipToken, options.quipApiURL);
@@ -45,21 +47,21 @@ class QuipProcessor {
         this.start = true;
         this.threadsProcessed = 0;
 
-        const user = await this.quipService.getUser();
-        if(!user) {
+        this.quipUser = await this.quipService.getUser();
+        if(!this.quipUser) {
             this.logger.error("Can't load the User");
             this.stopExport();
             return;
         }
 
         let folderIdsToExport = [
-            //user.desktop_folder_id,
-            //user.archive_folder_id,
-            //user.starred_folder_id,
-            user.private_folder_id,
-            //user.trash_folder_id
-            ...user.shared_folder_ids,
-            ...user.group_folder_ids
+            //this.quipUser.desktop_folder_id,
+            //this.quipUser.archive_folder_id,
+            //this.quipUser.starred_folder_id,
+            this.quipUser.private_folder_id,
+            //this.quipUser.trash_folder_id
+            ...this.quipUser.shared_folder_ids,
+            ...this.quipUser.group_folder_ids
         ];
 
         if(folderIds && folderIds.length > 0) {
@@ -81,34 +83,84 @@ class QuipProcessor {
         this.phase = phase;
     }
 
-    _getMatches(text, regexp/*, threadId*/) {
+    _getMatches(text, regexp) {
         const matches = [];
 
         let regexpResult = regexp.exec(text);
 
         while (regexpResult != null) {
-            //if(regexpResult[2] === threadId) {
-                matches.push({
-                    replacement: regexpResult[1],
-                    threadId: regexpResult[2],
-                    blobId: regexpResult[3],
-                    fileName: regexpResult[4]
-                });
-            //}
+            matches.push({
+                replacement: regexpResult[1],
+                threadId: regexpResult[2],
+                blobId: regexpResult[3],
+                fileName: regexpResult[4]
+            });
             regexpResult = regexp.exec(text);
         }
 
         return matches;
     }
 
+    async _resolveReferences(html, pathDeepness) {
+        //look up for document or folder references
+        const regexp = new RegExp(`href=\\"(${this.quipUser.url}/([\\w-]+))\\"`, 'gim');
+        const matchesReference = this._getMatches(html, regexp);
+
+        //replace references to documents
+        if(this.options.resolveReferences) {
+            for(const reference of matchesReference) {
+                html = await this._processReference(html, reference, pathDeepness);
+            }
+        }
+
+        return html;
+    }
+
+    async _processReference(html, reference, pathDeepness) {
+        let referencedObject = this.referencesMap.get(reference.threadId);
+
+        let path, folder, title;
+        if(referencedObject) {
+            path    = referencedObject.path;
+            folder  = referencedObject.folder;
+            title   = referencedObject.title;
+        }
+
+        if(!path) {
+            //correct path for threads
+            const referencedThread = await this.quipService.getThread(reference.threadId);
+            if(!referencedThread) {
+                return html;
+            }
+            referencedObject = this.referencesMap.get(referencedThread.thread.id);
+            if(referencedObject) {
+                path = referencedObject.path;
+                folder = false;
+                title = referencedThread.thread.title;
+            }
+        }
+
+        if(!path || !title) {
+            return html;
+        }
+
+        if(folder) {
+            path = '../'.repeat(pathDeepness) + path + title;
+        } else {
+            path = '../'.repeat(pathDeepness) + path + sanitizeFilename(title.trim()) + '.html';
+        }
+
+        return html.replace(reference.replacement, path);
+    }
+
     async _processDocumentThread(quipThread, path) {
         //look up for images in html
         let regexp = new RegExp("src='(/blob/([\\w-]+)/([\\w-]+))'", 'gim');
-        const matchesImg = this._getMatches(quipThread.html, regexp, quipThread.thread.id);
+        const matchesImg = this._getMatches(quipThread.html, regexp);
 
         //look up for links in html
         regexp = new RegExp('href=\\"(.*/blob/(.+)/(.+)\\?name=(.+))\\"', 'gim');
-        const matchesLink = this._getMatches(quipThread.html, regexp, quipThread.thread.id);
+        const matchesLink = this._getMatches(quipThread.html, regexp);
 
         const pathDeepness = path.split("/").length-1;
         let wrappedHtml = quipThread.html;
@@ -126,7 +178,7 @@ class QuipProcessor {
 
         if(this.documentTemplate) {
             //wrap html code
-            wrappedHtml = ejs.render(this.documentTemplate,documentRenderOptions);
+            wrappedHtml = ejs.render(this.documentTemplate, documentRenderOptions);
         }
 
         //replace blob references for links
@@ -139,6 +191,11 @@ class QuipProcessor {
             wrappedHtml = await this._processFile(wrappedHtml, image, path, this.options.embeddedImages);
         }
 
+        //replace references to documents
+        if(this.options.resolveReferences) {
+            wrappedHtml = await this._resolveReferences(wrappedHtml, pathDeepness);
+        }
+
         this.saveCallback(wrappedHtml, sanitizeFilename(`${quipThread.thread.title.trim()}.html`), 'THREAD', path);
     }
 
@@ -273,19 +330,33 @@ class QuipProcessor {
         });
     }
 
-    async _countThreadsAndFolders(quipFolder) {
+    async _countThreadsAndFolders(quipFolder, path) {
         const threadIds = [];
         const folderIds = [];
 
+        if(this.options.resolveReferences) {
+            this.referencesMap.set(quipFolder.folder.id, {
+                path,
+                folder: true,
+                title: quipFolder.folder.title
+            });
+        }
+
         if(!quipFolder.children || quipFolder.children.length === 0) {
             return;
         }
 
+        const pathForChildren = `${path}${quipFolder.folder.title}/`;
+
         for(const index in quipFolder.children) {
             const quipChild = quipFolder.children[index];
-
             if(quipChild.thread_id) { //thread
                 threadIds.push(quipChild.thread_id);
+                if(this.options.resolveReferences) {
+                    this.referencesMap.set(quipChild.thread_id, {
+                        path: pathForChildren
+                    });
+                }
             } else if(quipChild.folder_id) { //folder
                 folderIds.push(quipChild.folder_id);
             }
@@ -308,7 +379,7 @@ class QuipProcessor {
         }
 
         for(const index in childFolders) {
-            await this._countThreadsAndFolders(childFolders[index]);
+            await this._countThreadsAndFolders(childFolders[index], pathForChildren);
         }
     }
 
@@ -326,7 +397,7 @@ class QuipProcessor {
         }
 
         for(const index in quipFolders) {
-            await this._countThreadsAndFolders(quipFolders[index]);
+            await this._countThreadsAndFolders(quipFolders[index], "");
         }
 
         this._changePhase('EXPORT');