feat: created a globbing file tree walker utility

src/vaults/types.ts

@@ -134,6 +134,58 @@ type VaultName = string;
 
 type VaultActions = Partial<Record<VaultAction, null>>;
 
+type FileTree = Array<TreeNode>;
+type TreeNode = DirectoryNode | FileNode | ContentNode;
+type FilePath = string;
+type INode = number;
+type CNode = number;
+
+type StatEncoded = {
+  isSymbolicLink: boolean;
+  type: 'FILE' | 'DIRECTORY' | 'OTHER';
+  dev: number;
+  ino: number;
+  mode: number;
+  nlink: number;
+  uid: number;
+  gid: number;
+  rdev: number;
+  size: number;
+  blksize: number;
+  blocks: number;
+  atime: number;
+  mtime: number;
+  ctime: number;
+  birthtime: number;
+};
+
+type DirectoryNode = {
+  type: 'directory';
+  path: FilePath;
+  iNode: INode;
+  parent: INode;
+  children: Array<INode>;
+  stat?: StatEncoded;
+};
+
+type FileNode = {
+  type: 'file';
+  path: FilePath;
+  iNode: INode;
+  parent: INode;
+  cNode: CNode;
+  stat?: StatEncoded;
+};
+
+// Keeping this separate from `FileNode` so we can optionally not include it.
+type ContentNode = {
+  type: 'content';
+  path: undefined;
+  fileName: string;
+  cNode: CNode;
+  contents: string;
+};
+
 export { vaultActions };
 
 export type {
@@ -148,6 +200,15 @@ export type {
   FileSystemWritable,
   VaultName,
   VaultActions,
+  FileTree,
+  TreeNode,
+  FilePath,
+  INode,
+  CNode,
+  StatEncoded,
+  DirectoryNode,
+  FileNode,
+  ContentNode,
 };
 
 export { tagLast, refs };

src/vaults/utils.ts

@@ -1,14 +1,20 @@
-import type { EncryptedFS } from 'encryptedfs';
+import type { EncryptedFS, Stat } from 'encryptedfs';
+import type { FileSystem } from '../types';
 import type {
   VaultRef,
   VaultAction,
   CommitId,
   FileSystemReadable,
   FileSystemWritable,
+  TreeNode,
+  DirectoryNode,
+  INode,
+  StatEncoded,
 } from './types';
 import type { NodeId } from '../ids/types';
 import type { Path } from 'encryptedfs/dist/types';
 import path from 'path';
+import { minimatch } from 'minimatch';
 import { pathJoin } from 'encryptedfs/dist/utils';
 import * as vaultsErrors from './errors';
 import { tagLast, refs, vaultActions } from './types';
@@ -123,6 +129,165 @@ async function mkdirExists(efs: FileSystemWritable, directory: string) {
   }
 }
 
+function genStat(stat: Stat): StatEncoded {
+  return {
+    isSymbolicLink: stat.isSymbolicLink(),
+    type: stat.isFile() ? 'FILE' : stat.isDirectory() ? 'DIRECTORY' : 'OTHER',
+    dev: stat.dev,
+    ino: stat.ino,
+    mode: stat.mode,
+    nlink: stat.nlink,
+    uid: stat.uid,
+    gid: stat.gid,
+    rdev: stat.rdev,
+    size: stat.size,
+    blksize: stat.blksize,
+    blocks: stat.blocks,
+    atime: stat.atime.getTime(),
+    mtime: stat.mtime.getTime(),
+    ctime: stat.ctime.getTime(),
+    birthtime: stat.birthtime.getTime(),
+  };
+}
+
+async function* globWalk({
+  fs,
+  basePath = '.',
+  pattern = '**/*',
+  yieldRoot = true,
+  yieldParents = false,
+  yieldDirectories = true,
+  yieldFiles = true,
+  yieldContents = false,
+  yieldStats = false,
+}: {
+  fs: FileSystem | FileSystemReadable;
+  basePath?: string;
+  pattern?: string;
+  yieldRoot?: boolean;
+  yieldParents?: boolean;
+  yieldDirectories?: boolean;
+  yieldFiles?: boolean;
+  yieldContents?: boolean;
+  yieldStats?: boolean;
+}): AsyncGenerator<TreeNode, void, void> {
+  const files: Array<string> = [];
+  const directoryMap: Map<number, DirectoryNode> = new Map();
+  // Path, node, parent
+  const queue: Array<[string, INode, INode]> = [];
+  let iNode = 1;
+  const basePathNormalised = path.normalize(basePath);
+  let current: [string, INode, INode] | undefined = [basePathNormalised, 1, 0];
+
+  const getParents = (parentINode: INode) => {
+    const parents: Array<DirectoryNode> = [];
+    let currentParent = parentINode;
+    while (true) {
+      const directory = directoryMap.get(currentParent);
+      directoryMap.delete(currentParent);
+      if (directory == null) break;
+      parents.unshift(directory);
+      currentParent = directory.parent;
+    }
+    return parents;
+  };
+
+  // Iterate over tree
+  const patternPath = path.join(basePathNormalised, pattern);
+  while (current != null) {
+    const [currentPath, node, parentINode] = current;
+
+    const stat = await fs.promises.stat(currentPath);
+    if (stat.isDirectory()) {
+      // `.` and `./` will not partially match the pattern, so we exclude the initial path
+      if (
+        !minimatch(currentPath, patternPath, { partial: true }) &&
+        currentPath !== basePathNormalised
+      ) {
+        current = queue.shift();
+        continue;
+      }
+      // @ts-ignore: While the types don't fully match, it matches enough for our usage.
+      const childrenPaths = await fs.promises.readdir(currentPath);
+      const children = childrenPaths.map(
+        (v) =>
+          [path.join(currentPath!, v.toString()), ++iNode, node] as [
+            string,
+            INode,
+            INode,
+          ],
+      );
+      queue.push(...children);
+      // Only yield root if we specify it
+      if (yieldRoot || node !== 1) {
+        directoryMap.set(node, {
+          type: 'directory',
+          path: currentPath,
+          iNode: node,
+          parent: parentINode,
+          children: children.map((v) => v[1]),
+          stat: yieldStats ? genStat(stat) : undefined,
+        });
+      }
+      // Wildcards can find directories so we need yield them too
+      if (minimatch(currentPath, patternPath)) {
+        // Remove current from parent list
+        directoryMap.delete(node);
+        // Yield parents
+        if (yieldParents) {
+          for (const parent of getParents(parentINode)) yield parent;
+        }
+        // Yield directory
+        if (yieldDirectories) {
+          yield {
+            type: 'directory',
+            path: currentPath,
+            iNode: node,
+            parent: parentINode,
+            children: children.map((v) => v[1]),
+            stat: yieldStats ? genStat(stat) : undefined,
+          };
+        }
+      }
+    } else if (stat.isFile()) {
+      if (!minimatch(currentPath, patternPath)) {
+        current = queue.shift();
+        continue;
+      }
+      // Get the directories in order
+      if (yieldParents) {
+        for (const parent of getParents(parentINode)) yield parent;
+      }
+      // Yield file.
+      if (yieldFiles) {
+        yield {
+          type: 'file',
+          path: currentPath,
+          iNode: node,
+          parent: parentINode,
+          cNode: files.length,
+          stat: yieldStats ? genStat(stat) : undefined,
+        };
+      }
+      files.push(currentPath);
+    }
+    current = queue.shift();
+  }
+  if (!yieldContents) return;
+  // Iterate over file contents
+  for (let i = 0; i < files.length; i++) {
+    const filePath = files[i];
+    yield {
+      type: 'content',
+      path: undefined,
+      fileName: path.basename(filePath),
+      cNode: i,
+      // @ts-ignore: While the types don't fully match, it matches enough for our usage.
+      contents: (await fs.promises.readFile(filePath)).toString(),
+    };
+  }
+}
+
 export {
   tagLast,
   refs,
@@ -138,6 +303,7 @@ export {
   walkFs,
   deleteObject,
   mkdirExists,
+  globWalk,
 };
 
 export { createVaultIdGenerator, encodeVaultId, decodeVaultId } from '../ids';

tests/vaults/VaultOps.test.ts

@@ -2,6 +2,7 @@ import type { VaultId } from '@/vaults/types';
 import type { Vault } from '@/vaults/Vault';
 import type KeyRing from '@/keys/KeyRing';
 import type { LevelPath } from '@matrixai/db';
+import type { FileTree } from '@/vaults/types';
 import fs from 'fs';
 import path from 'path';
 import os from 'os';
@@ -525,4 +526,55 @@ describe('VaultOps', () => {
     },
     globalThis.defaultTimeout * 4,
   );
+
+  describe('globWalk', () => {
+    const relativeBase = '.';
+    const dir1: string = 'dir1';
+    const dir11: string = path.join(dir1, 'dir11');
+    const file0b: string = 'file0.b';
+    const file1a: string = path.join(dir1, 'file1.a');
+    const file2b: string = path.join(dir1, 'file2.b');
+    const file3a: string = path.join(dir11, 'file3.a');
+    const file4b: string = path.join(dir11, 'file4.b');
+
+    beforeEach(async () => {
+      await vault.writeF(async (fs) => {
+        await fs.promises.mkdir(dir1);
+        await fs.promises.mkdir(dir11);
+        await fs.promises.writeFile(file0b, 'content-file0');
+        await fs.promises.writeFile(file1a, 'content-file1');
+        await fs.promises.writeFile(file2b, 'content-file2');
+        await fs.promises.writeFile(file3a, 'content-file3');
+        await fs.promises.writeFile(file4b, 'content-file4');
+      });
+    });
+
+    test('Works with efs', async () => {
+      const files = await vault.readF(async (fs) => {
+        const tree: FileTree = [];
+        for await (const treeNode of vaultsUtils.globWalk({
+          fs: fs,
+          basePath: '.',
+          yieldDirectories: true,
+          yieldFiles: true,
+          yieldParents: true,
+          yieldRoot: true,
+          yieldContents: false,
+        })) {
+          tree.push(treeNode);
+        }
+        return tree.map((v) => v.path ?? '');
+      });
+      expect(files).toContainAllValues([
+        relativeBase,
+        dir1,
+        dir11,
+        file0b,
+        file1a,
+        file2b,
+        file3a,
+        file4b,
+      ]);
+    });
+  });
 });