diff --git a/src/vaults/types.ts b/src/vaults/types.ts index 3c1643c1a..6e54e3b10 100644 --- a/src/vaults/types.ts +++ b/src/vaults/types.ts @@ -134,6 +134,58 @@ type VaultName = string; type VaultActions = Partial>; +type FileTree = Array; +type TreeNode = DirectoryNode | FileNode | ContentNode; +type FilePath = string; +type INode = number; +type CNode = number; + +type StatEncoded = { + isSymbolicLink: boolean; + type: 'FILE' | 'DIRECTORY' | 'OTHER'; + dev: number; + ino: number; + mode: number; + nlink: number; + uid: number; + gid: number; + rdev: number; + size: number; + blksize: number; + blocks: number; + atime: number; + mtime: number; + ctime: number; + birthtime: number; +}; + +type DirectoryNode = { + type: 'directory'; + path: FilePath; + iNode: INode; + parent: INode; + children: Array; + stat?: StatEncoded; +}; + +type FileNode = { + type: 'file'; + path: FilePath; + iNode: INode; + parent: INode; + cNode: CNode; + stat?: StatEncoded; +}; + +// Keeping this separate from `FileNode` so we can optionally not include it. +type ContentNode = { + type: 'content'; + path: undefined; + fileName: string; + cNode: CNode; + contents: string; +}; + export { vaultActions }; export type { @@ -148,6 +200,15 @@ export type { FileSystemWritable, VaultName, VaultActions, + FileTree, + TreeNode, + FilePath, + INode, + CNode, + StatEncoded, + DirectoryNode, + FileNode, + ContentNode, }; export { tagLast, refs }; diff --git a/src/vaults/utils.ts b/src/vaults/utils.ts index 449cd4adb..48764317a 100644 --- a/src/vaults/utils.ts +++ b/src/vaults/utils.ts @@ -1,14 +1,20 @@ -import type { EncryptedFS } from 'encryptedfs'; +import type { EncryptedFS, Stat } from 'encryptedfs'; +import type { FileSystem } from '../types'; import type { VaultRef, VaultAction, CommitId, FileSystemReadable, FileSystemWritable, + TreeNode, + DirectoryNode, + INode, + StatEncoded, } from './types'; import type { NodeId } from '../ids/types'; import type { Path } from 'encryptedfs/dist/types'; import path from 'path'; +import { minimatch } from 'minimatch'; import { pathJoin } from 'encryptedfs/dist/utils'; import * as vaultsErrors from './errors'; import { tagLast, refs, vaultActions } from './types'; @@ -123,6 +129,165 @@ async function mkdirExists(efs: FileSystemWritable, directory: string) { } } +function genStat(stat: Stat): StatEncoded { + return { + isSymbolicLink: stat.isSymbolicLink(), + type: stat.isFile() ? 'FILE' : stat.isDirectory() ? 'DIRECTORY' : 'OTHER', + dev: stat.dev, + ino: stat.ino, + mode: stat.mode, + nlink: stat.nlink, + uid: stat.uid, + gid: stat.gid, + rdev: stat.rdev, + size: stat.size, + blksize: stat.blksize, + blocks: stat.blocks, + atime: stat.atime.getTime(), + mtime: stat.mtime.getTime(), + ctime: stat.ctime.getTime(), + birthtime: stat.birthtime.getTime(), + }; +} + +async function* globWalk({ + fs, + basePath = '.', + pattern = '**/*', + yieldRoot = true, + yieldParents = false, + yieldDirectories = true, + yieldFiles = true, + yieldContents = false, + yieldStats = false, +}: { + fs: FileSystem | FileSystemReadable; + basePath?: string; + pattern?: string; + yieldRoot?: boolean; + yieldParents?: boolean; + yieldDirectories?: boolean; + yieldFiles?: boolean; + yieldContents?: boolean; + yieldStats?: boolean; +}): AsyncGenerator { + const files: Array = []; + const directoryMap: Map = new Map(); + // Path, node, parent + const queue: Array<[string, INode, INode]> = []; + let iNode = 1; + const basePathNormalised = path.normalize(basePath); + let current: [string, INode, INode] | undefined = [basePathNormalised, 1, 0]; + + const getParents = (parentINode: INode) => { + const parents: Array = []; + let currentParent = parentINode; + while (true) { + const directory = directoryMap.get(currentParent); + directoryMap.delete(currentParent); + if (directory == null) break; + parents.unshift(directory); + currentParent = directory.parent; + } + return parents; + }; + + // Iterate over tree + const patternPath = path.join(basePathNormalised, pattern); + while (current != null) { + const [currentPath, node, parentINode] = current; + + const stat = await fs.promises.stat(currentPath); + if (stat.isDirectory()) { + // `.` and `./` will not partially match the pattern, so we exclude the initial path + if ( + !minimatch(currentPath, patternPath, { partial: true }) && + currentPath !== basePathNormalised + ) { + current = queue.shift(); + continue; + } + // @ts-ignore: While the types don't fully match, it matches enough for our usage. + const childrenPaths = await fs.promises.readdir(currentPath); + const children = childrenPaths.map( + (v) => + [path.join(currentPath!, v.toString()), ++iNode, node] as [ + string, + INode, + INode, + ], + ); + queue.push(...children); + // Only yield root if we specify it + if (yieldRoot || node !== 1) { + directoryMap.set(node, { + type: 'directory', + path: currentPath, + iNode: node, + parent: parentINode, + children: children.map((v) => v[1]), + stat: yieldStats ? genStat(stat) : undefined, + }); + } + // Wildcards can find directories so we need yield them too + if (minimatch(currentPath, patternPath)) { + // Remove current from parent list + directoryMap.delete(node); + // Yield parents + if (yieldParents) { + for (const parent of getParents(parentINode)) yield parent; + } + // Yield directory + if (yieldDirectories) { + yield { + type: 'directory', + path: currentPath, + iNode: node, + parent: parentINode, + children: children.map((v) => v[1]), + stat: yieldStats ? genStat(stat) : undefined, + }; + } + } + } else if (stat.isFile()) { + if (!minimatch(currentPath, patternPath)) { + current = queue.shift(); + continue; + } + // Get the directories in order + if (yieldParents) { + for (const parent of getParents(parentINode)) yield parent; + } + // Yield file. + if (yieldFiles) { + yield { + type: 'file', + path: currentPath, + iNode: node, + parent: parentINode, + cNode: files.length, + stat: yieldStats ? genStat(stat) : undefined, + }; + } + files.push(currentPath); + } + current = queue.shift(); + } + if (!yieldContents) return; + // Iterate over file contents + for (let i = 0; i < files.length; i++) { + const filePath = files[i]; + yield { + type: 'content', + path: undefined, + fileName: path.basename(filePath), + cNode: i, + // @ts-ignore: While the types don't fully match, it matches enough for our usage. + contents: (await fs.promises.readFile(filePath)).toString(), + }; + } +} + export { tagLast, refs, @@ -138,6 +303,7 @@ export { walkFs, deleteObject, mkdirExists, + globWalk, }; export { createVaultIdGenerator, encodeVaultId, decodeVaultId } from '../ids'; diff --git a/tests/vaults/VaultOps.test.ts b/tests/vaults/VaultOps.test.ts index ea34f63eb..6b57097e9 100644 --- a/tests/vaults/VaultOps.test.ts +++ b/tests/vaults/VaultOps.test.ts @@ -2,6 +2,7 @@ import type { VaultId } from '@/vaults/types'; import type { Vault } from '@/vaults/Vault'; import type KeyRing from '@/keys/KeyRing'; import type { LevelPath } from '@matrixai/db'; +import type { FileTree } from '@/vaults/types'; import fs from 'fs'; import path from 'path'; import os from 'os'; @@ -525,4 +526,55 @@ describe('VaultOps', () => { }, globalThis.defaultTimeout * 4, ); + + describe('globWalk', () => { + const relativeBase = '.'; + const dir1: string = 'dir1'; + const dir11: string = path.join(dir1, 'dir11'); + const file0b: string = 'file0.b'; + const file1a: string = path.join(dir1, 'file1.a'); + const file2b: string = path.join(dir1, 'file2.b'); + const file3a: string = path.join(dir11, 'file3.a'); + const file4b: string = path.join(dir11, 'file4.b'); + + beforeEach(async () => { + await vault.writeF(async (fs) => { + await fs.promises.mkdir(dir1); + await fs.promises.mkdir(dir11); + await fs.promises.writeFile(file0b, 'content-file0'); + await fs.promises.writeFile(file1a, 'content-file1'); + await fs.promises.writeFile(file2b, 'content-file2'); + await fs.promises.writeFile(file3a, 'content-file3'); + await fs.promises.writeFile(file4b, 'content-file4'); + }); + }); + + test('Works with efs', async () => { + const files = await vault.readF(async (fs) => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: '.', + yieldDirectories: true, + yieldFiles: true, + yieldParents: true, + yieldRoot: true, + yieldContents: false, + })) { + tree.push(treeNode); + } + return tree.map((v) => v.path ?? ''); + }); + expect(files).toContainAllValues([ + relativeBase, + dir1, + dir11, + file0b, + file1a, + file2b, + file3a, + file4b, + ]); + }); + }); }); diff --git a/tests/vaults/utils.test.ts b/tests/vaults/utils.test.ts index de9d97663..123f9f081 100644 --- a/tests/vaults/utils.test.ts +++ b/tests/vaults/utils.test.ts @@ -1,4 +1,4 @@ -import type { VaultId } from '@/vaults/types'; +import type { FileTree, VaultId } from '@/vaults/types'; import fs from 'fs'; import os from 'os'; import path from 'path'; @@ -7,38 +7,6 @@ import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; import { IdRandom } from '@matrixai/id'; import * as vaultsUtils from '@/vaults/utils'; import * as keysUtils from '@/keys/utils'; -import {FileSystem} from "@"; - -type FileTree = Array; -type TreeNode = DirectoryNode | FileNode | ContentNode; -type FilePath = string; -type Inode = number; -type Cnode = number; - -type DirectoryNode = { - type: 'directory'; - path: FilePath; - iNode: Inode; - parent: Inode; - children: Array; - //relevant stats... -} - -type FileNode = { - type: 'file'; - path: FilePath; - iNode: Inode; - parent: Inode; - cNode: Cnode; - //relevant stats... -} - -// Keeping this separate from `FileNode` so we can optionally not include it. -type ContentNode = { - type: 'content'; - cNode: Cnode; - contents: string; -} describe('Vaults utils', () => { const logger = new Logger('Vaults utils tests', LogLevel.WARN, [ @@ -116,61 +84,434 @@ describe('Vaults utils', () => { expect(vaultsUtils.decodeVaultId('zF4VfxTOOSHORTxTV9')).toBeUndefined(); }); - test('globbing walker', async () => { + describe('globWalk', () => { + let cwd: string; - await fs.promises.writeFile(path.join(dataDir, 'file1'), 'content1'); - await fs.promises.mkdir(path.join(dataDir, 'dir1')); - await fs.promises.writeFile(path.join(dataDir, 'dir1', 'file2'), 'content2'); - // We need a basic walker that traverses the file tree directories first, then files. + const relativeBase = '.'; + const dir1: string = 'dir1'; + const dir2: string = 'dir2'; + const dir11: string = path.join(dir1, 'dir11'); + const dir12: string = path.join(dir1, 'dir12'); + const dir21: string = path.join(dir2, 'dir21'); + const dir22: string = path.join(dir2, 'dir22'); + const file0b: string = 'file0.b'; + const file1a: string = path.join(dir11, 'file1.a'); + const file2b: string = path.join(dir11, 'file2.b'); + const file3a: string = path.join(dir12, 'file3.a'); + const file4b: string = path.join(dir12, 'file4.b'); + const file5a: string = path.join(dir21, 'file5.a'); + const file6b: string = path.join(dir21, 'file6.b'); + const file7a: string = path.join(dir22, 'file7.a'); + const file8b: string = path.join(dir22, 'file8.b'); + const file9a: string = path.join(dir22, 'file9.a'); + beforeEach(async () => { + await fs.promises.mkdir(path.join(dataDir, dir1)); + await fs.promises.mkdir(path.join(dataDir, dir11)); + await fs.promises.mkdir(path.join(dataDir, dir12)); + await fs.promises.mkdir(path.join(dataDir, dir2)); + await fs.promises.mkdir(path.join(dataDir, dir21)); + await fs.promises.mkdir(path.join(dataDir, dir22)); + await fs.promises.writeFile(path.join(dataDir, file0b), 'content-file0'); + await fs.promises.writeFile(path.join(dataDir, file1a), 'content-file1'); + await fs.promises.writeFile(path.join(dataDir, file2b), 'content-file2'); + await fs.promises.writeFile(path.join(dataDir, file3a), 'content-file3'); + await fs.promises.writeFile(path.join(dataDir, file4b), 'content-file4'); + await fs.promises.writeFile(path.join(dataDir, file5a), 'content-file5'); + await fs.promises.writeFile(path.join(dataDir, file6b), 'content-file6'); + await fs.promises.writeFile(path.join(dataDir, file7a), 'content-file7'); + await fs.promises.writeFile(path.join(dataDir, file8b), 'content-file8'); + await fs.promises.writeFile(path.join(dataDir, file9a), 'content-file9'); + cwd = process.cwd(); + process.chdir(dataDir); + }); + afterEach(async () => { + process.chdir(cwd); + }); - async function *walk(fs: FileSystem, startPath: string): AsyncGenerator { - const files: Array = []; - // path, node, parent - const queue: Array<[string, Inode, Inode]> = []; - let iNode = 1; - let current: [string, Inode, Inode] | undefined = [startPath, 1, 0]; - // iterate over tree - while (current != null) { - const [currentPath, node, parent] = current; - const stat = await fs.promises.stat(currentPath); - if (stat.isDirectory()) { - const childrenPaths = await fs.promises.readdir(currentPath); - const children = childrenPaths.map(v => [path.join(currentPath!, v), ++iNode, node] as [string, Inode, Inode]) - queue.push(...children); - yield { - type: "directory", - path: currentPath, - iNode: node, - parent, - children: children.map(v => v[1]), - // TODO: stats - } - } else if (stat.isFile()) { - yield { - type: 'file', - path: currentPath, - iNode: node, - parent: parent, - cNode: files.length, - // TODO: stats - }; - files.push(currentPath); - } else console.log('neither file or directory'); - current = queue.shift(); + test('Works with relative base path `.`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + yieldDirectories: true, + yieldFiles: true, + yieldParents: true, + yieldRoot: true, + yieldContents: false, + })) { + tree.push(treeNode); } - // iterate over file contents - for (let i = 0; i < files.length; i++) { - yield { - type: 'content', - cNode: i, - contents: (await fs.promises.readFile(files[i])).toString(), - } + const files = tree.map((v) => v.path ?? ''); + expect(files).toContainAllValues([ + relativeBase, + dir1, + dir2, + dir11, + dir12, + dir21, + dir22, + file0b, + file1a, + file2b, + file3a, + file4b, + file5a, + file6b, + file7a, + file8b, + file9a, + ]); + }); + test('Works with relative base path `./`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: './', + yieldDirectories: true, + yieldFiles: true, + yieldParents: true, + yieldRoot: true, + yieldContents: false, + })) { + tree.push(treeNode); } - } - - for await (const pathString of walk(fs, dataDir)) { - console.log(pathString); - }; - }) + const files = tree.map((v) => v.path ?? ''); + expect(files).toContainAllValues([ + './', + dir1, + dir2, + dir11, + dir12, + dir21, + dir22, + file0b, + file1a, + file2b, + file3a, + file4b, + file5a, + file6b, + file7a, + file8b, + file9a, + ]); + }); + test('Works with relative base path `./dir1`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: './dir1', + yieldDirectories: true, + yieldFiles: true, + yieldParents: true, + yieldRoot: true, + yieldContents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path ?? ''); + expect(files).toContainAllValues([ + dir1, + dir11, + dir12, + file1a, + file2b, + file3a, + file4b, + ]); + }); + test('Works with absolute base path', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: dataDir, + yieldDirectories: true, + yieldFiles: true, + yieldParents: true, + yieldRoot: true, + yieldContents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path ?? ''); + expect(files).toContainAllValues( + [ + relativeBase, + dir1, + dir2, + dir11, + dir12, + dir21, + dir22, + file0b, + file1a, + file2b, + file3a, + file4b, + file5a, + file6b, + file7a, + file8b, + file9a, + ].map((v) => path.join(dataDir, v)), + ); + }); + test('Yields parent directories with `yieldParents`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + yieldParents: true, + yieldFiles: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).toContainAllValues([ + relativeBase, + dir2, + dir1, + dir11, + dir12, + dir21, + dir22, + ]); + }); + test('Does not yield the base path with `yieldParents` and `yieldRoot`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + yieldRoot: false, + yieldParents: true, + yieldFiles: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).not.toInclude(relativeBase); + expect(files).toContainAllValues([ + dir2, + dir1, + dir11, + dir12, + dir21, + dir22, + ]); + }); + test('Does not yield the base path with `yieldParents` and `yieldRoot` and absolute paths', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: dataDir, + yieldRoot: false, + yieldParents: true, + yieldFiles: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).not.toInclude(dataDir); + expect(files).toContainAllValues( + [dir2, dir1, dir11, dir12, dir21, dir22].map((v) => + path.join(dataDir, v), + ), + ); + }); + test('Yields file contents directories with `yieldContents`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + yieldFiles: false, + yieldDirectories: false, + yieldContents: true, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => (v.type === 'content' ? v.contents : '')); + expect(files).toContainAllValues([ + 'content-file0', + 'content-file9', + 'content-file1', + 'content-file2', + 'content-file3', + 'content-file4', + 'content-file5', + 'content-file6', + 'content-file7', + 'content-file8', + ]); + }); + test('Yields stats with `yieldStats`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + yieldStats: true, + yieldFiles: true, + yieldDirectories: true, + yieldContents: false, + })) { + tree.push(treeNode); + } + tree.forEach((v) => + v.type === 'directory' || v.type === 'file' + ? expect(v.stat).toBeDefined() + : '', + ); + }); + // Globbing examples + test('glob with wildcard', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + pattern: '*', + yieldFiles: true, + yieldDirectories: true, + yieldParents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).toContainAllValues([dir1, dir2, file0b]); + }); + test('glob with wildcard ignores directories with `yieldDirectories: false`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + pattern: '*', + yieldFiles: true, + yieldDirectories: false, + yieldParents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).not.toContainAllValues([relativeBase, dir1, dir2]); + expect(files).toContainAllValues([file0b]); + }); + test('glob with wildcard ignores files with `yieldFiles: false`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + pattern: '*', + yieldFiles: false, + yieldDirectories: true, + yieldParents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).not.toContainAllValues([file0b]); + expect(files).toContainAllValues([dir1, dir2]); + }); + test('glob with globstar', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + pattern: '**', + yieldFiles: true, + yieldDirectories: true, + yieldParents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).not.toInclude(relativeBase); + expect(files).toContainAllValues([ + dir1, + dir2, + file0b, + dir11, + dir12, + dir21, + dir22, + file1a, + file2b, + file3a, + file4b, + file5a, + file6b, + file7a, + file8b, + file9a, + ]); + }); + test('glob with globstar and directory pattern', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + pattern: '**/dir2/**', + yieldFiles: true, + yieldDirectories: true, + yieldParents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).not.toContainAllValues([ + relativeBase, + dir1, + dir2, + file0b, + dir11, + dir12, + file1a, + file2b, + file3a, + file4b, + ]); + expect(files).toContainAllValues([ + dir21, + dir22, + file5a, + file6b, + file7a, + file8b, + file9a, + ]); + }); + test('glob with globstar and wildcard', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + pattern: '**/*.a', + yieldFiles: true, + yieldDirectories: true, + yieldParents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).not.toContainAllValues([ + relativeBase, + dir1, + dir2, + file0b, + dir11, + dir12, + dir21, + dir22, + file2b, + file4b, + file6b, + file8b, + ]); + expect(files).toContainAllValues([ + file1a, + file3a, + file5a, + file7a, + file9a, + ]); + }); + }); });