fix: improved the file-cache

visulima · Sep 19, 2024 · 90fa704 · 90fa704
1 parent f7e2d23
commit 90fa704
Show file tree

Hide file tree

Showing 3 changed files with 104 additions and 57 deletions.
diff --git a/packages/packem/__tests__/unit/utils/file-cache.test.ts b/packages/packem/__tests__/unit/utils/file-cache.test.ts
@@ -1,36 +1,31 @@
 import { rm } from "node:fs/promises";
 
-import { findCacheDirSync } from "@visulima/find-cache-dir";
 import { isAccessibleSync, readFileSync } from "@visulima/fs";
 import type { Pail } from "@visulima/pail";
+import { join } from "@visulima/path";
 import { temporaryDirectory } from "tempy";
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 
 import FileCache from "../../../src/utils/file-cache";
 
 const hoisted = vi.hoisted(() => {
     return {
-        cachePath: vi.fn(),
         fs: { isAccessibleSync: vi.fn(), readFileSync: vi.fn(), writeFileSync: vi.fn() },
         logger: { debug: vi.fn(), warn: vi.fn() } as unknown as Pail,
     };
 });
 
-vi.mock("@visulima/find-cache-dir", () => {
-    return { findCacheDirSync: vi.fn().mockReturnValue(hoisted.cachePath) };
-});
-
 vi.mock("@visulima/fs", () => {
     return { isAccessibleSync: hoisted.fs.isAccessibleSync, readFileSync: hoisted.fs.readFileSync, writeFileSync: hoisted.fs.writeFileSync };
 });
 
 describe("fileCache", () => {
     let temporaryDirectoryPath: string;
+    let cacheDirectoryPath: string;
 
     beforeEach(async () => {
         temporaryDirectoryPath = temporaryDirectory();
-
-        vi.mocked(findCacheDirSync).mockReturnValue(temporaryDirectoryPath);
+        cacheDirectoryPath = join(temporaryDirectoryPath, "cache")
     });
 
     afterEach(async () => {
@@ -41,43 +36,38 @@ describe("fileCache", () => {
         expect.assertions(1);
 
         // eslint-disable-next-line no-new
-        new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger);
+        new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger);
 
-        expect(hoisted.logger.debug).toHaveBeenCalledWith(expect.stringContaining("Cache path is: " + temporaryDirectoryPath));
+        expect(hoisted.logger.debug).toHaveBeenCalledWith({
+            message: "Cache path is: " + cacheDirectoryPath,
+            prefix: "file-cache"
+        });
     });
 
     it("should update isEnabled state when setter is called", () => {
         expect.assertions(1);
 
-        const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger);
+        const fileCache = new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger);
 
         fileCache.isEnabled = false;
 
         expect(fileCache.isEnabled).toBeFalsy();
     });
 
-    it("should return correct cache path when getter is called", () => {
-        expect.assertions(1);
-
-        const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger);
-
-        expect(fileCache.cachePath).toBeDefined();
-    });
-
     it("should return true if file is accessible in the cache", () => {
         expect.assertions(1);
 
         vi.mocked(isAccessibleSync).mockReturnValue(true);
 
-        const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger);
+        const fileCache = new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger);
 
         expect(fileCache.has("testFile")).toBeTruthy();
     });
 
     it("should retrieve data from memory cache if available", () => {
         expect.assertions(3);
 
-        const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger);
+        const fileCache = new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger);
         const data = { key: "value" };
 
         vi.mocked(isAccessibleSync).mockReturnValue(true);
@@ -96,19 +86,21 @@ describe("fileCache", () => {
         vi.mocked(readFileSync).mockReturnValue(jsonData);
         vi.mocked(isAccessibleSync).mockReturnValue(true);
 
-        const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger);
+        const fileCache = new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger);
 
         expect(fileCache.get("/path/to/file")).toStrictEqual({ key: "value" });
     });
 
     it("should handle undefined cache path gracefully in constructor", () => {
         expect.assertions(1);
 
-        vi.mocked(findCacheDirSync).mockReturnValue(undefined);
         // eslint-disable-next-line no-new
-        new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger);
+        new FileCache(temporaryDirectoryPath, undefined, "hash123", hoisted.logger);
 
-        expect(hoisted.logger.debug).toHaveBeenCalledWith("Could not create cache directory.");
+        expect(hoisted.logger.debug).toHaveBeenCalledWith({
+            message: "Could not create cache directory.",
+            prefix: "file-cache"
+        });
     });
 
     it("should handle non-JSON data correctly in get method", () => {
@@ -119,30 +111,30 @@ describe("fileCache", () => {
         vi.mocked(readFileSync).mockReturnValue(nonJsonData);
         vi.mocked(isAccessibleSync).mockReturnValue(true);
 
-        const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger);
+        const fileCache = new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger);
 
         expect(fileCache.get("/path/to/file")).toStrictEqual(nonJsonData);
     });
 
     it("should handle undefined data input gracefully in set method", () => {
         expect.assertions(1);
 
-        const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger);
+        const fileCache = new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger);
         expect(() => fileCache.set("testFile", undefined)).not.toThrow();
     });
 
     it("should return false if cache is disabled in has method", () => {
         expect.assertions(1);
 
-        const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger);
+        const fileCache = new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger);
         fileCache.isEnabled = false;
         expect(fileCache.has("testFile")).toBeFalsy();
     });
 
     it("should return undefined if cache is disabled in get method", () => {
         expect.assertions(1);
 
-        const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger);
+        const fileCache = new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger);
         fileCache.isEnabled = false;
         expect(fileCache.get("testFile")).toBeUndefined();
     });

diff --git a/packages/packem/src/create-bundler.ts b/packages/packem/src/create-bundler.ts
@@ -1,8 +1,11 @@
+import { readdirSync } from "node:fs";
+import { rm } from "node:fs/promises";
 import Module from "node:module";
 import { cwd } from "node:process";
 
 import { bold, cyan } from "@visulima/colorize";
-import { emptyDir, ensureDirSync, isAccessible, isAccessibleSync } from "@visulima/fs";
+import { findCacheDirSync } from "@visulima/find-cache-dir";
+import { emptyDir, ensureDirSync, isAccessible, isAccessibleSync, readJsonSync } from "@visulima/fs";
 import { duration } from "@visulima/humanizer";
 import type { PackageJson } from "@visulima/package";
 import type { Pail } from "@visulima/pail";
@@ -500,6 +503,31 @@ const cleanDistributionDirectories = async (context: BuildContext): Promise<void
     }
 };
 
+const removeOldCacheFolders = async (cachePath: string | undefined, logger: Pail): Promise<void> => {
+    if (cachePath && isAccessibleSync(join(cachePath, "keystore.json"))) {
+        const keyStore: Record<string, string> = readJsonSync(join(cachePath, "keystore.json"));
+
+        // eslint-disable-next-line security/detect-non-literal-fs-filename
+        const cacheDirectories = readdirSync(cachePath, {
+            withFileTypes: true,
+        }).filter((dirent) => dirent.isDirectory());
+
+        for await (const dirent of cacheDirectories) {
+            if (!keyStore[dirent.name]) {
+                await rm(join(cachePath, dirent.name), {
+                    force: true,
+                    recursive: true,
+                });
+
+                logger.info({
+                    message: "Removing " + dirent.name + " file cache, the cache key is not used anymore.",
+                    prefix: "file-cache"
+                })
+            }
+        }
+    }
+};
+
 const createBundler = async (
     rootDirectory: string,
     mode: Mode,
@@ -608,18 +636,13 @@ const createBundler = async (
             }),
         );
 
+        const cachePath = findCacheDirSync(packageJson.name as string, {
+            cwd: rootDirectory,
+        });
+
         for await (const config of arrayify(buildConfig)) {
             const cacheKey = packageJsonCacheKey + getHash(JSON.stringify(config));
-
-            const fileCache = new FileCache(rootDirectory, cacheKey, logger);
-
-            // clear cache if the cache key has changed
-            if (fileCache.cachePath && !isAccessibleSync(join(fileCache.cachePath, cacheKey)) && isAccessibleSync(fileCache.cachePath)) {
-                logger.info("Clearing file cache because the cache key has changed.");
-
-                await emptyDir(fileCache.cachePath);
-            }
-
+            const fileCache = new FileCache(rootDirectory, cachePath, cacheKey, logger);
             const context = await createContext(logger, rootDirectory, mode, environment, debug ?? false, restInputConfig, config, packageJson, tsconfig, jiti);
 
             fileCache.isEnabled = context.options.fileCache as boolean;
@@ -670,6 +693,8 @@ const createBundler = async (
             logger.raw("\n⚡️ Build run in " + getDuration());
         }
 
+        await removeOldCacheFolders(cachePath, logger);
+
         // Restore all wrapped console methods
         logger.restoreAll();
         // eslint-disable-next-line @typescript-eslint/no-explicit-any

diff --git a/packages/packem/src/utils/file-cache.ts b/packages/packem/src/utils/file-cache.ts
@@ -1,5 +1,4 @@
-import { findCacheDirSync } from "@visulima/find-cache-dir";
-import { isAccessibleSync, readFileSync, writeFileSync } from "@visulima/fs";
+import { isAccessibleSync, readFileSync, readJsonSync, writeFileSync, writeJsonSync } from "@visulima/fs";
 import type { Pail } from "@visulima/pail";
 import { join, toNamespacedPath } from "@visulima/path";
 
@@ -18,35 +17,37 @@ class FileCache {
 
     readonly #cachePath: undefined | string;
 
-    readonly #packageJsonHash: string;
+    readonly #hashKey: string;
 
     #isEnabled = true;
 
     readonly #memoryCache = new Map<string>();
 
-    public constructor(cwd: string, packageJsonHash: string, logger: Pail) {
+    public constructor(cwd: string, cachePath: string | undefined, hashKey: string, logger: Pail) {
         this.#cwd = cwd;
-        this.#cachePath = findCacheDirSync("@visulima/packem", {
-            cwd,
-        });
-
-        this.#packageJsonHash = packageJsonHash;
+        this.#hashKey = hashKey;
 
-        if (this.#cachePath === undefined) {
-            logger.debug("Could not create cache directory.");
+        if (cachePath === undefined) {
+            logger.debug({
+                message: "Could not create cache directory.",
+                prefix: "file-cache",
+            });
         } else {
-            logger.debug(`Cache path is: ${this.#cachePath}`);
+            this.#cachePath = cachePath;
+
+            logger.debug({
+                message: "Cache path is: " + this.#cachePath,
+                prefix: "file-cache",
+            });
         }
+
+        this.createOrUpdateKeyStorage(hashKey, logger);
     }
 
     public set isEnabled(value: boolean) {
         this.#isEnabled = value;
     }
 
-    public get cachePath(): string | undefined {
-        return this.#cachePath;
-    }
-
     public has(name: string, subDirectory?: string): boolean {
         if (!this.#isEnabled) {
             return false;
@@ -119,7 +120,36 @@ class FileCache {
 
         optimizedName = optimizedName.replaceAll(":", "-");
 
-        return join(this.#cachePath as string, this.#packageJsonHash, subDirectory?.replaceAll(":", "-") ?? "", toNamespacedPath(optimizedName));
+        return join(this.#cachePath as string, this.#hashKey, subDirectory?.replaceAll(":", "-") ?? "", toNamespacedPath(optimizedName));
+    }
+
+    private createOrUpdateKeyStorage(hashKey: string, logger: Pail): void {
+        try {
+            let keyStore: Record<string, string> = {};
+
+            const keyStorePath = join(this.#cachePath as string, "keystore.json");
+
+            if (isAccessibleSync(keyStorePath)) {
+                keyStore = readJsonSync(keyStorePath);
+            }
+
+            // eslint-disable-next-line security/detect-object-injection
+            if (keyStore[hashKey] === undefined) {
+                // eslint-disable-next-line security/detect-object-injection
+                keyStore[hashKey] = new Date().toISOString();
+            }
+
+            writeJsonSync(keyStorePath, keyStore, {
+                overwrite: true,
+            });
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        } catch (error: any) {
+            logger.debug({
+                context: error,
+                message: error.message,
+                prefix: "file-cache",
+            });
+        }
     }
 }