From c639c31a8ffed6a25357c5ff0172162ffa9b9497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Jim=C3=A9nez=20Rivera?= Date: Sun, 2 Feb 2025 19:12:46 +0100 Subject: [PATCH 1/5] Add paths in fileInDevice --- src/types/callbacks.type.ts | 1 + src/virtual-drive.ts | 9 ++- .../detect-context-menu-action.service.ts | 62 ++++++------------- src/watcher/events/on-raw.service.ts | 40 +++++------- 4 files changed, 44 insertions(+), 68 deletions(-) diff --git a/src/types/callbacks.type.ts b/src/types/callbacks.type.ts index 8702ed9b..9b56f31a 100644 --- a/src/types/callbacks.type.ts +++ b/src/types/callbacks.type.ts @@ -1,6 +1,7 @@ export type NapiCallbackFunction = (...args: any[]) => any; export type InputSyncCallbacks = { + // TODO: make it mandatory and add the exact type fetchDataCallback?: NapiCallbackFunction; validateDataCallback?: NapiCallbackFunction; cancelFetchDataCallback?: NapiCallbackFunction; diff --git a/src/virtual-drive.ts b/src/virtual-drive.ts index c9a7f31b..1867b2a4 100644 --- a/src/virtual-drive.ts +++ b/src/virtual-drive.ts @@ -218,7 +218,14 @@ class VirtualDrive { callbacks: Callbacks, logoPath: string ): Promise { - this.callbacks = callbacks; + this.callbacks = { + ...callbacks, + fetchDataCallback: (...args) => { + const path = args[0]; + this.watcher.fileInDevice.add(path); + return callbacks.fetchDataCallback?.(...args); + } + }; return await addon.registerSyncRoot( this.syncRootPath, providerName, diff --git a/src/watcher/detect-context-menu-action.service.ts b/src/watcher/detect-context-menu-action.service.ts index 1ed3a4ff..d35159e2 100644 --- a/src/watcher/detect-context-menu-action.service.ts +++ b/src/watcher/detect-context-menu-action.service.ts @@ -10,15 +10,13 @@ export class DetectContextMenuActionService { const { prev, curr } = details; const status = self.virtualDriveFn.CfGetPlaceHolderState(path); - const attribute = self.virtualDriveFn.CfGetPlaceHolderAttributes(path); const itemId = self.virtualDriveFn.CfGetPlaceHolderIdentity(path); - const isInDevice = self.fileInDevice.has(path); + const isInDevice = self.fileInDevice.has(itemId) || self.fileInDevice.has(path); self.logger.info({ - event: "onRaw", + event: "change", path, status, - attribute, itemId, isInDevice, prev: { @@ -30,52 +28,30 @@ export class DetectContextMenuActionService { size: curr.size, ctimeMs: curr.ctimeMs, mtimeMs: curr.mtimeMs, - blocks: curr.blocks, }, }); - if ( - prev.size === curr.size && - prev.ctimeMs !== curr.ctimeMs && - prev.mtimeMs === curr.mtimeMs && - status.pinState === PinState.AlwaysLocal && - status.syncState === SyncState.InSync && - !isInDevice - ) { - self.fileInDevice.add(path); - - if (curr.blocks !== 0) { - // This event is triggered from the addon - return "Doble click en el archivo"; - } - - self.queueManager.enqueue({ path, type: typeQueue.hydrate, isFolder, fileId: itemId }); - return "Mantener siempre en el dispositivo"; - } - - if ( - prev.size === curr.size && - prev.ctimeMs !== curr.ctimeMs && - status.pinState == PinState.OnlineOnly && - status.syncState == SyncState.InSync - ) { - // TODO: we need to disable this for now even if dehydate it's called two times - // because files that are a .zip have blocks === 0, so they never dehydrate - // because it's seems that it's already been dehydrated - // if (curr.blocks === 0) { - // return "Liberando espacio"; - // } - - self.fileInDevice.delete(path); - self.queueManager.enqueue({ path, type: typeQueue.dehydrate, isFolder, fileId: itemId }); - return "Liberar espacio"; - } - + // TODO: check same size but different content if (prev.size !== curr.size) { self.queueManager.enqueue({ path, type: typeQueue.changeSize, isFolder, fileId: itemId }); - self.fileInDevice.add(path); + self.fileInDevice.add(itemId); return "Cambio de tamaño"; } + + if (prev.ctimeMs !== curr.ctimeMs && status.syncState === SyncState.InSync) { + if (status.pinState === PinState.AlwaysLocal && !isInDevice) { + self.fileInDevice.add(itemId); + self.queueManager.enqueue({ path, type: typeQueue.hydrate, isFolder, fileId: itemId }); + return "Mantener siempre en el dispositivo"; + } + + if (status.pinState == PinState.OnlineOnly && isInDevice) { + self.fileInDevice.delete(path); + self.fileInDevice.delete(itemId); + self.queueManager.enqueue({ path, type: typeQueue.dehydrate, isFolder, fileId: itemId }); + return "Liberar espacio"; + } + } } } diff --git a/src/watcher/events/on-raw.service.ts b/src/watcher/events/on-raw.service.ts index ed3c1c56..8f6ac77e 100644 --- a/src/watcher/events/on-raw.service.ts +++ b/src/watcher/events/on-raw.service.ts @@ -1,36 +1,28 @@ import { stat } from "fs/promises"; -import { extname } from "path"; import { DetectContextMenuActionService } from "../detect-context-menu-action.service"; import { Watcher } from "../watcher"; export class OnRawService { - constructor(private readonly detectContextMenuAction: DetectContextMenuActionService = new DetectContextMenuActionService()) {} + constructor(private readonly detectContextMenuAction = new DetectContextMenuActionService()) {} async execute({ self, event, path, details }: TProps) { - if (event === "change" && details.prev && details.curr) { - if (extname(path) === "") { - self.logger.info({ event: "onRaw", path, details: "No extension" }); - return; - } - - const item = await stat(path); - if (item.isDirectory()) { - self.logger.info({ event: "onRaw", path, details: "Is directory" }); - return; - } - - // // Ignorar archivos vacíos - // if (item.size === 0) { - // self.logger.info("Archivo vacío ignorado", path); - // return; - // } - - const action = await this.detectContextMenuAction.execute({ self, details, path, isFolder: false }); - - if (action) { - self.logger.info({ event: "onRaw", path, action }); + try { + if (event === "change" && details.prev && details.curr) { + const item = await stat(path); + if (item.isDirectory()) { + self.logger.info({ event: "change", path, details: "Is directory" }); + return; + } + + const action = await this.detectContextMenuAction.execute({ self, details, path, isFolder: false }); + + if (action) { + self.logger.info({ event: "change", path, action }); + } } + } catch (error) { + self.logger.error("Error en change", error); } } } From e1e171f9e275c3e6467b823269c5c1dad426c196 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Jim=C3=A9nez=20Rivera?= Date: Sun, 2 Feb 2025 19:16:57 +0100 Subject: [PATCH 2/5] Update virtual-drive.ts --- src/virtual-drive.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/virtual-drive.ts b/src/virtual-drive.ts index 1867b2a4..c2cc0ff9 100644 --- a/src/virtual-drive.ts +++ b/src/virtual-drive.ts @@ -54,6 +54,8 @@ class VirtualDrive { FILE_ATTRIBUTE_NORMAL: 0x1, }; + // TODO: getPlaceholderStates in the beginning + this.watcher = new Watcher(); this.syncRootPath = syncRootPath; From 652ae16c122dac635c17c41d7b3030a4f1a7c5c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Jim=C3=A9nez=20Rivera?= Date: Mon, 3 Feb 2025 12:52:07 +0100 Subject: [PATCH 3/5] Update detect-context-menu-action.service.ts --- src/watcher/detect-context-menu-action.service.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/watcher/detect-context-menu-action.service.ts b/src/watcher/detect-context-menu-action.service.ts index 2e6ef5e7..09376cfa 100644 --- a/src/watcher/detect-context-menu-action.service.ts +++ b/src/watcher/detect-context-menu-action.service.ts @@ -10,9 +10,8 @@ export class DetectContextMenuActionService { const { prev, curr } = details; const status = self.addon.getPlaceholderState({ path }); - const attribute = self.addon.getPlaceholderAttribute({ path }); const itemId = self.addon.getFileIdentity({ path }); - const isInDevice = self.fileInDevice.has(path); + const isInDevice = self.fileInDevice.has(itemId) || self.fileInDevice.has(path); self.logger.info({ event: "change", From 100200249a58e4aba97e3300a39a1e66688718e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Jim=C3=A9nez=20Rivera?= Date: Mon, 3 Feb 2025 14:56:46 +0100 Subject: [PATCH 4/5] Get placeholder states --- .../callbacks/notify-fetch-data.callback.ts | 5 +- examples/populate.ts | 5 +- examples/register.ts | 23 ++-- src/get-placeholder-states.ts | 30 +++++ src/is-file-in-device.ts | 12 ++ src/types/callbacks.type.ts | 8 +- src/virtual-drive.ts | 104 +++++++----------- src/virtual-drive.unit.test.ts | 7 +- 8 files changed, 110 insertions(+), 84 deletions(-) create mode 100644 src/get-placeholder-states.ts create mode 100644 src/is-file-in-device.ts diff --git a/examples/callbacks/notify-fetch-data.callback.ts b/examples/callbacks/notify-fetch-data.callback.ts index 0b9103cb..7e4ffeff 100644 --- a/examples/callbacks/notify-fetch-data.callback.ts +++ b/examples/callbacks/notify-fetch-data.callback.ts @@ -1,10 +1,9 @@ import { getInfoItem } from "examples/info-items-manager"; +import { TFetchDataCallback } from "@/types/callbacks.type"; import { sleep } from "@/utils"; -type TCallback = (data: boolean, path: string, errorHandler?: () => void) => Promise<{ finished: boolean; progress: number }>; - -export const fetchDataCallback = async (id: string, callback: TCallback) => { +export const fetchDataCallback = async (id: string, callback: Parameters[1]) => { const path = await getInfoItem(id); let finish = false; diff --git a/examples/populate.ts b/examples/populate.ts index 4857ea51..0843a9d7 100644 --- a/examples/populate.ts +++ b/examples/populate.ts @@ -6,7 +6,9 @@ import { v4 } from "uuid"; import settings from "./settings"; -const rootFile1 = join(settings.syncRootPath, v4()); +const rootFileName1 = v4(); +const rootZipFileName = `${v4()}.zip`; +const rootFile1 = join(settings.syncRootPath, rootFileName1); const rootFile2ChangeSize = join(settings.syncRootPath, `change-size-${v4()}.txt`); const rootFile3 = join(settings.syncRootPath, `${v4()}.txt`); const rootFile3Moved = join(settings.syncRootPath, `moved-${v4()}.txt`); @@ -22,6 +24,7 @@ execSync(`echo Hello, world! > ${rootFile2ChangeSize}`); execSync(`echo Hello, world! >> ${rootFile2ChangeSize}`); // Sync execSync(`echo Hello, world! > ${rootFile3}`); execSync(`type nul > ${rootFile4}`); // No sync (0 bytes) +execSync(`cd ${settings.syncRootPath} && tar -cf ${rootZipFileName} ${rootFileName1}`); // Sync execSync(`mv ${rootFile3} ${rootFile3Moved}`); // Sync execSync(`mkdir ${rootFolder1}`); // Sync execSync(`mkdir ${rootFolder2}`); // Cloud (no files inside) diff --git a/examples/register.ts b/examples/register.ts index 5ed00fbe..90c91f87 100644 --- a/examples/register.ts +++ b/examples/register.ts @@ -20,14 +20,15 @@ const handlers = { handleAdd, handleHydrate, handleDehydrate, handleChangeSize } const notify = { onTaskSuccess: async () => undefined, onTaskProcessing: async () => undefined }; const queueManager = new QueueManager(handlers, notify, settings.queuePersistPath); -drive.registerSyncRoot(settings.driveName, settings.driveVersion, settings.providerid, callbacks, settings.iconPath); -drive.connectSyncRoot(); - -try { - initInfoItems(); - drive.watchAndWait(settings.syncRootPath, queueManager, settings.watcherLogPath); -} catch (error) { - logger.error(error); - drive.disconnectSyncRoot(); - VirtualDrive.unregisterSyncRoot(settings.syncRootPath); -} +drive.registerSyncRoot(settings.driveName, settings.driveVersion, settings.providerid, callbacks, settings.iconPath).then(() => { + drive.connectSyncRoot(); + + try { + initInfoItems(); + drive.watchAndWait(settings.syncRootPath, queueManager, settings.watcherLogPath); + } catch (error) { + logger.error(error); + drive.disconnectSyncRoot(); + VirtualDrive.unregisterSyncRoot(settings.syncRootPath); + } +}); diff --git a/src/get-placeholder-states.ts b/src/get-placeholder-states.ts new file mode 100644 index 00000000..f34a66a0 --- /dev/null +++ b/src/get-placeholder-states.ts @@ -0,0 +1,30 @@ +import { readdir } from "fs/promises"; +import { join } from "path"; + +import { isFileInDevice } from "./is-file-in-device"; +import VirtualDrive from "./virtual-drive"; + +type TProps = { + self: VirtualDrive; + path: string; +}; + +export const getPlaceholderStates = async ({ self, path }: TProps) => { + const files = await readdir(path, { withFileTypes: true }); + + const promises = files.map(async (file) => { + const filePath = join(path, file.name); + + if (file.isDirectory()) { + return getPlaceholderStates({ self, path: filePath }); + } else { + const status = self.getPlaceholderState(filePath); + if (isFileInDevice(status)) { + const id = self.getFileIdentity(filePath); + self.watcher.fileInDevice.add(id); + } + } + }); + + await Promise.all(promises); +}; diff --git a/src/is-file-in-device.ts b/src/is-file-in-device.ts new file mode 100644 index 00000000..74bddaca --- /dev/null +++ b/src/is-file-in-device.ts @@ -0,0 +1,12 @@ +import { PinState, SyncState } from "./types/placeholder.type"; + +type TProps = { + syncState: SyncState; + pinState: PinState; +}; + +export const isFileInDevice = ({ syncState, pinState }: TProps) => { + const inSync = syncState === SyncState.InSync; + const isHydrated = pinState === PinState.AlwaysLocal || pinState === PinState.Unspecified; + return inSync && isHydrated; +}; diff --git a/src/types/callbacks.type.ts b/src/types/callbacks.type.ts index 9b56f31a..96b09a5d 100644 --- a/src/types/callbacks.type.ts +++ b/src/types/callbacks.type.ts @@ -1,8 +1,12 @@ export type NapiCallbackFunction = (...args: any[]) => any; +export type TFetchDataCallback = ( + id: string, + callback: (data: boolean, path: string, errorHandler?: () => void) => Promise<{ finished: boolean; progress: number }>, +) => void; + export type InputSyncCallbacks = { - // TODO: make it mandatory and add the exact type - fetchDataCallback?: NapiCallbackFunction; + fetchDataCallback: TFetchDataCallback; validateDataCallback?: NapiCallbackFunction; cancelFetchDataCallback?: NapiCallbackFunction; fetchPlaceholdersCallback?: NapiCallbackFunction; diff --git a/src/virtual-drive.ts b/src/virtual-drive.ts index 5e761d31..1228ab99 100644 --- a/src/virtual-drive.ts +++ b/src/virtual-drive.ts @@ -1,44 +1,40 @@ import path, { join, win32 } from "path"; import fs from "fs"; import { Watcher } from "./watcher/watcher"; -import { ExtraCallbacks, InputSyncCallbacks } from "./types/callbacks.type"; +import { Callbacks } from "./types/callbacks.type"; import { Status } from "./types/placeholder.type"; import { IQueueManager } from "./queue/queueManager"; import { createLogger } from "./logger"; import { Addon } from "./addon-wrapper"; +import { getPlaceholderStates } from "./get-placeholder-states"; +import winston from "winston"; const addon = new Addon(); -type Callbacks = InputSyncCallbacks & ExtraCallbacks; +const PLACEHOLDER_ATTRIBUTES = { + FILE_ATTRIBUTE_READONLY: 0x1, + FILE_ATTRIBUTE_HIDDEN: 0x2, + FOLDER_ATTRIBUTE_READONLY: 0x1, + FILE_ATTRIBUTE_NORMAL: 0x1, +}; + class VirtualDrive { - PLACEHOLDER_ATTRIBUTES: { [key: string]: number }; syncRootPath: string; - callbacks?: Callbacks; - - // private watcherBuilder: WatcherBuilder; - private watcher: Watcher; - - constructor(syncRootPath: string, loggerPath?: string) { - this.PLACEHOLDER_ATTRIBUTES = { - FILE_ATTRIBUTE_READONLY: 0x1, - FILE_ATTRIBUTE_HIDDEN: 0x2, - FOLDER_ATTRIBUTE_READONLY: 0x1, - FILE_ATTRIBUTE_NORMAL: 0x1, - }; + callbacks!: Callbacks; + watcher = new Watcher(); + logger: winston.Logger; + constructor(syncRootPath: string, loggerPath: string) { // TODO: getPlaceholderStates in the beginning this.watcher = new Watcher(); - this.syncRootPath = syncRootPath; + this.syncRootPath = this.convertToWindowsPath(syncRootPath); + loggerPath = this.convertToWindowsPath(loggerPath); this.createSyncRootFolder(); - - let pathElements = this.syncRootPath.split("\\\\"); - pathElements.pop(); - let parentPath = pathElements.join("\\\\"); - - this.addLoggerPath(loggerPath ?? parentPath); + this.addLoggerPath(loggerPath); + this.logger = createLogger(loggerPath); } addLoggerPath(logPath: string) { @@ -66,39 +62,6 @@ class VirtualDrive { return addon.getPlaceholderWithStatePending(); } - getInputSyncCallbacks(): InputSyncCallbacks { - if (this.callbacks === undefined) { - throw new Error("Callbacks are not defined"); - } - - const inputSyncCallbackKeys: (keyof InputSyncCallbacks)[] = [ - "fetchDataCallback", - "validateDataCallback", - "cancelFetchDataCallback", - "fetchPlaceholdersCallback", - "cancelFetchPlaceholdersCallback", - "notifyFileOpenCompletionCallback", - "notifyFileCloseCompletionCallback", - "notifyDehydrateCallback", - "notifyDehydrateCompletionCallback", - "notifyDeleteCallback", - "notifyDeleteCompletionCallback", - "notifyRenameCallback", - "notifyRenameCompletionCallback", - "noneCallback", - ]; - - const result: InputSyncCallbacks = {}; - - for (const key of inputSyncCallbackKeys) { - if (this.callbacks[key] !== undefined) { - result[key] = this.callbacks[key]; - } - } - - return result; - } - createSyncRootFolder() { if (!fs.existsSync(this.syncRootPath)) { fs.mkdirSync(this.syncRootPath, { recursive: true }); @@ -117,8 +80,12 @@ class VirtualDrive { return addon.deleteFileSyncRoot({ path: this.fixPath(relativePath) }); } - async connectSyncRoot(): Promise { - return addon.connectSyncRoot({ callbacks: this.getInputSyncCallbacks() }); + async connectSyncRoot() { + if (this.callbacks === undefined) { + throw new Error("Callbacks are not defined"); + } + + return addon.connectSyncRoot({ callbacks: this.callbacks }); } createPlaceholderFile( @@ -193,11 +160,18 @@ class VirtualDrive { this.callbacks = { ...callbacks, fetchDataCallback: (...args) => { - const path = args[0]; - this.watcher.fileInDevice.add(path); + const id = args[0]; + this.watcher.fileInDevice.add(id); return callbacks.fetchDataCallback?.(...args); } }; + + try { + await getPlaceholderStates({ self: this, path: this.syncRootPath}); + } catch (exc) { + this.logger.error("getPlaceholderStates", exc) + } + return addon.registerSyncRoot({ syncRootPath: this.syncRootPath, providerName, @@ -227,7 +201,7 @@ class VirtualDrive { this.watcher.queueManager = queueManager; - this.watcher.logger = createLogger(loggerPath); + this.watcher.logger = this.logger; this.watcher.syncRootPath = path; this.watcher.options = { @@ -271,7 +245,7 @@ class VirtualDrive { path.basename(fullPath), itemId, size, - this.PLACEHOLDER_ATTRIBUTES.FILE_ATTRIBUTE_NORMAL, + PLACEHOLDER_ATTRIBUTES.FILE_ATTRIBUTE_NORMAL, creationTime, lastWriteTime, Date.now(), @@ -304,7 +278,7 @@ class VirtualDrive { itemId, true, size, - this.PLACEHOLDER_ATTRIBUTES.FILE_ATTRIBUTE_NORMAL, + PLACEHOLDER_ATTRIBUTES.FILE_ATTRIBUTE_NORMAL, creationTime, lastWriteTime, Date.now(), @@ -338,7 +312,7 @@ class VirtualDrive { itemId, true, size, - this.PLACEHOLDER_ATTRIBUTES.FILE_ATTRIBUTE_NORMAL, + PLACEHOLDER_ATTRIBUTES.FILE_ATTRIBUTE_NORMAL, creationTime, lastWriteTime, Date.now(), @@ -364,7 +338,7 @@ class VirtualDrive { itemId, true, 0, - this.PLACEHOLDER_ATTRIBUTES.FILE_ATTRIBUTE_NORMAL, + PLACEHOLDER_ATTRIBUTES.FILE_ATTRIBUTE_NORMAL, Date.now(), Date.now(), Date.now(), @@ -378,7 +352,7 @@ class VirtualDrive { path.basename(fullPath), itemId, size, - this.PLACEHOLDER_ATTRIBUTES.FILE_ATTRIBUTE_NORMAL, + PLACEHOLDER_ATTRIBUTES.FILE_ATTRIBUTE_NORMAL, creationTime, lastWriteTime, Date.now(), diff --git a/src/virtual-drive.unit.test.ts b/src/virtual-drive.unit.test.ts index 6496b406..14c9c711 100644 --- a/src/virtual-drive.unit.test.ts +++ b/src/virtual-drive.unit.test.ts @@ -1,9 +1,11 @@ import fs from "fs"; import { v4 } from "uuid"; import { Mock } from "vitest"; +import { mockDeep } from "vitest-mock-extended"; import { addon } from "@/addon"; +import { Callbacks } from "./types/callbacks.type"; import VirtualDrive from "./virtual-drive"; vi.mock("fs"); @@ -138,13 +140,14 @@ describe("VirtualDrive", () => { const providerVersion = "1.0.0"; const providerId = v4(); const logoPath = "C:\\iconPath"; - const callbacks = {}; + const callbacks = mockDeep(); // Act + expect(drive.callbacks).toBe(undefined); await drive.registerSyncRoot(providerName, providerVersion, providerId, callbacks, logoPath); // Assert - expect(drive.callbacks).toBe(callbacks); + expect(drive.callbacks).not.toBe(undefined); expect(addon.registerSyncRoot).toHaveBeenCalledWith(syncRootPath, providerName, providerVersion, providerId, logoPath); }); }); From 09091c6c8917fcc5f8ccde52088544971a3e5e2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Jim=C3=A9nez=20Rivera?= Date: Thu, 6 Feb 2025 15:15:28 +0100 Subject: [PATCH 5/5] Update virtual-drive.ts --- src/virtual-drive.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/virtual-drive.ts b/src/virtual-drive.ts index 1b871d2e..aaa00790 100644 --- a/src/virtual-drive.ts +++ b/src/virtual-drive.ts @@ -8,7 +8,6 @@ import { createLogger } from "./logger"; import { Addon } from "./addon-wrapper"; import { getPlaceholderStates } from "./get-placeholder-states"; import winston from "winston"; -import winston from "winston"; const addon = new Addon();