[_] feat: add last access tracking

bin/cli/init.ts

@@ -12,6 +12,7 @@ import { MongoDBBucketEntryShardsRepository } from '../../lib/core/bucketEntrySh
 import { MongoDBUploadsRepository } from '../../lib/core/uploads/MongoDBUploadsRepository';
 import { MongoDBTokensRepository } from '../../lib/core/tokens/MongoDBTokensRepository';
 import { MongoDBContactsRepository } from '../../lib/core/contacts/MongoDBContactsRepository';
+import { MongoDBFileStateRepository } from '../../lib/core/fileState/MongoDBFileStateRepository';
 
 import { BucketEntriesUsecase } from '../../lib/core/bucketEntries/usecase';
 import { ShardsUsecase } from '../../lib/core/shards/usecase';
@@ -30,6 +31,7 @@ import { TokensRepository } from '../../lib/core/tokens/Repository';
 import { ContactsRepository } from '../../lib/core/contacts/Repository';
 import { MongoDB } from '../delete-objects/temp-shard.model';
 import { DatabaseFramesReader, DatabaseBucketEntriesReaderWithoutBucket } from '../delete-objects/ObjectStorage';
+import { FileStateRepository } from '../../lib/core/fileState/Repository';
 
 const Config = require('../../lib/config');
 
@@ -42,7 +44,7 @@ export type PrepareFunctionReturnType = {
   },
   repo: {
     bucketEntriesRepository: BucketEntriesRepository,
-    bucketEntryShardsRepository:BucketEntryShardsRepository,
+    bucketEntryShardsRepository: BucketEntryShardsRepository,
     bucketsRepository: BucketsRepository,
     usersRepository: UsersRepository,
     framesRepository: FramesRepository,
@@ -94,6 +96,7 @@ export async function prepare(): Promise<PrepareFunctionReturnType> {
   const uploadsRepository = new MongoDBUploadsRepository(models.Upload);
   const tokensRepository = new MongoDBTokensRepository(models.Token);
   const contactsRepository = new MongoDBContactsRepository(models.Contact);
+  const fileStateRepository: FileStateRepository = new MongoDBFileStateRepository(models.FileState);
 
   const shardsUsecase = new ShardsUsecase(
     mirrorsRepository,
@@ -109,15 +112,16 @@ export async function prepare(): Promise<PrepareFunctionReturnType> {
     pointersRepository,
     mirrorsRepository,
     shardsUsecase,
-    usersRepository
+    usersRepository,
+    fileStateRepository
   );
   const bucketsUsecase = new BucketsUsecase(
     bucketEntryShardsRepository,
     bucketEntriesRepository,
     mirrorsRepository,
     framesRepository,
     shardsRepository,
-    bucketsRepository, 
+    bucketsRepository,
     uploadsRepository,
     usersRepository,
     tokensRepository,

bin/utils/database.ts

@@ -7,31 +7,32 @@ const Storage = require('storj-service-storage-models') as any;
 
 const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
 
-export interface Models { 
-    BucketEntry: any, 
-    BucketEntryShard: any, 
-    Bucket: any, 
-    Shard: any, 
+export interface Models {
+    BucketEntry: any,
+    BucketEntryShard: any,
+    Bucket: any,
+    Shard: any,
     Mirror: any,
     User: any,
     Frame: any,
     Pointer: any,
     Upload: any,
     Token: any,
-    Contact: any
+    Contact: any,
+    FileState: any,
 };
 
 export async function connectToDatabase(configJSON: any, mongoURL: string): Promise<Models> {
     const config = new Config(process.env.NODE_ENV, configJSON, '') as {
-        storage: { 
-            mongoUrl: string; 
-            mongoOpts: any 
+        storage: {
+            mongoUrl: string;
+            mongoOpts: any
         },
         QUEUE_USERNAME: string;
         QUEUE_PASSWORD: string;
         QUEUE_HOST: string;
     };
-      
+
     const storage = new Storage(
         mongoURL || config.storage.mongoUrl,
         config.storage.mongoOpts,

lib/core/bucketEntries/usecase.ts

@@ -13,6 +13,7 @@ import { BucketEntry } from './BucketEntry';
 import { UsersRepository } from '../users/Repository';
 import { User } from '../users/User';
 import { Bucket } from '../buckets/Bucket';
+import { FileStateRepository } from '../fileState/Repository';
 
 export class BucketEntryVersionNotFoundError extends Error {
   constructor() {
@@ -32,10 +33,11 @@ export class BucketEntriesUsecase {
     private pointersRepository: PointersRepository,
     private mirrorsRepository: MirrorsRepository,
     private shardsUsecase: ShardsUsecase,
-    private usersRepository: UsersRepository
+    private usersRepository: UsersRepository,
+    private fileStateRepository: FileStateRepository
   ) { }
 
-  async listByBucket(bucketId: Bucket['id'], limit=20, offset=0): Promise<BucketEntry[]> {
+  async listByBucket(bucketId: Bucket['id'], limit = 20, offset = 0): Promise<BucketEntry[]> {
     const bucketEntries = await this.bucketEntriesRepository.findByBucket(bucketId, limit, offset);
 
     return bucketEntries;
@@ -56,7 +58,7 @@ export class BucketEntriesUsecase {
   async removeFileFromUser(bucketId: string, fileId: string, userId: User['uuid']) {
     const bucket = await this.bucketsRepository.findOne({ id: bucketId });
 
-    if(!bucket) {
+    if (!bucket) {
       throw new BucketNotFoundError();
     }
 
@@ -77,9 +79,9 @@ export class BucketEntriesUsecase {
     const version = bucketEntry.version;
 
     if (!version || version === 1) {
-      await this.removeFilesV1([ bucketEntry ]);
+      await this.removeFilesV1([bucketEntry]);
     } else if (version === 2) {
-      await this.removeFilesV2([ bucketEntry ]);
+      await this.removeFilesV2([bucketEntry]);
       const bucket = await this.bucketsRepository.findOne({ id: bucketEntry.bucket });
 
       if (bucket?.userId) {
@@ -98,7 +100,7 @@ export class BucketEntriesUsecase {
     const bucketEntries = await this.bucketEntriesRepository.findByIds(fileIds);
     const bucketEntriesV2 = bucketEntries.filter(b => b.version && b.version === 2);
     const bucketEntriesV1 = bucketEntries.filter(b => !b.version || b.version === 1);
-   
+
     if (bucketEntriesV1.length > 0) {
       await this.removeFilesV1(bucketEntriesV1);
     }
@@ -115,7 +117,7 @@ export class BucketEntriesUsecase {
       Object.keys(bucketsGroupedByUsers).forEach((userId) => {
         storageToModifyPerUser[userId] = 0;
       });
-      
+
       Object.keys(bucketsGroupedByUsers).forEach((userId) => {
         const buckets = bucketsGroupedByUsers[userId];
 
@@ -130,8 +132,8 @@ export class BucketEntriesUsecase {
 
         await this.usersRepository.addTotalUsedSpaceBytes(user, storageToSubstract);
       }
-    }  
-    
+    }
+
     return fileIds;
   }
 
@@ -146,7 +148,7 @@ export class BucketEntriesUsecase {
 
     if (shardsHashes.length > 0) {
       await this.shardsUsecase.deleteShardsStorageByHashes(shardsHashes);
-      await this.shardsRepository.deleteByHashes(shardsHashes);  
+      await this.shardsRepository.deleteByHashes(shardsHashes);
     }
 
     if (pointerIds.length > 0) {
@@ -156,7 +158,7 @@ export class BucketEntriesUsecase {
     if (frames.length > 0) {
       await this.framesRepository.deleteByIds(frames.map(f => f.id));
     }
-    
+
     await this.bucketEntriesRepository.deleteByIds(files.map(f => f.id));
   }
 
@@ -169,13 +171,14 @@ export class BucketEntriesUsecase {
 
     if (shards.length > 0) {
       await this.shardsUsecase.deleteShardsStorageByUuids(shards as any);
-      await this.shardsRepository.deleteByIds(shards.map(s => s.id));   
+      await this.shardsRepository.deleteByIds(shards.map(s => s.id));
     }
 
     if (bucketEntryShardsIds.length > 0) {
       await this.bucketEntryShardsRepository.deleteByIds(bucketEntryShardsIds);
     }
-    
+
     await this.bucketEntriesRepository.deleteByIds(fileIds);
+    await this.fileStateRepository.deleteByBucketEntryIds(fileIds);
   }
 }

lib/core/fileState/FileState.ts

@@ -0,0 +1,4 @@
+export interface FileState {
+  bucketEntry: string;
+  lastAccessDate: string;
+}

lib/core/fileState/MongoDBFileStateRepository.ts

@@ -0,0 +1,30 @@
+import { Model } from 'mongoose';
+import { FileState } from './FileState';
+import { FileStateRepository } from './Repository';
+
+
+export class MongoDBFileStateRepository implements FileStateRepository {
+  constructor(private model: Model<any>) { }
+
+  async findByBucketEntry(bucketEntry: FileState['bucketEntry']): Promise<FileState | null> {
+    const fileState = await this.model.findOne({ bucketEntry });
+
+    if (!fileState) {
+      return null;
+    }
+
+    return fileState;
+  }
+
+  async setLastAccessDate(bucketEntryId: FileState['bucketEntry'], accessDate = new Date()): Promise<FileState | null> {
+    return this.model.findOneAndUpdate(
+      { bucketEntry: bucketEntryId },
+      { $set: { lastAccessDate: accessDate } },
+      { new: true, upsert: true }
+    );
+  }
+
+  async deleteByBucketEntryIds(ids: FileState['bucketEntry'][]): Promise<void> {
+    await this.model.deleteMany({ bucketEntry: { $in: ids } });
+  }
+}

lib/core/fileState/Repository.ts

@@ -0,0 +1,6 @@
+import { FileState } from "./FileState";
+
+export interface FileStateRepository {
+  setLastAccessDate(bucketEntryId: FileState['bucketEntry'], accessDate?: Date): Promise<FileState | null>
+  deleteByBucketEntryIds(ids: FileState['bucketEntry'][]): Promise<void>
+}

lib/core/fileState/usecase.ts

@@ -0,0 +1,17 @@
+import { FileStateRepository } from './Repository';
+
+export class FileStateUsecase {
+
+    constructor(
+        private fileStateRepository: FileStateRepository,
+    ) { }
+
+    async updateOrSetLastAccessDate(bucketEntryId: string) {
+        const fileState = await this.fileStateRepository.setLastAccessDate(bucketEntryId);
+        return fileState
+    }
+
+    async removeFileStateByEntryId(bucketEntryId: string) {
+        await this.fileStateRepository.deleteByBucketEntryIds([bucketEntryId]);
+    }
+}

lib/server/http/index.ts

@@ -31,6 +31,8 @@ import { ShardsUsecase } from "../../core/shards/usecase";
 import { BucketEntryShardsRepository } from "../../core/bucketEntryShards/Repository";
 import { MongoDBBucketEntryShardsRepository } from "../../core/bucketEntryShards/MongoDBBucketEntryShardsRepository";
 import { Notifications } from "../notifications";
+import { FileStateRepository } from "../../core/fileState/Repository";
+import { MongoDBFileStateRepository } from "../../core/fileState/MongoDBFileStateRepository";
 
 const { authenticate } = require('storj-service-middleware');
 
@@ -44,12 +46,13 @@ interface Models {
   Contact: any;
   Shard: any;
   BucketEntryShard: any;
+  FileState: any;
 }
 
 export function bindNewRoutes(
-  app: Application, 
-  storage: { models: Models }, 
-  mailer: Mailer, 
+  app: Application,
+  storage: { models: Models },
+  mailer: Mailer,
   profile: Profile,
   log: Logger,
   networkQueue: any,
@@ -66,14 +69,15 @@ export function bindNewRoutes(
   const pointersRepository: PointersRepository = new MongoDBPointersRepository(models.Pointer);
   const contactsRepository: ContactsRepository = new MongoDBContactsRepository(models.Contact);
   const shardsRepository: ShardsRepository = new MongoDBShardsRepository(models.Shard);
+  const fileStateRepository: FileStateRepository = new MongoDBFileStateRepository(models.FileState);
 
   const mailUsecase: MailUsecase = new SendGridMailUsecase(mailer, profile);
   const eventBus = new EventBus(log, mailUsecase, notifications);
 
   const usersUsecase = new UsersUsecase(
-    usersRepository, 
+    usersRepository,
     framesRepository,
-    bucketsRepository, 
+    bucketsRepository,
     mailUsecase,
     eventBus
   );
@@ -104,7 +108,8 @@ export function bindNewRoutes(
     pointersRepository,
     mirrorsRepository,
     shardsUsecase,
-    usersRepository
+    usersRepository,
+    fileStateRepository
   );
 
   const basicAuthMiddleware = authenticate(storage);
@@ -113,8 +118,8 @@ export function bindNewRoutes(
 
   const usersController = new HTTPUsersController(usersUsecase, log);
   const gatewayController = new HTTPGatewayController(
-    gatewayUsecase, 
-    bucketEntriesUsecase, 
+    gatewayUsecase,
+    bucketEntriesUsecase,
     usersUsecase,
     log,
     eventBus

lib/server/routes/buckets.js

@@ -35,6 +35,8 @@ const { MongoDBPointersRepository } = require('../../core/pointers/MongoDBPointe
 const { ShardsUsecase } = require('../../core/shards/usecase');
 const { MongoDBTokensRepository } = require('../../core/tokens/MongoDBTokensRepository');
 const { MongoDBContactsRepository } = require('../../core/contacts/MongoDBContactsRepository');
+const { MongoDBFileStateRepository } = require('../../core/fileState/MongoDBFileStateRepository');
+const { FileStateUsecase } = require('../../core/fileState/usecase');
 
 /**
  * Handles endpoints for all bucket and file related operations
@@ -66,6 +68,7 @@ function BucketsRouter(options) {
   const pointersRepository = new MongoDBPointersRepository(this.storage.models.Pointer);
   const tokensRepository = new MongoDBTokensRepository(this.storage.models.Token);
   const contactsRepository = new MongoDBContactsRepository(this.storage.models.Contact);
+  const fileStateRepository = new MongoDBFileStateRepository(this.storage.models.FileState);
 
   this.usecase = new BucketsUsecase(
     bucketEntryShardsRepository,
@@ -95,7 +98,12 @@ function BucketsRouter(options) {
     pointersRepository,
     mirrorsRepository,
     this.shardsUseCase,
-    usersRepository
+    usersRepository,
+    fileStateRepository
+  );
+
+  this.fileStateUsecase = new FileStateUsecase(
+    fileStateRepository
   );
 }
 
@@ -1117,6 +1125,11 @@ BucketsRouter.prototype.getFileInfo = async function (req, res, next) {
       fileInfo.shards = fileShards;
     }
 
+    await this.fileStateUsecase.updateOrSetLastAccessDate(req.params.file)
+      .catch(error=>
+        log.error('getFileInfo: Error updating last access date for user %s and file %s : %s', req.user.uuid, req.params.file, error.message)
+      );
+
     return res.status(200).send(fileInfo);
   } catch (err) {
     if (err instanceof BucketEntryNotFoundError || err instanceof BucketEntryFrameNotFoundError || err instanceof BucketNotFoundError) {
@@ -1263,6 +1276,11 @@ BucketsRouter.prototype.finishUpload = async function (req, res, next) {
       auth
     );
 
+    await this.fileStateUsecase.updateOrSetLastAccessDate(bucketEntry.id)
+      .catch(error=>
+        log.error('finishUpload: Error inserting last access date for user %s and file %s : %s', req.user.uuid, bucketEntry.id, error.message)
+      );
+
     res.status(200).send(bucketEntry);
   } catch (err) {
     if (err instanceof BucketNotFoundError) {

package.json

@@ -119,10 +119,10 @@
     "redis": "^3.1.0",
     "secp256k1": "^4.0.2",
     "storj-lib": "github:internxt/core#v8.7.3-beta",
-    "storj-mongodb-adapter": "github:internxt/mongodb-adapter#10.0.1-beta",
+    "storj-mongodb-adapter": "github:internxt/mongodb-adapter#10.1.0-beta",
     "storj-service-error-types": "github:internxt/service-error-types",
     "storj-service-middleware": "github:internxt/service-middleware",
-    "storj-service-storage-models": "github:internxt/service-storage-models#11.0.1",
+    "storj-service-storage-models": "github:internxt/service-storage-models#11.1.0",
     "stripe": "^8.49.0",
     "through": "^2.3.8",
     "typescript": "^4.6.3",