Merge pull request #2773 from mikiher/fix-library-files-inconsistencies

Fix library files inconsistencies

server/Logger.js

@@ -93,7 +93,7 @@ class Logger {
 
     // Save log to file
     if (level >= this.logLevel) {
-      await this.logManager.logToFile(logObj)
+      await this.logManager?.logToFile(logObj)
     }
   }
 

server/models/LibraryItem.js

@@ -1,4 +1,4 @@
-const { DataTypes, Model, WhereOptions } = require('sequelize')
+const { DataTypes, Model } = require('sequelize')
 const Logger = require('../Logger')
 const oldLibraryItem = require('../objects/LibraryItem')
 const libraryFilters = require('../utils/queries/libraryFilters')
@@ -116,7 +116,7 @@ class LibraryItem extends Model {
 
   /**
    * Currently unused because this is too slow and uses too much mem
-   * @param {[WhereOptions]} where
+   * @param {import('sequelize').WhereOptions} [where]
    * @returns {Array<objects.LibraryItem>} old library items
    */
   static async getAllOldLibraryItems(where = null) {
@@ -773,12 +773,14 @@ class LibraryItem extends Model {
 
   /**
    * 
-   * @param {WhereOptions} where 
+   * @param {import('sequelize').WhereOptions} where 
+   * @param {import('sequelize').BindOrReplacements} replacements
    * @returns {Object} oldLibraryItem
    */
-  static async findOneOld(where) {
+  static async findOneOld(where, replacements = {}) {
     const libraryItem = await this.findOne({
       where,
+      replacements,
       include: [
         {
           model: this.sequelize.models.book,

server/scanner/BookScanner.js

@@ -20,6 +20,7 @@ const LibraryScan = require("./LibraryScan")
 const OpfFileScanner = require('./OpfFileScanner')
 const NfoFileScanner = require('./NfoFileScanner')
 const AbsMetadataFileScanner = require('./AbsMetadataFileScanner')
+const EBookFile = require("../objects/files/EBookFile")
 
 /**
  * Metadata for books pulled from files
@@ -84,7 +85,7 @@ class BookScanner {
 
       // Update audio files that were modified
       if (libraryItemData.audioLibraryFilesModified.length) {
-        let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesModified)
+        let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesModified.map(lf => lf.new))
         media.audioFiles = media.audioFiles.map((audioFileObj) => {
           let matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.metadata.path === audioFileObj.metadata.path)
           if (!matchedScannedAudioFile) {
@@ -138,11 +139,25 @@ class BookScanner {
     }
 
     // Check if cover was removed
-    if (media.coverPath && !libraryItemData.imageLibraryFiles.some(lf => lf.metadata.path === media.coverPath) && !(await fsExtra.pathExists(media.coverPath))) {
+    if (media.coverPath && libraryItemData.imageLibraryFilesRemoved.some(lf => lf.metadata.path === media.coverPath) && !(await fsExtra.pathExists(media.coverPath))) {
       media.coverPath = null
       hasMediaChanges = true
     }
 
+    // Update cover if it was modified
+    if (media.coverPath && libraryItemData.imageLibraryFilesModified.length) {
+      let coverMatch = libraryItemData.imageLibraryFilesModified.find(iFile => iFile.old.metadata.path === media.coverPath)
+      if (coverMatch) {
+        const coverPath = coverMatch.new.metadata.path
+        if (coverPath !== media.coverPath) {
+          libraryScan.addLog(LogLevel.DEBUG, `Updating book cover "${media.coverPath}" => "${coverPath}" for book "${media.title}"`)
+          media.coverPath = coverPath
+          media.changed('coverPath', true)
+          hasMediaChanges = true
+        }
+      }
+    }
+
     // Check if cover is not set and image files were found
     if (!media.coverPath && libraryItemData.imageLibraryFiles.length) {
       // Prefer using a cover image with the name "cover" otherwise use the first image
@@ -157,6 +172,19 @@ class BookScanner {
       hasMediaChanges = true
     }
 
+    // Update ebook if it was modified
+    if (media.ebookFile && libraryItemData.ebookLibraryFilesModified.length) {
+      let ebookMatch = libraryItemData.ebookLibraryFilesModified.find(eFile => eFile.old.metadata.path === media.ebookFile.metadata.path)
+      if (ebookMatch) {
+        const ebookFile = new EBookFile(ebookMatch.new)
+        ebookFile.ebookFormat = ebookFile.metadata.ext.slice(1).toLowerCase()
+        libraryScan.addLog(LogLevel.DEBUG, `Updating book ebook file "${media.ebookFile.metadata.path}" => "${ebookFile.metadata.path}" for book "${media.title}"`)
+        media.ebookFile = ebookFile.toJSON()
+        media.changed('ebookFile', true)
+        hasMediaChanges = true
+      }
+    }
+
     // Check if ebook is not set and ebooks were found
     if (!media.ebookFile && !librarySettings.audiobooksOnly && libraryItemData.ebookLibraryFiles.length) {
       // Prefer to use an epub ebook then fallback to the first ebook found

server/scanner/LibraryItemScanData.js

@@ -4,6 +4,12 @@ const LibraryItem = require('../models/LibraryItem')
 const globals = require('../utils/globals')
 
 class LibraryItemScanData {
+  /**
+   * @typedef LibraryFileModifiedObject
+   * @property {LibraryItem.LibraryFileObject} old
+   * @property {LibraryItem.LibraryFileObject} new
+   */
+
   constructor(data) {
     /** @type {string} */
     this.libraryFolderId = data.libraryFolderId
@@ -39,7 +45,7 @@ class LibraryItemScanData {
     this.libraryFilesRemoved = []
     /** @type {LibraryItem.LibraryFileObject[]} */
     this.libraryFilesAdded = []
-    /** @type {LibraryItem.LibraryFileObject[]} */
+    /** @type {LibraryFileModifiedObject[]} */
     this.libraryFilesModified = []
   }
 
@@ -77,9 +83,9 @@ class LibraryItemScanData {
     return (this.audioLibraryFilesRemoved.length + this.audioLibraryFilesAdded.length + this.audioLibraryFilesModified.length) > 0
   }
 
-  /** @type {LibraryItem.LibraryFileObject[]} */
+  /** @type {LibraryFileModifiedObject[]} */
   get audioLibraryFilesModified() {
-    return this.libraryFilesModified.filter(lf => globals.SupportedAudioTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
+    return this.libraryFilesModified.filter(lf => globals.SupportedAudioTypes.includes(lf.old.metadata.ext?.slice(1).toLowerCase() || ''))
   }
 
   /** @type {LibraryItem.LibraryFileObject[]} */
@@ -97,12 +103,42 @@ class LibraryItemScanData {
     return this.libraryFiles.filter(lf => globals.SupportedAudioTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
   }
 
+  /** @type {LibraryFileModifiedObject[]} */
+  get imageLibraryFilesModified() {
+    return this.libraryFilesModified.filter(lf => globals.SupportedImageTypes.includes(lf.old.metadata.ext?.slice(1).toLowerCase() || ''))
+  }
+
+  /** @type {LibraryItem.LibraryFileObject[]} */
+  get imageLibraryFilesRemoved() {
+    return this.libraryFilesRemoved.filter(lf => globals.SupportedImageTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
+  }
+
+  /** @type {LibraryItem.LibraryFileObject[]} */
+  get imageLibraryFilesAdded() {
+    return this.libraryFilesAdded.filter(lf => globals.SupportedImageTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
+  }
+
   /** @type {LibraryItem.LibraryFileObject[]} */
   get imageLibraryFiles() {
     return this.libraryFiles.filter(lf => globals.SupportedImageTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
   }
 
-  /** @type {import('../objects/files/LibraryFile')[]} */
+  /** @type {LibraryFileModifiedObject[]} */
+  get ebookLibraryFilesModified() {
+    return this.libraryFilesModified.filter(lf => globals.SupportedEbookTypes.includes(lf.old.metadata.ext?.slice(1).toLowerCase() || ''))
+  }
+
+  /** @type {LibraryItem.LibraryFileObject[]} */
+  get ebookLibraryFilesRemoved() {
+    return this.libraryFilesRemoved.filter(lf => globals.SupportedEbookTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
+  }
+
+  /** @type {LibraryItem.LibraryFileObject[]} */
+  get ebookLibraryFilesAdded() {
+    return this.libraryFilesAdded.filter(lf => globals.SupportedEbookTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
+  }
+
+  /** @type {LibraryItem.LibraryFileObject[]} */
   get ebookLibraryFiles() {
     return this.libraryFiles.filter(lf => globals.SupportedEbookTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
   }
@@ -153,7 +189,7 @@ class LibraryItemScanData {
         existingLibraryItem[key] = this[key]
         this.hasChanges = true
 
-        if (key === 'relPath') {
+        if (key === 'relPath' || key === 'path') {
           this.hasPathChange = true
         }
       }
@@ -202,8 +238,9 @@ class LibraryItemScanData {
         this.hasChanges = true
       } else {
         libraryFilesAdded = libraryFilesAdded.filter(lf => lf !== matchingLibraryFile)
+        let existingLibraryFileBefore = structuredClone(existingLibraryFile)
         if (this.compareUpdateLibraryFile(existingLibraryItem.path, existingLibraryFile, matchingLibraryFile, libraryScan)) {
-          this.libraryFilesModified.push(existingLibraryFile)
+          this.libraryFilesModified.push({old: existingLibraryFileBefore, new: existingLibraryFile})
           this.hasChanges = true
         }
       }

server/scanner/LibraryItemScanner.js

@@ -21,22 +21,23 @@ class LibraryItemScanner {
    * Scan single library item
    * 
    * @param {string} libraryItemId 
-   * @param {{relPath:string, path:string}} [renamedPaths] used by watcher when item folder was renamed
+   * @param {{relPath:string, path:string}} [updateLibraryItemDetails] used by watcher when item folder was renamed
    * @returns {number} ScanResult
    */
-  async scanLibraryItem(libraryItemId, renamedPaths = null) {
+  async scanLibraryItem(libraryItemId, updateLibraryItemDetails = null) {
     // TODO: Add task manager
     const libraryItem = await Database.libraryItemModel.findByPk(libraryItemId)
     if (!libraryItem) {
       Logger.error(`[LibraryItemScanner] Library item not found "${libraryItemId}"`)
       return ScanResult.NOTHING
     }
 
+    const libraryFolderId = updateLibraryItemDetails?.libraryFolderId || libraryItem.libraryFolderId
     const library = await Database.libraryModel.findByPk(libraryItem.libraryId, {
       include: {
         model: Database.libraryFolderModel,
         where: {
-          id: libraryItem.libraryFolderId
+          id: libraryFolderId
         }
       }
     })
@@ -51,11 +52,11 @@ class LibraryItemScanner {
 
     const scanLogger = new ScanLogger()
     scanLogger.verbose = true
-    scanLogger.setData('libraryItem', renamedPaths?.relPath || libraryItem.relPath)
+    scanLogger.setData('libraryItem', updateLibraryItemDetails?.relPath || libraryItem.relPath)
 
-    const libraryItemPath = renamedPaths?.path || fileUtils.filePathToPOSIX(libraryItem.path)
+    const libraryItemPath = updateLibraryItemDetails?.path || fileUtils.filePathToPOSIX(libraryItem.path)
     const folder = library.libraryFolders[0]
-    const libraryItemScanData = await this.getLibraryItemScanData(libraryItemPath, library, folder, false)
+    const libraryItemScanData = await this.getLibraryItemScanData(libraryItemPath, library, folder, updateLibraryItemDetails?.isFile || false)
 
     let libraryItemDataUpdated = await libraryItemScanData.checkLibraryItemData(libraryItem, scanLogger)
 

server/scanner/LibraryScanner.js

@@ -154,7 +154,11 @@ class LibraryScanner {
       let libraryItemData = libraryItemDataFound.find(lid => lid.path === existingLibraryItem.path)
       if (!libraryItemData) {
         // Fallback to finding matching library item with matching inode value
-        libraryItemData = libraryItemDataFound.find(lid => lid.ino === existingLibraryItem.ino)
+        libraryItemData = libraryItemDataFound.find(lid =>
+          ItemToItemInoMatch(lid, existingLibraryItem) ||
+          ItemToFileInoMatch(lid, existingLibraryItem) ||
+          ItemToFileInoMatch(existingLibraryItem, lid)
+        )
         if (libraryItemData) {
           libraryScan.addLog(LogLevel.INFO, `Library item with path "${existingLibraryItem.path}" was not found, but library item inode "${existingLibraryItem.ino}" was found at path "${libraryItemData.path}"`)
         }
@@ -522,22 +526,25 @@ class LibraryScanner {
 
       // Check if book dir group is already an item
       let existingLibraryItem = await Database.libraryItemModel.findOneOld({
+        libraryId: library.id,
         path: potentialChildDirs
       })
 
-      let renamedPaths = {}
+      let updatedLibraryItemDetails = {}
       if (!existingLibraryItem) {
-        const dirIno = await fileUtils.getIno(fullPath)
-        existingLibraryItem = await Database.libraryItemModel.findOneOld({
-          ino: dirIno
-        })
+        const isSingleMedia = isSingleMediaFile(fileUpdateGroup, itemDir)
+        existingLibraryItem =
+          await findLibraryItemByItemToItemInoMatch(library.id, fullPath) ||
+          await findLibraryItemByItemToFileInoMatch(library.id, fullPath, isSingleMedia) ||
+          await findLibraryItemByFileToItemInoMatch(library.id, fullPath, isSingleMedia, fileUpdateGroup[itemDir])
         if (existingLibraryItem) {
-          Logger.debug(`[LibraryScanner] scanFolderUpdates: Library item found by inode value=${dirIno}. "${existingLibraryItem.relPath} => ${itemDir}"`)
           // Update library item paths for scan
           existingLibraryItem.path = fullPath
           existingLibraryItem.relPath = itemDir
-          renamedPaths.path = fullPath
-          renamedPaths.relPath = itemDir
+          updatedLibraryItemDetails.path = fullPath
+          updatedLibraryItemDetails.relPath = itemDir
+          updatedLibraryItemDetails.libraryFolderId = folder.id
+          updatedLibraryItemDetails.isFile = isSingleMedia
         }
       }
       if (existingLibraryItem) {
@@ -554,10 +561,9 @@ class LibraryScanner {
             continue
           }
         }
-
         // Scan library item for updates
         Logger.debug(`[LibraryScanner] Folder update for relative path "${itemDir}" is in library item "${existingLibraryItem.media.metadata.title}" - scan for updates`)
-        itemGroupingResults[itemDir] = await LibraryItemScanner.scanLibraryItem(existingLibraryItem.id, renamedPaths)
+        itemGroupingResults[itemDir] = await LibraryItemScanner.scanLibraryItem(existingLibraryItem.id, updatedLibraryItemDetails)
         continue
       } else if (library.settings.audiobooksOnly && !hasAudioFiles(fileUpdateGroup, itemDir)) {
         Logger.debug(`[LibraryScanner] Folder update for relative path "${itemDir}" has no audio files`)
@@ -594,6 +600,14 @@ class LibraryScanner {
 }
 module.exports = new LibraryScanner()
 
+function ItemToFileInoMatch(libraryItem1, libraryItem2) {
+  return libraryItem1.isFile && libraryItem2.libraryFiles.some(lf => lf.ino === libraryItem1.ino)
+}
+
+function ItemToItemInoMatch(libraryItem1, libraryItem2) {
+  return libraryItem1.ino === libraryItem2.ino
+}
+
 function hasAudioFiles(fileUpdateGroup, itemDir) {
   return isSingleMediaFile(fileUpdateGroup, itemDir) ?
     scanUtils.checkFilepathIsAudioFile(fileUpdateGroup[itemDir]) :
@@ -603,3 +617,55 @@ function hasAudioFiles(fileUpdateGroup, itemDir) {
 function isSingleMediaFile(fileUpdateGroup, itemDir) {
   return itemDir === fileUpdateGroup[itemDir]
 }
+
+async function findLibraryItemByItemToItemInoMatch(libraryId, fullPath) {
+  const ino = await fileUtils.getIno(fullPath)
+  if (!ino) return null
+  const existingLibraryItem = await Database.libraryItemModel.findOneOld({
+    libraryId: libraryId,
+    ino: ino
+  })
+  if (existingLibraryItem)
+    Logger.debug(`[LibraryScanner] Found library item with matching inode "${ino}" at path "${existingLibraryItem.path}"`)
+  return existingLibraryItem
+}
+
+async function findLibraryItemByItemToFileInoMatch(libraryId, fullPath, isSingleMedia) {
+  if (!isSingleMedia) return null
+  // check if it was moved from another folder by comparing the ino to the library files
+  const ino = await fileUtils.getIno(fullPath)
+  if (!ino) return null
+  const existingLibraryItem = await Database.libraryItemModel.findOneOld([
+    {
+      libraryId: libraryId
+    },
+    sequelize.where(sequelize.literal('(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)'), {
+      [sequelize.Op.gt]: 0
+    })
+  ], {
+    inode: ino
+  })
+  if (existingLibraryItem)
+    Logger.debug(`[LibraryScanner] Found library item with a library file matching inode "${ino}" at path "${existingLibraryItem.path}"`)
+  return existingLibraryItem
+}
+
+async function findLibraryItemByFileToItemInoMatch(libraryId, fullPath, isSingleMedia, itemFiles) {
+  if (isSingleMedia) return null
+  // check if it was moved from the root folder by comparing the ino to the ino of the scanned files
+  let itemFileInos = []
+  for (const itemFile of itemFiles) {
+    const ino = await fileUtils.getIno(Path.posix.join(fullPath, itemFile))
+    if (ino) itemFileInos.push(ino)
+  }
+  if (!itemFileInos.length) return null
+  const existingLibraryItem = await Database.libraryItemModel.findOneOld({
+    libraryId: libraryId,
+    ino: {
+      [sequelize.Op.in]: itemFileInos
+    }
+  })
+  if (existingLibraryItem)
+    Logger.debug(`[LibraryScanner] Found library item with inode matching one of "${itemFileInos.join(',')}" at path "${existingLibraryItem.path}"`)
+  return existingLibraryItem
+}