add IDHash and use it after a pHash match

Author: ryoga-chan

Gemfile

@@ -48,6 +48,7 @@ gem 'image_processing', '~> 1.13', require: %w[ ruby-vips image_processing ]
 ENV['PHASHION_USE_GITHUB_SRC'] ?
   gem('phashion', '~> 1.2', github: 'ryoga-chan/phashion') :
   gem('phashion', '~> 1.2', source: 'https://rubygems.pkg.github.com/ryoga-chan')
+gem 'dhash-vips', '~> 0.2'       # dHash and IDHash for image similarity -- https://github.com/Nakilon/dhash-vips
 
 #gem "solid_cache"               # database-backed adapter for Rails.cache
 #gem "solid_queue"               # database-backed adapter for Active Job

app/controllers/doujinshi_controller.rb

@@ -481,8 +481,8 @@ def search_cover
     end
 
     if cover_hash
-      CoverMatchingJob.perform_now cover_hash
-      return redirect_to(hash: cover_hash, format: params[:format])
+      CoverMatchingJob.perform_now cover_hash[:phash], cover_hash[:idhash]
+      return redirect_to(hash: cover_hash[:idhash], format: params[:format])
     end
 
     fname = File.join(Setting['dir.sorting'], 'cover-search.yml').to_s

app/controllers/home_controller.rb

@@ -97,8 +97,8 @@ def shared_content_receiver
       # single file: search by cover image
       when params[:images]&.one?
         if cover_hash = CoverMatchingJob.hash_image(params[:images].first.path)
-          CoverMatchingJob.perform_now cover_hash
-          redirect_to search_cover_doujinshi_path(hash: cover_hash)
+          CoverMatchingJob.perform_now cover_hash[:phash], cover_hash[:idhash]
+          redirect_to search_cover_doujinshi_path(hash: cover_hash[:idhash])
         else
           flash.now[:alert] = "fingerprinting image error"
         end

app/controllers/process_controller.rb

@@ -232,7 +232,7 @@ def delete_archive_cwd
 
     ProcessArchiveDecompressJob.rm_entry folder: @dname
 
-    CoverMatchingJob.rm_results_file @info[:cover_hash]
+    CoverMatchingJob.rm_results_file @info[:cover_hash][:idhash]
 
     msg   = params[:archive_too] == 'true' ? "archive and folder deleted:" : "folder deleted for"
     title = @info[:relative_path].one?     ? @info[:relative_path].first   : @info[:title]
@@ -424,15 +424,15 @@ def edit
           cover_path = ProcessArchiveDecompressJob.cover_path @dname, @info
           @info[:cover_hash] = CoverMatchingJob.hash_image cover_path
           File.open(File.join(@dname, 'info.yml'), 'w'){|f| f.puts @info.to_yaml }
-          CoverMatchingJob.perform_now @info[:cover_hash]
+          CoverMatchingJob.perform_now @info[:cover_hash][:phash], @info[:cover_hash][:idhash]
         end
 
         @dupes, @dupes_deleted = [], []
 
         # search dupes by cover similarity
         # check matching status/results
         if @info[:cover_hash].present? && !@info[:cover_results].is_a?(Hash)
-          cover_matching = CoverMatchingJob.results @info[:cover_hash]
+          cover_matching = CoverMatchingJob.results @info[:cover_hash][:idhash]
           if cover_matching.is_a?(Hash)
             @info[:cover_results        ] = cover_matching[:results        ]
             @info[:cover_results_deleted] = cover_matching[:results_deleted]

app/jobs/cover_matching_job.rb

@@ -3,7 +3,8 @@ class CoverMatchingJob < ApplicationJob
   # http://www.mikeperham.com/2010/05/21/detecting-duplicate-images-with-phashion/
   # << Our testing showed that 15 bits is a good value to start with, it detected
   #    all duplicates with a minimum of false positives >>
-  MAX_HAMMING_DISTANCE = 13 # = less than ~20% (13/64) different bits
+  MAX_HAMMING_DISTANCE_PHASH  = 13 # = less than ~20% (13/64) different bits
+  MAX_HAMMING_DISTANCE_IDHASH = 25 # empirical threshold
 
   queue_as :search
 
@@ -16,30 +17,26 @@ def self.hash_image_buffer(image_data, hash_only: false)
       width:  ProcessArchiveCompressJob::THUMB_WIDTH,
       height: ProcessArchiveCompressJob::THUMB_HEIGHT
 
-    # calculate its pHash
-    fname = File.join(Setting['dir.sorting'], "#{fkey}.webp").to_s
-    File.open(fname, 'wb'){|f| f.write thumb[:image].webpsave_buffer }
-    phash = Kernel.suppress_output{ '%016x' % Phashion::Image.new(fname).fingerprint }
-    FileUtils.rm_f fname # remove temp image
+    h = thumb[:image].fingerprints
 
-    return phash if hash_only
+    return h if hash_only
 
-    { phash: phash, landscape: thumb[:landscape],
-      image: Base64.encode64(thumb[:image].webpsave_buffer).chomp }
+    h.merge! landscape: thumb[:landscape],
+             image: Base64.encode64(thumb[:image].webpsave_buffer).chomp
   end # self.hash_image_buffer
 
   def self.hash_image(image_path)
     result = hash_image_buffer File.binread(image_path)
 
     # create metadata + embedded image
-    File.atomic_write(File.join(Setting['dir.sorting'], "#{result[:phash]}.yml").to_s) do |f|
+    File.atomic_write(File.join(Setting['dir.sorting'], "#{result[:idhash]}.yml").to_s) do |f|
       f.puts(result.merge({
         status:     :comparing,
         started_at: Time.now,
       }).to_yaml)
     end
 
-    result[:phash]
+    result
   end # self.hash_image
 
   # return final results and delete temp file after matching completed
@@ -58,14 +55,15 @@ def self.rm_results_file(image_hash)
     FileUtils.rm_f File.join(Setting['dir.sorting'], "#{image_hash}.yml").to_s
   end # self.rm_results_file
 
-  def self.find(model, phash, max_distance: MAX_HAMMING_DISTANCE, from_id: 0)
+  def self.find(model, phash, idhash, from_id: 0)
     # https://stackoverflow.com/questions/2281580/is-there-any-way-to-convert-an-integer-3-in-decimal-form-to-its-binary-equival/2310694#2310694
     # https://stackoverflow.com/questions/49601249/string-to-binary-and-back-using-pure-sqlite
     # GENERATE TERMS: puts (0..63).map{|i| "(x>>#{i.to_s.rjust 2}&1)" }.each_slice(5).map{|s| s.join(' + ') }.join(" +\n")
     # NOTE: a XOR b = (a|b)-(a&b) = (~(a&b))&(a|b)
     query = <<~SQL
       SELECT
           id
+        , cover_idhash
         , (x    &1) + (x>> 1&1) + (x>> 2&1) + (x>> 3&1) + (x>> 4&1) +
           (x>> 5&1) + (x>> 6&1) + (x>> 7&1) + (x>> 8&1) + (x>> 9&1) +
           (x>>10&1) + (x>>11&1) + (x>>12&1) + (x>>13&1) + (x>>14&1) +
@@ -81,38 +79,61 @@ def self.find(model, phash, max_distance: MAX_HAMMING_DISTANCE, from_id: 0)
           (x>>60&1) + (x>>61&1) + (x>>62&1) + (x>>63&1) AS hamming_distance
       FROM (
         SELECT id
-             , (~(a&b))&(a|b) AS x  -- x = a XOR b
+             , cover_idhash
+             , (~(a&b))&(a|b) AS x    -- x = a XOR b
         FROM (
           SELECT id
-               , 0x#{phash}   AS a
-               , cover_phash  AS b
+               , #{phash.to_i} AS a
+               , cover_phash   AS b
+               , cover_idhash
           FROM #{model.table_name}
-          WHERE cover_phash IS NOT NULL
+          WHERE cover_phash  IS NOT NULL
+            AND cover_idhash IS NOT NULL
             AND id > #{from_id.to_i}
         )
       )
-      WHERE hamming_distance < #{max_distance.to_i}
+      WHERE hamming_distance < #{MAX_HAMMING_DISTANCE_PHASH}
       ORDER BY hamming_distance
       LIMIT 10
     SQL
 
-    model.
-      find_by_sql(query).
-      inject({}){|h, d| h.merge d.id => ((1 - d.hamming_distance.to_f / 64) * 100).round }
+    idhash = idhash.to_i(16) if idhash.is_a?(String)
+
+    model.find_by_sql(query).inject({}) do |h, d|
+      # compute the more accurate IDHash distance
+      DHashVips::IDHash.distance3(d.cover_idhash.to_i(16), idhash) < MAX_HAMMING_DISTANCE_IDHASH \
+        ? h.merge!(d.id => ((1 - d.hamming_distance.to_f / 64) * 100).round)
+        : h
+    end
   end # self.find
 
+  # phash* is a signed integer, idhash is a hexadecimal string
+  # returns nil or the similarity percentage of the matching
+  def self.similarity(phash1, phash2, idhash1, idhash2)
+    d1 = Phashion.hamming_distance [phash1].pack('q').unpack('Q').first,
+                                   [phash2].pack('q').unpack('Q').first
+
+    if d1 < MAX_HAMMING_DISTANCE_PHASH
+      # compute the more accurate IDHash distance
+      d2 = DHashVips::IDHash.distance3 idhash1.to_i(16), idhash2.to_i(16)
+      return ((1 - d1.to_f / 64) * 100).round if d2 < MAX_HAMMING_DISTANCE_IDHASH
+    end
+
+    nil
+  end # self.similarity
+
   # read image data from temp file and do a matching against all saved doujinshi
   # by computing hamming distance between pHashes
-  def perform(image_hash, max_distance: MAX_HAMMING_DISTANCE)
-    fname = File.join(Setting['dir.sorting'], "#{image_hash}.yml").to_s
+  def perform(phash, idhash)
+    fname = File.join(Setting['dir.sorting'], "#{idhash}.yml").to_s
     info  = YAML.unsafe_load_file fname
 
     # write results
     info.merge! \
       status:          :completed,
       finished_at:     Time.now,
-      results:         CoverMatchingJob.find(Doujin       , image_hash, max_distance: max_distance),
-      results_deleted: CoverMatchingJob.find(DeletedDoujin, image_hash, max_distance: max_distance)
+      results:         CoverMatchingJob.find(Doujin       , phash, idhash),
+      results_deleted: CoverMatchingJob.find(DeletedDoujin, phash, idhash)
 
     File.atomic_write(fname){|f| f.puts info.to_yaml }
   end # perform

app/jobs/process_archive_compress_job.rb

@@ -167,7 +167,7 @@ def perform(src_dir)
         File.atomic_write(File.join(src_dir, 'finalize.perc')){|f| f.write perc.round(2) }
       end # Doujin.transaction
 
-      CoverMatchingJob.rm_results_file info[:cover_hash]
+      CoverMatchingJob.rm_results_file info[:cover_hash][:idhash]
     rescue
       info[:finalize_error    ] = $!.to_s
       info[:finalize_backtrace] = $!.backtrace

app/jobs/process_batch_inspect_job.rb

@@ -21,8 +21,7 @@ def perform(hash)
         info[:files][name] = image_entries.map &:name
 
         if cover = image_entries.first # extract cover image
-          info[:thumbs][name] = CoverMatchingJob.
-            hash_image_buffer cover.get_input_stream.read
+          info[:thumbs][name] = CoverMatchingJob.hash_image_buffer cover.get_input_stream.read
         end
       end
 
@@ -35,8 +34,11 @@ def perform(hash)
       matching_found = false
 
       groups.keys.each do |g|
-        hd = Phashion.hamming_distance info[:thumbs][f][:phash].to_i(16), info[:thumbs][g][:phash].to_i(16)
-        if hd < CoverMatchingJob::MAX_HAMMING_DISTANCE
+        match_found = CoverMatchingJob.similarity \
+          info[:thumbs][f][:phash ], info[:thumbs][g][:phash ],
+          info[:thumbs][f][:idhash], info[:thumbs][g][:idhash]
+
+        if match_found
           groups[g] << f
           matching_found = true
           break

app/jobs/process_batch_job.rb

@@ -126,8 +126,8 @@ def add_error(results, key, msg, opts)
       # run cover image hash matching
       cover_path = ProcessArchiveDecompressJob.cover_path dname, info
       info[:cover_hash] = CoverMatchingJob.hash_image cover_path
-      CoverMatchingJob.perform_now info[:cover_hash]
-      cover_matching = CoverMatchingJob.results info[:cover_hash]
+      CoverMatchingJob.perform_now info[:cover_hash][:phash], info[:cover_hash][:idhash]
+      cover_matching = CoverMatchingJob.results info[:cover_hash][:idhash]
       info[:cover_results] = cover_matching[:results]
       info[:cover_status ] = cover_matching[:status]
       File.open(info_fname, 'w'){|f| f.puts info.to_yaml }

app/jobs/process_index_group_job.rb

@@ -24,21 +24,21 @@ def perform(*args)
     rel = ProcessableDoujin.where.not(cover_phash: nil).where("id >= ?", last_id.to_i)
     num_entries = rel.count
     print_info_freq = [num_entries / 100 + 1, 10].max
-    rel.select("id, PRINTF('%016x', cover_phash) AS cover_phash_hex").find_each.with_index do |pd, i|
+    rel.select("id, cover_phash, cover_idhash").find_each.with_index do |pd, i|
       break if Rails.env.development? && i >= DEVEL_LIMIT
 
       ris = self.class.progress_update(step: (i+1), steps: num_entries, msg: 'comparing covers')
       Rails.logger.info(ris) if i % print_info_freq == 0
 
       ProcessableDoujin.transaction do
-        CoverMatchingJob.find(ProcessableDoujin, pd.cover_phash_hex, from_id: (last_id ? 0 : pd.id)).each do |id, perc|
+        CoverMatchingJob.find(ProcessableDoujin, pd.cover_phash, pd.cover_idhash, from_id: (last_id ? 0 : pd.id)).each do |id, perc|
           next if pd.id == id
           ids = [pd.id, id].sort
           pdd = ProcessableDoujinDupe.find_or_initialize_by(pd_parent_id: ids[0], pd_child_id: ids[1])
           pdd.update likeness: perc
         end # each dupe
 
-        CoverMatchingJob.find(Doujin, pd.cover_phash_hex).each do |id, perc|
+        CoverMatchingJob.find(Doujin, pd.cover_phash, pd.cover_idhash).each do |id, perc|
           pdd = ProcessableDoujinDupe.find_or_initialize_by(pd_parent_id: pd.id, doujin_id: id)
           pdd.update likeness: perc
         end # each dupe

app/jobs/process_index_refresh_job.rb

@@ -74,35 +74,30 @@ def self.rm_entry(path_or_id, track: false, rm_zip: false, merged: false, doujin
 
     if File.exist?(zip_path)
       if track
-        cover_hash = nil
+        dd = DeletedDoujin.new
 
         # count images and other files
-        file_counters = { num_images: 0, num_files: 0 }
         Zip::File.open(zip_path) do |zip|
-          zip.entries.sort_by_method(:name).each do |e|
-            next unless e.file?
+          entries = zip.split_entries(sort: true)
 
-            is_image = e.name.is_image_filename?
+          dd.num_images = entries[:images].size
+          dd.num_files  = entries[:files ].size
 
-            # generate phash for the first image file
-            if cover_hash.nil? && is_image
-              cover_hash = CoverMatchingJob.hash_image_buffer(e.get_input_stream.read)[:phash]
-            end
-
-            file_counters[is_image ? :num_images : :num_files] += 1
+          # generate phash for the first image file
+          if cover = entries[:images].first
+            h = CoverMatchingJob.hash_image_buffer(cover.get_input_stream.read, hash_only: true)
+            dd.cover_phash  = h[:phash]
+            dd.cover_idhash = h[:idhash]
           end
         end
 
         # track deletion
-        name = pd.name.tr(File::SEPARATOR, ' ')
-        dd = DeletedDoujin.create! file_counters.merge({
-          name:             name,
-          name_kakasi:      name.to_romaji,
-          size:             File.size(zip_path),
-          merged:           merged,
-          doujin_id:        doujin_id,
-        })
-        dd.cover_fingerprint! cover_hash if cover_hash.present?
+        name = pd.name.tr File::SEPARATOR, ' '
+        dd.update! name:        name,
+                   name_kakasi: name.to_romaji,
+                   size:        File.size(zip_path),
+                   merged:      merged,
+                   doujin_id:   doujin_id
       end # if track
 
       # remove file from disk

app/models/deleted_doujin.rb

@@ -36,9 +36,8 @@ def self.search(terms)
     rel.order(Arel.sql "COALESCE(NULLIF(alt_name_kakasi, ''), NULLIF(name_kakasi, ''))")
   end # self.search
 
-  def cover_fingerprint!(fp)
+  def cover_fingerprint!(h)
     raise :record_not_persisted unless persisted?
-    self.class.connection.execute \
-      %Q(UPDATE #{self.class.table_name} SET cover_phash = 0x#{fp} WHERE id = #{id})
+    update! cover_phash: h[:phash], cover_idhash: h[:idhash]
   end # cover_fingerprint!
 end

app/models/doujin.rb

@@ -231,10 +231,8 @@ def cover_fingerprint = CoverMatchingJob.hash_image_buffer(File.binread(thumb_di
 
   def cover_fingerprint!
     raise :record_not_persisted unless persisted?
-    fp = cover_fingerprint
-    self.class.connection.execute \
-      %Q(UPDATE #{self.class.table_name} SET cover_phash = 0x#{fp} WHERE id = #{id})
-    fp
+    h = cover_fingerprint
+    update! cover_phash: h[:phash], cover_idhash: h[:idhash]
   end # cover_fingerprint!
 
   # next page numer for `/doujinshi/ID/read` action
@@ -308,7 +306,7 @@ def save_deletion_data
     return unless @save_deletion_data
 
     fname = file_dl_name omit_ext: true
-    attrs = attributes.slice *%w[ size num_images num_files cover_phash ]
+    attrs = attributes.slice *%w[ size num_images num_files cover_phash cover_idhash ]
 
     DeletedDoujin.create attrs.merge({
       doujin_id:        id,

app/models/processable_doujin.rb

@@ -76,27 +76,24 @@ def cover_fingerprint
     fname = file_path full: true
     return unless File.exist?(fname)
 
-    phash = nil
+    h = {}
 
     Zip::File.open(fname) do |zip|
       zip_images = zip.image_entries(sort: true)
+      @images = zip_images.size
 
-      if zip_images.any?
-        phash = CoverMatchingJob.hash_image_buffer zip_images.first.get_input_stream.read, hash_only: true
-        @images = zip_images.size
+      if cover = zip_images.first
+        h = CoverMatchingJob.hash_image_buffer cover.get_input_stream.read, hash_only: true
       end
     end
 
-    phash
+    h
   end # cover_fingerprint
 
   def cover_fingerprint!
     raise :record_not_persisted unless persisted?
-    fp = cover_fingerprint
-    self.class.connection.execute \
-      %Q(UPDATE #{self.class.table_name} SET cover_phash = 0x#{fp} WHERE id = #{id})
-    update images: @images
-    fp
+    h = cover_fingerprint
+    update! cover_phash: h[:phash], cover_idhash: h[:idhash], images: @images
   end # cover_fingerprint!
 
   def generate_preview

config/initializers/logging.eval

@@ -7,7 +7,9 @@ if ENV['PUMA_DAEMON'] && Rails.env.production?
   logger_args = [File.join(Dir.tmpdir, "ruby:djmngr-server.log"), 2]
 end
 
-unless Rails.const_defined?(:Console) || defined?(Rake)
+unless Rails.const_defined?(:Console)    || # bin/rails console
+       Rails.const_defined?(:Generators) || # bin/rails generate
+       defined?(Rake)                       # bin/rails x:y:z
   puts "* Logging to #{logger_args.first.inspect}"
 end