Merge branch 'release/v5.6.0'

Author: pat

.github/workflows/ci.yml

@@ -4,14 +4,13 @@ on: [push, pull_request]
 
 jobs:
   sphinx:
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-22.04
 
     strategy:
       fail-fast: false
       matrix:
-        os: [ 'ubuntu-18.04' ]
         ruby: [ '2.7', '3.0', '3.1', '3.2' ]
-        rails: [ '5_0', '5_1', '5_2', '6_0', '6_1', '7_0' ]
+        rails: [ '5_0', '5_1', '5_2', '6_0', '6_1', '7_0', '7_1' ]
         database: [ 'mysql2', 'postgresql' ]
         sphinx_version: [ '2.2.11', '3.4.1' ]
         sphinx_engine: [ 'sphinx' ]
@@ -58,7 +57,7 @@ jobs:
         options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/test
         with:
           ruby-version: ${{ matrix.ruby }}
@@ -69,15 +68,15 @@ jobs:
         timeout-minutes: 12
 
   manticore:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     strategy:
       fail-fast: false
       matrix:
         ruby: [ '2.7', '3.0', '3.1', '3.2' ]
-        rails: [ '5_0', '5_1', '5_2', '6_0', '6_1', '7_0' ]
+        rails: [ '5_0', '5_1', '5_2', '6_0', '6_1', '7_0', '7_1' ]
         database: [ 'mysql2', 'postgresql' ]
-        sphinx_version: [ '3.5.4', '4.0.2' ]
+        sphinx_version: [ '4.0.2', '6.0.0' ]
         sphinx_engine: [ 'manticore' ]
         exclude:
           - ruby: '3.0'

Appraisals

@@ -45,3 +45,9 @@ appraise 'rails_7_0' do
   gem 'mysql2', '~> 0.5.0', :platform => :ruby
   gem 'pg',     '~> 1.0',   :platform => :ruby
 end if RUBY_PLATFORM != 'java' && RUBY_VERSION.to_f >= 2.7
+
+appraise 'rails_7_1' do
+  gem 'rails',  '~> 7.1.0'
+  gem 'mysql2', '~> 0.5.0', :platform => :ruby
+  gem 'pg',     '~> 1.0',   :platform => :ruby
+end if RUBY_PLATFORM != 'java' && RUBY_VERSION.to_f >= 2.7

CHANGELOG.markdown

@@ -2,6 +2,25 @@
 
 All notable changes to this project (at least, from v3.0.0 onwards) are documented in this file.
 
+## 5.6.0 - 2024-07-07
+
+### Added
+
+* Support for Manticore 6.0 ([#1242](https://github.com/pat/thinking-sphinx/pull/1242))
+* `sphinx`-prefixed search methods, in case the standard `search` is overridden from something unrelated. ([#1265](https://github.com/pat/thinking-sphinx/pull/1265))
+* `none` / `search_none` scopes that can be chained to searches and will return no results.
+* Added `ThinkingSphinx::Processor#sync` to synchronise updates/deletions based on a real-time index's scope, by @akostadinov in [@1258](https://github.com/pat/thinking-sphinx/pull/1258).
+
+### Changed
+
+* Improved Rails 7.1 support, by @jdelstrother in [#1252](https://github.com/pat/thinking-sphinx/pull/1252).
+
+### Fixed
+
+* Handle both SQL and RT indices correctly for inheritance column checks, by @akostadinov in [#1249](https://github.com/pat/thinking-sphinx/pull/1249).
+* Ensure tests and CI work with recent Manticore versions, by @jdelstrother in [#1263](https://github.com/pat/thinking-sphinx/pull/1263).
+* Use `rm -rf` to delete test and temporary directories (instead of `rm -r`).
+
 ## 5.5.1 - 2022-12-31
 
 [Release Notes](https://github.com/pat/thinking-sphinx/releases/tag/v5.5.1)

README.textile

@@ -1,6 +1,6 @@
 h1. Thinking Sphinx
 
-Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v5.5.1.
+Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v5.6.0.
 
 h2. Upgrading
 
@@ -31,7 +31,7 @@ The current release of Thinking Sphinx works with the following versions of its
 |_. Library |_. Minimum |_. Tested Against |
 | Ruby | v2.4 | v2.4, v2.5, v2.6, v2.7, v3.0, v3.1, v3.2 |
 | Sphinx | v2.2.11 | v2.2.11, v3.4.1 |
-| Manticore | v2.8 | v3.5, v4.0 |
+| Manticore | v2.8 | v4.0, v6.0 |
 | ActiveRecord | v4.2 | v4.2..v7.0 |
 
 It _might_ work with older versions of Ruby, but it's highly recommended to update to a supported release.
@@ -42,7 +42,7 @@ h3. Sphinx or Manticore
 
 If you're using Sphinx, v2.2.11 is recommended even though it's quite old, as it works well with PostgreSQL databases (but if you're using MySQL - or real-time indices - then v3.3.1 should also be fine).
 
-If you're opting for Manticore instead, v2.8 or newer works, but v3 or newer is recommended as that's what is actively tested against.
+If you're opting for Manticore instead, v2.8 or newer works, but v4 or newer is recommended as that's what is actively tested against. The v4.2 and 5.0 releases had bugs with facet searching, but that's been fixed in Manticore v6.0.
 
 h3. Rails and ActiveRecord
 
@@ -81,4 +81,4 @@ You can then run the unit tests with @rake spec:unit@, the acceptance tests with
 
 h2. Licence
 
-Copyright (c) 2007-2022, Thinking Sphinx is developed and maintained by Pat Allan, and is released under the open MIT Licence. Many thanks to "all who have contributed patches":https://github.com/pat/thinking-sphinx/contributors.
+Copyright (c) 2007-2024, Thinking Sphinx is developed and maintained by Pat Allan, and is released under the open MIT Licence. Many thanks to "all who have contributed patches":https://github.com/pat/thinking-sphinx/contributors.

bin/loadsphinx

@@ -70,15 +70,26 @@ load_manticore () {
       url="https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/focal/main/binary-amd64/manticore_3.5.4-210107-f70faec5_amd64.deb";;
     4.0.2)
       url="https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/focal/main/binary-amd64/manticore_4.0.2-210921-af497f245_amd64.deb";;
+    4.2.0)
+      url="https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/focal/main/binary-amd64/manticore_4.2.0-211223-15e927b28_amd64.deb";;
+    6.0.0)
+      url="skipped";;
     *)
       echo "No Manticore version $version available"
       exit 1;;
   esac
 
-  sudo apt-get install default-libmysqlclient-dev
-  curl --location $url -o manticore.deb
-  sudo dpkg -i ./manticore.deb
-  sudo apt-get install -f
+  if [ "$version" == "6.0.0" ]; then
+    curl --location https://repo.manticoresearch.com/manticore-repo.noarch.deb -o repo.deb
+    sudo dpkg -i repo.deb
+    sudo apt update
+    sudo apt install manticore
+  else
+    sudo apt-get install default-libmysqlclient-dev
+    curl --location $url -o manticore.deb
+    sudo dpkg -i ./manticore.deb
+    sudo apt-get install -f
+  fi
 }
 
 if [ "$engine" == "sphinx" ]; then

lib/thinking_sphinx.rb

@@ -34,6 +34,10 @@ def self.search_for_ids(query = '', options = {})
     ThinkingSphinx::Search::Merger.new(search).merge! nil, :ids_only => true
   end
 
+  def self.none
+    ThinkingSphinx::Search.new nil, :none => true
+  end
+
   def self.before_index_hooks
     @before_index_hooks
   end

lib/thinking_sphinx/active_record/base.rb

@@ -4,6 +4,21 @@ module ThinkingSphinx::ActiveRecord::Base
   extend ActiveSupport::Concern
 
   included do
+    # Avoid method collisions for public Thinking Sphinx methods added to all
+    # ActiveRecord models. The `sphinx_`-prefixed versions will always exist,
+    # and the non-prefixed versions will be added if a method of that name
+    # doesn't already exist.
+    #
+    # If a method is overwritten later by something else, that's also fine - the
+    # prefixed versions will still be there.
+    class_module = ThinkingSphinx::ActiveRecord::Base::ClassMethods
+    class_module.public_instance_methods.each do |method_name|
+      short_method = method_name.to_s.delete_prefix("sphinx_").to_sym
+      next if methods.include?(short_method)
+
+      define_singleton_method(short_method, method(method_name))
+    end
+
     if ActiveRecord::VERSION::STRING.to_i >= 5
       [
         ::ActiveRecord::Reflection::HasManyReflection,
@@ -25,24 +40,28 @@ def extensions
   end
 
   module ClassMethods
-    def facets(query = nil, options = {})
+    def sphinx_facets(query = nil, options = {})
       merge_search ThinkingSphinx.facets, query, options
     end
 
-    def search(query = nil, options = {})
+    def sphinx_search(query = nil, options = {})
       merge_search ThinkingSphinx.search, query, options
     end
 
-    def search_count(query = nil, options = {})
+    def sphinx_search_count(query = nil, options = {})
       search_for_ids(query, options).total_entries
     end
 
-    def search_for_ids(query = nil, options = {})
+    def sphinx_search_for_ids(query = nil, options = {})
       ThinkingSphinx::Search::Merger.new(
         search(query, options)
       ).merge! nil, :ids_only => true
     end
 
+    def sphinx_search_none
+      merge_search ThinkingSphinx.search, nil, none: true
+    end
+
     private
 
     def default_sphinx_scope?

lib/thinking_sphinx/active_record/filter_reflection.rb

@@ -2,7 +2,7 @@
 
 class ThinkingSphinx::ActiveRecord::FilterReflection
   ReflectionGenerator = case ActiveRecord::VERSION::STRING.to_f
-  when 5.2..7.0
+  when 5.2..7.1
     ThinkingSphinx::ActiveRecord::Depolymorph::OverriddenReflection
   when 4.1..5.1
     ThinkingSphinx::ActiveRecord::Depolymorph::AssociationReflection

lib/thinking_sphinx/active_record/log_subscriber.rb

@@ -2,24 +2,36 @@
 
 class ThinkingSphinx::ActiveRecord::LogSubscriber < ActiveSupport::LogSubscriber
   def guard(event)
-    identifier = color 'Sphinx', GREEN, true
+    identifier = colored_text "Sphinx"
     warn "  #{identifier}  #{event.payload[:guard]}"
   end
 
   def message(event)
-    identifier = color 'Sphinx', GREEN, true
+    identifier = colored_text "Sphinx"
     debug "  #{identifier}  #{event.payload[:message]}"
   end
 
   def query(event)
-    identifier = color('Sphinx Query (%.1fms)' % event.duration, GREEN, true)
+    identifier = colored_text("Sphinx Query (%.1fms)" % event.duration)
     debug "  #{identifier}  #{event.payload[:query]}"
   end
 
   def caution(event)
-    identifier = color 'Sphinx', GREEN, true
+    identifier = colored_text "Sphinx"
     warn "  #{identifier}  #{event.payload[:caution]}"
   end
+
+  private
+
+  if Rails.gem_version >= Gem::Version.new("7.1.0")
+    def colored_text(text)
+      color text, GREEN, bold: true
+    end
+  else
+    def colored_text(text)
+      color text, GREEN, true
+    end
+  end
 end
 
 ThinkingSphinx::ActiveRecord::LogSubscriber.attach_to :thinking_sphinx

lib/thinking_sphinx/commands/clear_real_time.rb

@@ -7,7 +7,7 @@ def call
       Dir["#{index.path}.*"].each { |path| FileUtils.rm path }
     end
 
-    FileUtils.rm_r(binlog_path) if File.exist?(binlog_path)
+    FileUtils.rm_rf(binlog_path) if File.exist?(binlog_path)
   end
 
   private

lib/thinking_sphinx/commands/clear_sql.rb

@@ -7,7 +7,7 @@ def call
       Dir["#{index.path}.*"].each { |path| FileUtils.rm path }
     end
 
-    FileUtils.rm_r Dir["#{configuration.indices_location}/ts-*.tmp"]
+    FileUtils.rm_rf Dir["#{configuration.indices_location}/ts-*.tmp"]
   end
 
   private

lib/thinking_sphinx/configuration/minimum_fields.rb

@@ -18,19 +18,19 @@ def reconcile
   attr_reader :indices
 
   def field_collections
-    plain_indices_without_inheritance.collect(&:sources).flatten +
-    indices_of_type('rt')
-  end
-
-  def indices_of_type(type)
-    indices.select { |index| index.type == type }
+    indices_without_inheritance_of_type('plain').collect(&:sources).flatten +
+      indices_without_inheritance_of_type('rt')
   end
 
   def inheritance_columns?(index)
     index.model.table_exists? && index.model.column_names.include?(index.model.inheritance_column)
   end
 
-  def plain_indices_without_inheritance
-    indices_of_type('plain').reject(&method(:inheritance_columns?))
+  def indices_without_inheritance_of_type(type)
+    indices_without_inheritance.select { |index| index.type == type }
+  end
+
+  def indices_without_inheritance
+    indices.reject(&method(:inheritance_columns?))
   end
 end

lib/thinking_sphinx/masks/scopes_mask.rb

@@ -26,6 +26,12 @@ def search_for_ids(query = nil, options = {})
     search query, options.merge(:ids_only => true)
   end
 
+  def none
+    ThinkingSphinx::Search::Merger.new(@search).merge! nil, :none => true
+  end
+
+  alias_method :search_none, :none
+
   private
 
   def apply_scope(scope, *args)

lib/thinking_sphinx/processor.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 class ThinkingSphinx::Processor
+  # @param instance [ActiveRecord::Base] an ActiveRecord object
+  # @param model [Class] the ActiveRecord model of the instance
+  # @param id [Integer] the instance indices primary key (might be different from model primary key)
   def initialize(instance: nil, model: nil, id: nil)
     raise ArgumentError if instance.nil? && (model.nil? || id.nil?)
 
@@ -12,16 +15,27 @@ def initialize(instance: nil, model: nil, id: nil)
   def delete
     return if instance&.new_record?
 
-    indices.each { |index|
-      ThinkingSphinx::Deletion.perform(
-        index, id || instance.public_send(index.primary_key)
-      )
-    }
+    indices.each { |index| perform_deletion(index) }
   end
 
+  # Will insert instance into all matching indices
   def upsert
     real_time_indices.each do |index|
-      ThinkingSphinx::RealTime::Transcriber.new(index).copy loaded_instance
+      found = loaded_instance(index)
+      ThinkingSphinx::RealTime::Transcriber.new(index).copy found if found
+    end
+  end
+
+  # Will upsert or delete instance into all matching indices based on index scope
+  def sync
+    real_time_indices.each do |index|
+      found = find_in(index)
+
+      if found
+        ThinkingSphinx::RealTime::Transcriber.new(index).copy found
+      else
+        ThinkingSphinx::Deletion.perform(index, index_id(index))
+      end
     end
   end
 
@@ -35,11 +49,23 @@ def indices
     ).to_a
   end
 
-  def loaded_instance
-    @loaded_instance ||= instance || model.find(id)
+  def find_in(index)
+    index.scope.find_by(index.primary_key => index_id(index))
+  end
+
+  def loaded_instance(index)
+    instance || find_in(index)
   end
 
   def real_time_indices
     indices.select { |index| index.is_a? ThinkingSphinx::RealTime::Index }
   end
+
+  def perform_deletion(index)
+    ThinkingSphinx::Deletion.perform(index, index_id(index))
+  end
+
+  def index_id(index)
+    id || instance.public_send(index.primary_key)
+  end
 end