diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2159a83..20dc403 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,7 +7,12 @@ permissions: jobs: test: + name: test (typesense ${{ matrix.typesense-version }}) runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + typesense-version: ['28.0', '30.1'] env: BUNDLE_JOBS: 4 @@ -16,7 +21,7 @@ jobs: services: typesense: - image: typesense/typesense:28.0 + image: typesense/typesense:${{ matrix.typesense-version }} ports: - 8108:8108 volumes: diff --git a/Gemfile b/Gemfile index 78fe3b9..ad7fd06 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,7 @@ source "http://rubygems.org" gem 'json', '>= 1.5.1' -gem "typesense", "~> 0.13.0" +gem "typesense", ">= 5.0.0", "< 6.0.0" if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'rbx' diff --git a/README.md b/README.md index 3652c4d..49c46d9 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,8 @@ Special thanks to the Algolia team for their original implementation, which prov - Environment-specific indexing - Support for faceted search and filtering - Customizable schema with predefined fields -- Multi-way and one-way synonyms support +- Multi-way and one-way synonyms support across Typesense v29 and v30+ +- Direct attachment of v30+ synonym sets and curation sets - Rake tasks for index management ## Installation @@ -91,6 +92,10 @@ class Product < ApplicationRecord } ] + # Attach existing global resources on Typesense v30+ + synonym_sets ["catalog-synonyms"] + curation_sets ["catalog-curations"] + # Symbols to index symbols_to_index ["-", "_"] @@ -103,6 +108,9 @@ class Product < ApplicationRecord end ``` +For Typesense v29, `multi_way_synonyms` and `one_way_synonyms` continue to use the legacy collection-level synonym APIs. +For Typesense v30 and newer, this gem stores those DSL synonyms in a collection-specific synonym set and attaches it automatically, while also supporting explicit `synonym_sets` / `curation_sets`. + ### Working with Relationships ```ruby diff --git a/lib/typesense-rails.rb b/lib/typesense-rails.rb index 33526d5..4fd82af 100644 --- a/lib/typesense-rails.rb +++ b/lib/typesense-rails.rb @@ -80,6 +80,8 @@ class IndexSettings OPTIONS = [ :multi_way_synonyms, :one_way_synonyms, + :synonym_sets, + :curation_sets, :predefined_fields, :fields, :default_sorting_field, @@ -271,11 +273,13 @@ def collection_name_with_timestamp(options) "#{typesense_collection_name(options)}_#{Time.now.to_i}" end - def typesense_create_collection(collection_name, settings = nil) + def typesense_create_collection(collection_name, settings = nil, existing_collection: nil) fields = settings.get_setting(:predefined_fields) || settings.get_setting(:fields) default_sorting_field = settings.get_setting(:default_sorting_field) multi_way_synonyms = settings.get_setting(:multi_way_synonyms) one_way_synonyms = settings.get_setting(:one_way_synonyms) + synonym_sets = settings.get_setting(:synonym_sets) + curation_sets = settings.get_setting(:curation_sets) symbols_to_index = settings.get_setting(:symbols_to_index) token_separators = settings.get_setting(:token_separators) enable_nested_fields = settings.get_setting(:enable_nested_fields) @@ -300,9 +304,14 @@ def typesense_create_collection(collection_name, settings = nil) ) Typesense.log(:debug, "Collection '#{collection_name}' created!") - typesense_multi_way_synonyms(collection_name, multi_way_synonyms) if multi_way_synonyms - - typesense_one_way_synonyms(collection_name, one_way_synonyms) if one_way_synonyms + apply_typesense_collection_resources( + collection_name, + multi_way_synonyms: multi_way_synonyms, + one_way_synonyms: one_way_synonyms, + synonym_sets: synonym_sets, + curation_sets: curation_sets, + existing_collection: existing_collection + ) end def typesense_upsert_alias(collection_name, alias_name) @@ -385,6 +394,103 @@ def typesense_one_way_synonyms(collection, synonyms) end end + def apply_typesense_collection_resources(collection_name, multi_way_synonyms: nil, one_way_synonyms: nil, synonym_sets: nil, curation_sets: nil, existing_collection: nil) + if typesense_server_major_version >= 30 + apply_v30_collection_resources( + collection_name, + multi_way_synonyms: multi_way_synonyms, + one_way_synonyms: one_way_synonyms, + synonym_sets: synonym_sets, + curation_sets: curation_sets, + existing_collection: existing_collection + ) + else + ensure_v30_resource_support!(synonym_sets, curation_sets) + typesense_multi_way_synonyms(collection_name, multi_way_synonyms) if multi_way_synonyms + typesense_one_way_synonyms(collection_name, one_way_synonyms) if one_way_synonyms + end + end + + def apply_v30_collection_resources(collection_name, multi_way_synonyms: nil, one_way_synonyms: nil, synonym_sets: nil, curation_sets: nil, existing_collection: nil) + inline_synonyms_present = multi_way_synonyms || one_way_synonyms + attached_synonym_sets = Array(existing_collection && existing_collection["synonym_sets"]) + Array(synonym_sets) + attached_curation_sets = Array(existing_collection && existing_collection["curation_sets"]) + Array(curation_sets) + + if inline_synonyms_present + synonym_set_name = default_synonym_set_name(collection_name) + ensure_synonym_set_exists(synonym_set_name) + upsert_synonym_set_items(synonym_set_name, multi_way_synonyms, one_way_synonyms) + attached_synonym_sets << synonym_set_name + end + + collection_patch = {} + collection_patch["synonym_sets"] = attached_synonym_sets.uniq if attached_synonym_sets.any? + collection_patch["curation_sets"] = attached_curation_sets.uniq if attached_curation_sets.any? + + return if collection_patch.empty? + + typesense_client.collections[collection_name].update(collection_patch) + end + + def ensure_v30_resource_support!(synonym_sets, curation_sets) + unsupported = [] + unsupported << "synonym_sets" if synonym_sets + unsupported << "curation_sets" if curation_sets + return if unsupported.empty? + + raise Typesense::BadConfiguration, "#{unsupported.join(' and ')} require Typesense v30.0 or newer" + end + + def default_synonym_set_name(collection_name) + "#{collection_name}_synonyms_index" + end + + def ensure_synonym_set_exists(synonym_set_name) + typesense_client.synonym_sets.upsert(synonym_set_name, { "items" => [] }) + end + + def upsert_synonym_set_items(synonym_set_name, multi_way_synonyms, one_way_synonyms) + items = [] + + Array(multi_way_synonyms).each do |synonym_hash| + synonym_hash.each do |synonym_name, synonym| + items << { "id" => synonym_name, "synonyms" => synonym } + end + end + + Array(one_way_synonyms).each do |synonym_hash| + synonym_hash.each do |synonym_name, synonym| + items << synonym.merge("id" => synonym_name) + end + end + + typesense_client.synonym_sets.upsert(synonym_set_name, { "items" => items }) + end + + def typesense_server_major_version + Typesense.server_major_version + end + + def reset_typesense_server_major_version! + Typesense.reset_server_version_cache! + end + + def typesense_debug_info + Typesense.debug_info + end + + def typesense_collection_resources(collection_name) + return {} if typesense_server_major_version < 30 + + collection = typesense_get_collection(collection_name) + return {} unless collection + + { + "synonym_sets" => Array(collection["synonym_sets"]), + "curation_sets" => Array(collection["curation_sets"]) + } + end + def typesense(options = {}, &block) self.typesense_settings = IndexSettings.new(options, &block) self.typesense_options = { type: typesense_full_const_get(model_name.to_s) }.merge(options) # :per_page => typesense_settings.get_setting(:hitsPerPage) || 10, :page => 1 @@ -528,8 +634,10 @@ def typesense_reindex(batch_size = Typesense::IndexSettings::DEFAULT_BATCH_SIZE) typesense_configurations.each do |options, settings| next if typesense_indexing_disabled?(options) + existing_collection_resources = {} begin master_index = typesense_ensure_init(options, settings, false) + existing_collection_resources = typesense_collection_resources(master_index[:alias_name]) delete_collection(master_index[:alias_name]) rescue ArgumentError @typesense_indexes[settings] = { collection_name: "", alias_name: typesense_collection_name(options) } @@ -542,7 +650,7 @@ def typesense_reindex(batch_size = Typesense::IndexSettings::DEFAULT_BATCH_SIZE) tmp_options.delete(:per_environment) # already included in the temporary index_name tmp_settings = settings.dup - create_collection(src_index_name, settings) + create_collection(src_index_name, settings, existing_collection: existing_collection_resources) typesense_find_in_batches(batch_size) do |group| if typesense_conditional_index?(options) diff --git a/lib/typesense/config.rb b/lib/typesense/config.rb index 45c1eae..cef41ed 100644 --- a/lib/typesense/config.rb +++ b/lib/typesense/config.rb @@ -16,6 +16,8 @@ def configuration=(configuration) @@pagination_backend = configuration[:pagination_backend] if configuration.key?(:pagination_backend) @@log_level = configuration[:log_level] if configuration.key?(:log_level) @@configuration = configuration + @client = nil + reset_server_version_cache! end def pagination_backend @@ -66,5 +68,21 @@ def client def setup_client @client = Typesense::Client.new(@@configuration) end + + def server_major_version + @server_major_version ||= begin + version = debug_info.fetch("version", "") + version == "nightly" ? 30 : version.split(".").first.to_i + end + end + + def debug_info + @debug_info ||= client.debug.retrieve + end + + def reset_server_version_cache! + @server_major_version = nil + @debug_info = nil + end end end diff --git a/lib/typesense/version.rb b/lib/typesense/version.rb index 323fba4..1c65311 100644 --- a/lib/typesense/version.rb +++ b/lib/typesense/version.rb @@ -1,3 +1,3 @@ module Typesense - GEM_VERSION = "1.0.0.rc5" + GEM_VERSION = "1.0.0.rc6" end diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index 9d026dc..7b6a7aa 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -173,6 +173,20 @@ def published? class Camera < Product end +class V30ResourceProduct < ActiveRecord::Base + self.table_name = "products" + self.inheritance_column = :_type_disabled + + include Typesense + + typesense auto_index: false, + index_name: safe_index_name("v30_resource_products") do + attribute :name + synonym_sets [safe_index_name("shared-product-synonyms")] + curation_sets [safe_index_name("shared-product-curations")] + end +end + class Color < ActiveRecord::Base include Typesense attr_accessor :not_indexed @@ -1049,6 +1063,199 @@ class SerializedObject < ActiveRecord::Base end end +describe "Typesense version compatibility" do + let(:settings) { Product.typesense_settings } + + it "uses collection-level synonyms on v29" do + allow(Product).to receive(:typesense_server_major_version).and_return(29) + expect(Product).to receive(:typesense_multi_way_synonyms) + .with("products_v29", settings.get_setting(:multi_way_synonyms)) + expect(Product).to receive(:typesense_one_way_synonyms) + .with("products_v29", settings.get_setting(:one_way_synonyms)) + + Product.apply_typesense_collection_resources( + "products_v29", + multi_way_synonyms: settings.get_setting(:multi_way_synonyms), + one_way_synonyms: settings.get_setting(:one_way_synonyms) + ) + end + + it "creates and attaches synonym sets on v30+" do + collection_resource = double("collection_resource") + collections_resource = double("collections_resource") + synonym_sets_resource = double("synonym_sets_resource") + client = double("typesense_client", collections: collections_resource, synonym_sets: synonym_sets_resource) + + allow(Product).to receive(:typesense_server_major_version).and_return(30) + allow(Product).to receive(:typesense_client).and_return(client) + expect(collections_resource).to receive(:[]).with("products_v30").and_return(collection_resource) + expect(synonym_sets_resource).to receive(:upsert).with("products_v30_synonyms_index", { "items" => [] }).ordered + expect(synonym_sets_resource).to receive(:upsert).with( + "products_v30_synonyms_index", + { + "items" => [ + { "id" => "phone-synonym", "synonyms" => %w[galaxy samsung samsung_electronics] }, + { "id" => "smart-phone-synonym", "root" => "smartphone", + "synonyms" => %w[nokia samsung motorola android] } + ] + } + ).ordered + expect(collection_resource).to receive(:update) do |payload| + expect(payload["synonym_sets"]).to match_array(%w[existing-shared shared-synonyms products_v30_synonyms_index]) + expect(payload["curation_sets"]).to match_array(%w[existing-curation shared-curations]) + end + + Product.apply_typesense_collection_resources( + "products_v30", + multi_way_synonyms: settings.get_setting(:multi_way_synonyms), + one_way_synonyms: settings.get_setting(:one_way_synonyms), + synonym_sets: ["shared-synonyms"], + curation_sets: ["shared-curations"], + existing_collection: { + "synonym_sets" => ["existing-shared"], + "curation_sets" => ["existing-curation"] + } + ) + end + + it "rejects v30-only collection resources on v29" do + allow(Product).to receive(:typesense_server_major_version).and_return(29) + + expect do + Product.apply_typesense_collection_resources( + "products_v29", + synonym_sets: ["shared-synonyms"] + ) + end.to raise_error(Typesense::BadConfiguration, /synonym_sets require Typesense v30.0 or newer/) + end +end + +describe "Typesense v30 resource integration" do + before do + skip("SynonymSets and CurationSets are only supported in Typesense v30+") unless typesense_v30_or_above? + end + + after do + begin + Typesense.client.synonym_sets.retrieve.each do |set| + Typesense.client.synonym_sets[set["name"]].delete if set["name"].include?(SAFE_INDEX_PREFIX) + end + rescue StandardError + nil + end + + begin + Typesense.client.curation_sets.retrieve.each do |set| + Typesense.client.curation_sets[set["name"]].delete if set["name"].include?(SAFE_INDEX_PREFIX) + end + rescue StandardError + nil + end + + begin + V30ResourceProduct.clear_index! + rescue StandardError + nil + end + + begin + Product.clear_index! + rescue StandardError + nil + end + end + + it "attaches generated synonym sets for inline synonym DSL" do + begin + Product.clear_index! + rescue StandardError + nil + end + collection_obj = Product.typesense_index + + collection = Typesense.client.collections[collection_obj[:collection_name]].retrieve + generated_synonym_set = "#{collection_obj[:collection_name]}_synonyms_index" + + expect(collection["synonym_sets"]).to include(generated_synonym_set) + + synonym_set = Typesense.client.synonym_sets[generated_synonym_set].retrieve + item_ids = synonym_set.fetch("items").map { |item| item["id"] } + + expect(item_ids).to include("phone-synonym", "smart-phone-synonym") + end + + it "attaches explicit synonym sets and curation sets to the collection" do + synonym_set_name = safe_index_name("shared-product-synonyms") + curation_set_name = safe_index_name("shared-product-curations") + + Typesense.client.synonym_sets.upsert( + synonym_set_name, + { + "items" => [ + { "id" => "shared-phone", "root" => "phone", "synonyms" => %w[handset mobile] } + ] + } + ) + Typesense.client.curation_sets.upsert( + curation_set_name, + { + "items" => [ + { + "id" => "promote-phone", + "rule" => { "query" => "phone", "match" => "exact" }, + "includes" => [{ "id" => "1", "position" => 1 }], + "excludes" => [], + "filter_curated_hits" => false, + "remove_matched_tokens" => false, + "stop_processing" => true + } + ] + } + ) + + V30ResourceProduct.typesense_index + + collection = Typesense.client.collections[V30ResourceProduct.collection_name].retrieve + + expect(collection["synonym_sets"]).to include(synonym_set_name) + expect(collection["curation_sets"]).to include(curation_set_name) + end + + it "preserves attached synonym sets and curation sets through reindex" do + synonym_set_name = safe_index_name("shared-product-synonyms") + curation_set_name = safe_index_name("shared-product-curations") + + Typesense.client.synonym_sets.upsert( + synonym_set_name, + { "items" => [{ "id" => "shared-phone", "root" => "phone", "synonyms" => %w[handset mobile] }] } + ) + Typesense.client.curation_sets.upsert( + curation_set_name, + { + "items" => [ + { + "id" => "promote-phone", + "rule" => { "query" => "phone", "match" => "exact" }, + "includes" => [{ "id" => "1", "position" => 1 }], + "excludes" => [], + "filter_curated_hits" => false, + "remove_matched_tokens" => false, + "stop_processing" => true + } + ] + } + ) + + V30ResourceProduct.typesense_index + V30ResourceProduct.reindex + + collection = Typesense.client.collections[V30ResourceProduct.collection_name].retrieve + + expect(collection["synonym_sets"]).to include(synonym_set_name) + expect(collection["curation_sets"]).to include(curation_set_name) + end +end + describe "Kaminari" do before(:all) do require "kaminari" diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index c82fded..cd633d2 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -52,3 +52,17 @@ def safe_index_list list = Typesense.client.collections.retrieve() list = list.select { |index| index["name"].include?(SAFE_INDEX_PREFIX) } end + +def typesense_version + Typesense.client.debug.retrieve["version"] +rescue StandardError + nil +end + +def typesense_v30_or_above? + version = typesense_version + return false unless version + return true if version == "nightly" + + version.to_s[/\d+/].to_i >= 30 +end diff --git a/typesense-rails.gemspec b/typesense-rails.gemspec index 4835cbc..1cc641d 100644 --- a/typesense-rails.gemspec +++ b/typesense-rails.gemspec @@ -52,7 +52,7 @@ Gem::Specification.new do |s| if Gem::Version.new(Gem::VERSION) >= Gem::Version.new("1.2.0") s.add_runtime_dependency(%q, [">= 1.5.1"]) - s.add_runtime_dependency(%q, [">= 0.13.0"]) + s.add_runtime_dependency(%q, [">= 5.0.0", "< 6.0.0"]) s.add_development_dependency(%q, [">= 2.3.15"]) s.add_development_dependency(%q, [">= 0"]) s.add_development_dependency(%q, [">= 0"]) @@ -60,10 +60,10 @@ Gem::Specification.new do |s| s.add_development_dependency "rdoc" else s.add_dependency(%q, [">= 1.5.1"]) - s.add_dependency(%q, [">= 0.13.0"]) + s.add_dependency(%q, [">= 5.0.0", "< 6.0.0"]) end else s.add_dependency(%q, [">= 1.5.1"]) - s.add_dependency(%q, [">= 0.13.0"]) + s.add_dependency(%q, [">= 5.0.0", "< 6.0.0"]) end end