From 8d5e9892db703a5b402791c57cbb928cd9ad6221 Mon Sep 17 00:00:00 2001 From: "David A. Cuadrado" Date: Wed, 28 Sep 2011 12:52:07 -0500 Subject: [PATCH 01/13] add mongoid adapter Signed-off-by: David A. Cuadrado --- .../client/model_adapters/mongoid_adapter.rb | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 lib/xapit/client/model_adapters/mongoid_adapter.rb diff --git a/lib/xapit/client/model_adapters/mongoid_adapter.rb b/lib/xapit/client/model_adapters/mongoid_adapter.rb new file mode 100644 index 0000000..3fd75b5 --- /dev/null +++ b/lib/xapit/client/model_adapters/mongoid_adapter.rb @@ -0,0 +1,27 @@ +module Xapit + module Client + class MongoidAdapter < AbstractModelAdapter + def self.for_class?(model_class) + model_class.kind_of?(Mongoid::Document) + end + + def setup + @model_class.after_create do |member| + member.class.xapit_index_builder.add_document(member) if Xapit.config[:enabled] + end + @model_class.after_update do |member| + member.class.xapit_index_builder.update_document(member) if Xapit.config[:enabled] + end + @model_class.after_destroy do |member| + member.class.xapit_index_builder.remove_document(member) if Xapit.config[:enabled] + end + end + + def index_all + @model_class.all.each do |member| + member.class.xapit_index_builder.add_document(member) + end + end + end + end +end From 4344a393742fb4a2673e8cdfee6f89f9cc008767 Mon Sep 17 00:00:00 2001 From: "David A. Cuadrado" Date: Wed, 28 Sep 2011 16:44:45 -0500 Subject: [PATCH 02/13] check whether active record or mongoid is included Signed-off-by: David A. Cuadrado --- lib/xapit/client/model_adapters/active_record_adapter.rb | 2 +- lib/xapit/client/model_adapters/mongoid_adapter.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/xapit/client/model_adapters/active_record_adapter.rb b/lib/xapit/client/model_adapters/active_record_adapter.rb index 4b63772..da2420a 100644 --- a/lib/xapit/client/model_adapters/active_record_adapter.rb +++ b/lib/xapit/client/model_adapters/active_record_adapter.rb @@ -2,7 +2,7 @@ module Xapit module Client class ActiveRecordAdapter < AbstractModelAdapter def self.for_class?(model_class) - model_class <= ActiveRecord::Base + defined?(ActiveRecord::Base) && model_class <= ActiveRecord::Base end def setup diff --git a/lib/xapit/client/model_adapters/mongoid_adapter.rb b/lib/xapit/client/model_adapters/mongoid_adapter.rb index 3fd75b5..cc323ad 100644 --- a/lib/xapit/client/model_adapters/mongoid_adapter.rb +++ b/lib/xapit/client/model_adapters/mongoid_adapter.rb @@ -2,7 +2,7 @@ module Xapit module Client class MongoidAdapter < AbstractModelAdapter def self.for_class?(model_class) - model_class.kind_of?(Mongoid::Document) + defined?(Mongoid::Document) && model_class.kind_of?(Mongoid::Document) end def setup From da8647efbab1ec01f479457bde673ba33f7810f3 Mon Sep 17 00:00:00 2001 From: "David A. Cuadrado" Date: Wed, 28 Sep 2011 16:49:25 -0500 Subject: [PATCH 03/13] initialize mongoid documents correctly Signed-off-by: David A. Cuadrado --- lib/xapit/client/model_adapters/mongoid_adapter.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/xapit/client/model_adapters/mongoid_adapter.rb b/lib/xapit/client/model_adapters/mongoid_adapter.rb index cc323ad..40591da 100644 --- a/lib/xapit/client/model_adapters/mongoid_adapter.rb +++ b/lib/xapit/client/model_adapters/mongoid_adapter.rb @@ -25,3 +25,8 @@ def index_all end end end + +if defined?(Mongoid::Document) + Mongoid::Document::ClassMethods.send(:include, Xapit::Client::Membership::ClassMethods) +end + From 31d469cb22dac851db74585283a8147742784adf Mon Sep 17 00:00:00 2001 From: "David A. Cuadrado" Date: Wed, 28 Sep 2011 18:06:39 -0500 Subject: [PATCH 04/13] include mongoid adapter Signed-off-by: David A. Cuadrado --- lib/xapit.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/xapit.rb b/lib/xapit.rb index adfc997..92e23a6 100644 --- a/lib/xapit.rb +++ b/lib/xapit.rb @@ -119,3 +119,4 @@ def symbolize_keys(arg) require 'xapit/client/model_adapters/abstract_model_adapter' require 'xapit/client/model_adapters/default_model_adapter' require 'xapit/client/model_adapters/active_record_adapter' +require 'xapit/client/model_adapters/mongoid_adapter' From 6f44a9432a460fa7efc85e0fb2dd82414bedfa57 Mon Sep 17 00:00:00 2001 From: "David A. Cuadrado" Date: Wed, 28 Sep 2011 18:23:04 -0500 Subject: [PATCH 05/13] make use of <= operator Signed-off-by: David A. Cuadrado --- lib/xapit/client/model_adapters/mongoid_adapter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/xapit/client/model_adapters/mongoid_adapter.rb b/lib/xapit/client/model_adapters/mongoid_adapter.rb index 40591da..c902592 100644 --- a/lib/xapit/client/model_adapters/mongoid_adapter.rb +++ b/lib/xapit/client/model_adapters/mongoid_adapter.rb @@ -2,7 +2,7 @@ module Xapit module Client class MongoidAdapter < AbstractModelAdapter def self.for_class?(model_class) - defined?(Mongoid::Document) && model_class.kind_of?(Mongoid::Document) + defined?(Mongoid::Document) && model_class <= Mongoid::Document end def setup From c4b50065a9b3a9c4d6fc8d09ee6bc48b2957255f Mon Sep 17 00:00:00 2001 From: "David A. Cuadrado" Date: Sun, 2 Oct 2011 22:23:14 -0500 Subject: [PATCH 06/13] update rack version Signed-off-by: David A. Cuadrado --- xapit.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xapit.gemspec b/xapit.gemspec index 03a7bcc..5ed6bee 100644 --- a/xapit.gemspec +++ b/xapit.gemspec @@ -10,7 +10,7 @@ Gem::Specification.new do |s| s.files = Dir["{lib,spec,features,rails_generators,tasks}/**/*", "[A-Z]*", "init.rb", "install.rb", "uninstall.rb"] - ["Gemfile.lock"] s.require_path = "lib" - s.add_dependency 'rack', '~> 1.2.2' + s.add_dependency 'rack', '~> 1.3.4' s.add_development_dependency 'rspec', '~> 2.5.0' s.add_development_dependency 'cucumber', '~> 0.10.2' From dc6827c10ba98bdd5e71a38723cca7387dea1b4a Mon Sep 17 00:00:00 2001 From: "Jorge H. Cuadrado" Date: Mon, 3 Oct 2011 16:52:31 -0500 Subject: [PATCH 07/13] Prevent circular references to to_json method Signed-off-by: Jorge H. Cuadrado --- lib/xapit/client/collection.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/xapit/client/collection.rb b/lib/xapit/client/collection.rb index 7f64a6a..9b11dd1 100644 --- a/lib/xapit/client/collection.rb +++ b/lib/xapit/client/collection.rb @@ -10,11 +10,11 @@ def initialize(clauses = []) end def in_classes(*classes) - scope(:in_classes, classes) + scope(:in_classes, classes.map(&:to_s)) end def not_in_classes(*classes) - scope(:not_in_classes, classes) + scope(:not_in_classes, classes.map(&:to_s)) end def search(phrase = nil) From 10c5d19b4cee292b177b3d915ab0feb5b03c0eb8 Mon Sep 17 00:00:00 2001 From: "David A. Cuadrado" Date: Tue, 4 Oct 2011 13:12:36 -0500 Subject: [PATCH 08/13] rack server must use the local database Signed-off-by: David A. Cuadrado --- lib/xapit.rb | 4 ++-- lib/xapit/server/app.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/xapit.rb b/lib/xapit.rb index 92e23a6..9313442 100644 --- a/lib/xapit.rb +++ b/lib/xapit.rb @@ -28,9 +28,9 @@ def reload @config.merge!(@loaded_config) if @loaded_config end - def database + def database(force_local = false) raise Disabled, "Unable to access Xapit database because it is disabled in configuration." unless Xapit.config[:enabled] - if config[:server] + if config[:server] && !force_local @database ||= Xapit::Client::RemoteDatabase.new(config[:server]) else @database ||= Xapit::Server::Database.new(config[:database_path]) diff --git a/lib/xapit/server/app.rb b/lib/xapit/server/app.rb index 03c36c3..ddf7720 100644 --- a/lib/xapit/server/app.rb +++ b/lib/xapit/server/app.rb @@ -13,7 +13,7 @@ def call(env) def action(command, json) data = Xapit.symbolize_keys(JSON.parse(json)) - render :content => Xapit.database.send(command, data).to_json + render :content => Xapit.database(true).send(command, data).to_json end def render(options = {}) From 13681d0bd280afed1574ae537c1f15bbe5f54e1d Mon Sep 17 00:00:00 2001 From: "David A. Cuadrado" Date: Wed, 5 Oct 2011 20:47:26 -0500 Subject: [PATCH 09/13] improve text indexer Signed-off-by: David A. Cuadrado --- lib/xapit/server/indexer.rb | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/lib/xapit/server/indexer.rb b/lib/xapit/server/indexer.rb index 27ba86a..ed04c15 100644 --- a/lib/xapit/server/indexer.rb +++ b/lib/xapit/server/indexer.rb @@ -45,12 +45,29 @@ def values # TODO refactor with stemmed_text_terms def text_terms + terms = [] each_attribute(:text) do |name, value, options| - value = value.to_s.split(/\s+/u).map { |w| w.gsub(/[^\w]/u, "") } unless value.kind_of? Array - value.map(&:to_s).map(&:downcase).map do |term| - [term, options[:weight] || 1] unless term.empty? + words = value + words = value.to_s.split(/\s+/u) unless value.kind_of? Array + + weight = options[:weight] || 1 + words.each do |word| + next if word.empty? || word.bytesize >= 245 + + word.downcase! + + terms << [word, weight] + terms << stemmed_term(word, weight) + + clean_word = ActiveSupport::Multibyte::Chars.new(word).normalize(:kd).gsub(/[^\w]/u, "").to_s + if !clean_word.empty? && clean_word != word + terms << [clean_word, weight] + terms << stemmed_term(clean_word, weight) + end end - end.flatten(1).compact + end + + terms.uniq end # TODO refactor with stemmed_text_terms @@ -98,6 +115,18 @@ def stemmer @stemmer ||= Xapian::Stem.new(Xapit.config[:stemming]) if Xapit.config[:stemming] end + def stemmed_term(word, weight = 1) + term = [word, weight] + if stemmer + stemmed = stemmer.call(word) + if stemmed != word + term = ["Z#{word}", weight] + end + end + + term + end + def base_terms [["C#{@data[:class]}", 1], [id_term, 1]] end From cb2df2c861f250b58ee9dcf51ab6ddc75f32d97c Mon Sep 17 00:00:00 2001 From: "David A. Cuadrado" Date: Wed, 5 Oct 2011 21:18:39 -0500 Subject: [PATCH 10/13] allow to set the stemmer language by document Signed-off-by: David A. Cuadrado --- lib/xapit/client/index_builder.rb | 27 ++++++++++++++++++++++++++- lib/xapit/server/indexer.rb | 17 +++++++++-------- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/lib/xapit/client/index_builder.rb b/lib/xapit/client/index_builder.rb index 2ed5dea..640bd98 100644 --- a/lib/xapit/client/index_builder.rb +++ b/lib/xapit/client/index_builder.rb @@ -2,8 +2,11 @@ module Xapit module Client class IndexBuilder attr_reader :attributes + attr_reader :language + def initialize @attributes = {} + @language = Xapit.config[:stemming] end def text(*args, &block) @@ -37,7 +40,13 @@ def update_document(member) end def document_data(member) - data = {:class => member.class.name, :id => member.id, :attributes => {}} + data = { + :class => member.class.name, + :id => member.id, + :attributes => {}, + :language => find_language(member) + } + attributes.each do |name, options| options = options.dup # so we can remove block without changing original hash value = member.send(name) @@ -53,8 +62,24 @@ def facets end end + def language(lang) + @language = lang + end + private + def find_language(member) + lang = if @language.kind_of?(Symbol) + member.send(@language) + elsif @language.kind_of?(Proc) + @language.call(member) + else + @language + end + + lang || 'english' + end + def add_attribute(type, *args, &block) options = args.last.kind_of?(Hash) ? args.pop : {} args.each do |attribute| diff --git a/lib/xapit/server/indexer.rb b/lib/xapit/server/indexer.rb index ed04c15..70f2fc2 100644 --- a/lib/xapit/server/indexer.rb +++ b/lib/xapit/server/indexer.rb @@ -43,7 +43,6 @@ def values values end - # TODO refactor with stemmed_text_terms def text_terms terms = [] each_attribute(:text) do |name, value, options| @@ -72,7 +71,7 @@ def text_terms # TODO refactor with stemmed_text_terms def stemmed_text_terms - if stemmer + if stemmer = build_stemmer each_attribute(:text) do |name, value, options| value = value.to_s.split(/\s+/u).map { |w| w.gsub(/[^\w]/u, "") } unless value.kind_of? Array value.map(&:to_s).map(&:downcase).map do |term| @@ -111,17 +110,19 @@ def save_facets private - def stemmer - @stemmer ||= Xapian::Stem.new(Xapit.config[:stemming]) if Xapit.config[:stemming] + def build_stemmer + begin + Xapian::Stem.new(@data[:language]) + rescue ArgumentError + return nil + end end def stemmed_term(word, weight = 1) term = [word, weight] - if stemmer + if stemmer = build_stemmer stemmed = stemmer.call(word) - if stemmed != word - term = ["Z#{word}", weight] - end + term = ["Z#{word}", weight] if stemmed != word end term From 9394651416fa194aa395b8818adb2812131f2da4 Mon Sep 17 00:00:00 2001 From: "David A. Cuadrado" Date: Thu, 6 Oct 2011 12:04:19 -0500 Subject: [PATCH 11/13] include active support Signed-off-by: David A. Cuadrado --- lib/xapit.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/xapit.rb b/lib/xapit.rb index 9313442..e921c81 100644 --- a/lib/xapit.rb +++ b/lib/xapit.rb @@ -3,6 +3,7 @@ require 'rack' require 'json' require 'net/http' +require 'active_support' module Xapit # A general Xapit exception From b6a7a40c7fe2a381644899a56c3cd8253af8b966 Mon Sep 17 00:00:00 2001 From: "David A. Cuadrado" Date: Thu, 6 Oct 2011 12:20:43 -0500 Subject: [PATCH 12/13] indexing all should use the local database otherwise it would be very slow Signed-off-by: David A. Cuadrado --- lib/xapit/client/index_builder.rb | 12 ++++++------ .../client/model_adapters/active_record_adapter.rb | 2 +- lib/xapit/client/model_adapters/mongoid_adapter.rb | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/xapit/client/index_builder.rb b/lib/xapit/client/index_builder.rb index 640bd98..9e9483a 100644 --- a/lib/xapit/client/index_builder.rb +++ b/lib/xapit/client/index_builder.rb @@ -27,16 +27,16 @@ def facet(name, custom_name = nil, &block) add_attribute(:facet, name, options, &block) end - def add_document(member) - Xapit.database.add_document(document_data(member)) + def add_document(member, force_local = false) + Xapit.database(force_local).add_document(document_data(member)) end - def remove_document(member) - Xapit.database.remove_document(document_data(member)) + def remove_document(member, force_local = false) + Xapit.database(force_local).remove_document(document_data(member)) end - def update_document(member) - Xapit.database.update_document(document_data(member)) + def update_document(member, force_local = false) + Xapit.database(force_local).update_document(document_data(member)) end def document_data(member) diff --git a/lib/xapit/client/model_adapters/active_record_adapter.rb b/lib/xapit/client/model_adapters/active_record_adapter.rb index da2420a..ff8e65d 100644 --- a/lib/xapit/client/model_adapters/active_record_adapter.rb +++ b/lib/xapit/client/model_adapters/active_record_adapter.rb @@ -19,7 +19,7 @@ def setup def index_all @model_class.find_each do |member| - member.class.xapit_index_builder.add_document(member) + member.class.xapit_index_builder.add_document(member, true) end end end diff --git a/lib/xapit/client/model_adapters/mongoid_adapter.rb b/lib/xapit/client/model_adapters/mongoid_adapter.rb index c902592..cbf6369 100644 --- a/lib/xapit/client/model_adapters/mongoid_adapter.rb +++ b/lib/xapit/client/model_adapters/mongoid_adapter.rb @@ -19,7 +19,7 @@ def setup def index_all @model_class.all.each do |member| - member.class.xapit_index_builder.add_document(member) + member.class.xapit_index_builder.add_document(member, true) end end end From f8fbce7aa38fdcdbfd90f0c1abdb18fe0de800cb Mon Sep 17 00:00:00 2001 From: Patrick Aljord Date: Sun, 9 Oct 2011 20:42:12 -0500 Subject: [PATCH 13/13] added total_count alias --- lib/xapit/client/collection.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/xapit/client/collection.rb b/lib/xapit/client/collection.rb index 9b11dd1..224f3fe 100644 --- a/lib/xapit/client/collection.rb +++ b/lib/xapit/client/collection.rb @@ -85,6 +85,7 @@ def equal?(other) def total_entries query[:total].to_i end + alias :total_count :total_entries def current_page (clause_value(:page) || 1).to_i