Merge pull request #101 from theablefew/fix/keyword_assumption

Fix keyword expansion and add configuration for auto keyword expansion

lib/stretchy.rb

@@ -41,32 +41,39 @@ class QueryOptionMissing < StandardError; end
 
   class Configuration
 
-    attr_accessor :client
-    attr_accessor :opensearch
+    attr_accessor :options
+
+    delegate_missing_to :@options
 
     def initialize
-        @client = Elasticsearch::Client.new url: 'http://localhost:9200'
-        @opensearch = false
+        @options = Hashie::Mash.new(
+          default_keyword_field: :keyword,
+          add_keyword_field_to_text_attributes: true,
+          auto_target_keywords: true,
+          opensearch: false,
+          client: Elasticsearch::Client.new(url: 'http://localhost:9200')
+        )
     end
 
-    def client=(client)
-        @client = client
-        self.opensearch = true if @client.class.name =~ /OpenSearch/
+    def client=(klient)
+        @options[:client] = klient
+        self.opensearch = true if klient.class.name =~ /OpenSearch/
     end
 
     def search_backend_const
-        @opensearch ? OpenSearch : Elasticsearch
+        self.opensearch? ? OpenSearch : Elasticsearch
     end
 
     def opensearch=(bool)
-        @opensearch = bool
+        @options[:opensearch] = bool
         OpenSearchCompatibility.opensearch_patch! if bool
     end
 
     def opensearch?
-        @opensearch
+        @options[:opensearch]
     end
 
+
   end
 
   class << self

lib/stretchy/attributes/transformers/keyword_transformer.rb

@@ -38,60 +38,50 @@ def cast_value_keys
             end
           end
 
-          def keyword?(arg)
-            attr = @attribute_types[arg.to_s] 
-            return false unless attr
-            attr.is_a?(Stretchy::Attributes::Type::Keyword)
+          def keyword_available?(arg)
+            attrib = @attribute_types[arg.to_s.split(".").first] 
+            return false unless attrib 
+            attrib.respond_to?(:keyword_field?) && attrib.keyword_field?
+          end
+
+          def keyword_field_for(arg)
+            attrib = @attribute_types[arg.to_s.split(".").first]
+            keyword_field = attrib.respond_to?(:fields) ? attrib.fields.find { |k,d| d[:type].to_sym == :keyword }&.first : nil
+            keyword_field || Stretchy.configuration.default_keyword_field
           end
 
           def protected?(arg)
             return false if arg.nil?
             Stretchy::Relations::AggregationMethods.registry.include?(arg.to_sym)
           end
 
+          # Add `.keyword` to attributes that have a keyword subfield but aren't `:keywords`
+          # this is for text fields that have a keyword subfield
+          # `:text` and `:string` fields add a `:keyword` subfield to the attribute mapping automatically
           def transform(item, *ignore) 
-            item.each_with_object({}) do |(k, v), new_item|
-              if ignore && ignore.include?(k)
-                new_item[k] = v
+            return unless Stretchy.configuration.auto_target_keywords
+            item.each_with_object({}) do |(key, value), new_item|
+              if ignore && ignore.include?(key)
+                new_item[key] = value
                 next
               end
-              new_key = (!protected?(k) && keyword?(k)) ? "#{k}.keyword" : k
 
-              new_value = v
+              new_key = (!protected?(key) && keyword_available?(key)) ? "#{key}.#{keyword_field_for(key)}" : key
+
+              new_value = value
 
               if new_value.is_a?(Hash)
-                new_value = transform(new_value)
+                new_value = transform(new_value, *ignore)
               elsif new_value.is_a?(Array)
-                new_value = new_value.map { |i| i.is_a?(Hash) ? transform(i) : i }
+                new_value = new_value.map { |i| i.is_a?(Hash) ? transform(i, *ignore) : i }
               elsif new_value.is_a?(String) || new_value.is_a?(Symbol) 
-                new_value = "#{new_value}.keyword" if keyword?(new_value)
+                new_value = "#{new_value}.#{keyword_field_for(new_value)}" if keyword_available?(new_value) && new_value.to_s !~  Regexp.new("\.#{keyword_field_for(new_value)}$")
               end
 
               new_item[new_key] = new_value
             end
           end
 
-          # If terms are used, we assume that the field is a keyword field
-          # and append .keyword to the field name
-          # {terms: {field: 'gender'}}
-          # or nested aggs
-          # {terms: {field: 'gender'}, aggs: {name: {terms: {field: 'position.name'}}}}
-          # should be converted to
-          # {terms: {field: 'gender.keyword'}, aggs: {name: {terms: {field: 'position.name.keyword'}}}}
-          # {date_histogram: {field: 'created_at', interval: 'day'}}
-          def assume_keyword_field(args={}, parent_match=false)
-            if args.is_a?(Hash)
-              args.each do |k, v|
-                if v.is_a?(Hash) 
-                  assume_keyword_field(v, KEYWORD_AGGREGATION_FIELDS.include?(k))
-                else
-                  next unless v.is_a?(String) || v.is_a?(Symbol)
-                  args[k] = ([:field, :fields].include?(k.to_sym) && v !~ /\.keyword$/ && parent_match) ? "#{v}.keyword" : v.to_s
-                end
-              end
-            end
-          end
-
       end
     end
   end

lib/stretchy/attributes/type/base.rb

@@ -5,19 +5,23 @@ class Base < ActiveModel::Type::Value
 
               OPTIONS = []
 
-
               def initialize(**args)
 
                 define_option_methods!
 
                 args.each do |k, v|
                   if self.class::OPTIONS.include?(k)
                     instance_variable_set("@#{k}", v) 
-                    args.delete(k)
                   end
+                  args.delete(k)
                 end
                 super
               end
+
+              def keyword_field?
+                return false unless respond_to? :fields
+                fields.present? && fields.include?(:keyword)
+              end
 
               def mappings(name)
                 options = {type: type_for_database}

lib/stretchy/attributes/type/binary.rb

@@ -24,7 +24,7 @@ module Type
             #   end
             # ```
             #
-            class Binary < ActiveModel::Type::Value
+            class Binary < Stretchy::Attributes::Type::Base
                 OPTIONS = [:doc_values, :store]
                 attr_reader *OPTIONS 
 

lib/stretchy/attributes/type/date_time.rb

@@ -32,7 +32,7 @@ module Stretchy::Attributes::Type
   #
   class DateTime < Stretchy::Attributes::Type::Base
     OPTIONS = [:doc_values, :format, :locale, :ignore_malformed, :index, :null_value, :on_script_error, :script, :store, :meta]
-    attr_reader *OPTIONS
+    attr_reader *OPTIONS + self.superclass::OPTIONS
     include ActiveModel::Type::Helpers::Timezone
     include ActiveModel::Type::Helpers::AcceptsMultiparameterTime.new(
       defaults: { 4 => 0, 5 => 0 }

lib/stretchy/attributes/type/hash.rb

@@ -36,11 +36,15 @@ def type_for_database
     end
 
     def mappings(name)
-      options = {}
+      options = {type: type_for_database}
       OPTIONS.each { |option| options[option] = send(option) unless send(option).nil? }
       { name => options }.as_json
     end
 
+    def keyword_field?
+      true
+    end
+
     private
 
     def cast_value(value)

lib/stretchy/attributes/type/keyword.rb

@@ -27,5 +27,6 @@ class Keyword < Stretchy::Attributes::Type::Base
       def type
         :keyword
       end
+
     end
   end

lib/stretchy/attributes/type/numeric/base.rb

@@ -31,7 +31,7 @@ module Stretchy::Attributes::Type::Numeric
   # ```
   #
   class Base < Stretchy::Attributes::Type::Base #:nodoc:
-    OPTIONS = [:coerce, :doc_values, :ignore_malformed, :index, :meta, :null_value, :on_script_error, :script, :store, :time_series_dimension, :time_series_metric]
+    OPTIONS = [:coerce, :doc_values, :ignore_malformed, :index, :meta, :null_value, :on_script_error, :script, :store, :time_series_dimension, :time_series_metric] + self.superclass::OPTIONS
 
     def type
       raise NotImplementedError, "You must use one of the numeric types: integer, long, short, byte, double, float, half_float, scaled_float."

lib/stretchy/attributes/type/text.rb

@@ -3,6 +3,12 @@ module Stretchy::Attributes::Type
     #
     # This class is used to define a text attribute for a model. It provides support for the Elasticsearch text data type, which is a type of data type that can hold text strings.
     #
+    # >[!NOTE]
+    # >
+    # > The default for the `:text` type is to have a keyword multified if `field:` is not specified and `fields:` is not explicitly false.
+    # > This can be disabled by setting `Stretchy.configuration.add_keyword_field_to_text_attributes` to false.
+    # > The default keyword field name is `:keyword`, but this can be changed by setting `Stretchy.configuration.default_keyword_field`.
+    #
     # ### Parameters
     #
     # - `type:` `:text`.
@@ -39,7 +45,13 @@ module Stretchy::Attributes::Type
     #
     class Text < Stretchy::Attributes::Type::Base
       OPTIONS = [:analyzer, :eager_global_ordinals, :fielddata, :fielddata_frequency_filter, :fields, :index, :index_options, :index_prefixes, :index_phrases, :norms, :position_increment_gap, :store, :search_analyzer, :search_quote_analyzer, :similarity, :term_vector, :meta]
-
+
+      def initialize(**args)
+        # Add a keyword field by default if no fields are specified
+        args.reverse_merge!(fields: {keyword: {type: :keyword, ignore_above: 256}}) if args[:fields].nil? && Stretchy.configuration.add_keyword_field_to_text_attributes
+        super 
+      end
+
       def type
         :text
       end
@@ -48,6 +60,11 @@ def type_for_database
         :text
       end
 
+      # The default for the `:text` type is to have a keyword field if no fields are specified.
+      def keyword_field?
+        fields.find { |k,d| d[:type].to_sym == :keyword}.present?
+      end
+
       def mappings(name)
         options = {type: type_for_database}
         OPTIONS.each { |option| options[option] = send(option) unless send(option).nil? }

lib/stretchy/model/callbacks.rb

@@ -8,6 +8,7 @@ module Callbacks
         included do
           mattr_accessor :_circuit_breaker_callbacks, default: []
 
+          define_model_callbacks :initialize, only: :after
           define_model_callbacks :create, :save, :update, :destroy
           define_model_callbacks :find, :touch, only: :after
         end

lib/stretchy/record.rb

@@ -46,6 +46,7 @@ def self.inherited(base)
                 def initialize(attributes = {})
                     @highlights = attributes.delete(:_highlights) 
                     super(attributes)
+                    run_callbacks :initialize 
                 end
 
             end

lib/stretchy/relations/aggregation_methods/aggregation.rb

@@ -47,7 +47,7 @@ def aggregation(name, options = {}, &block)
         end
 
         def aggregation!(name, options = {}, &block) # :nodoc:
-            self.aggregation_values += [{name: name, args: assume_keyword_field(options)}]
+            self.aggregation_values += [{name: name, args: options}]
             self
         end
 

lib/stretchy/relations/query_builder.rb

@@ -102,7 +102,7 @@ def to_elastic
         build_highlights unless highlights.blank?
         build_fields unless fields.blank?
         build_source unless source.blank?
-        build_aggregations unless aggregations.blank?
+        build_aggregations(aggregations, structure) unless aggregations.blank?
         structure.attributes!.with_indifferent_access
       end
 
@@ -264,10 +264,10 @@ def build_highlights
         end
       end
 
-      def build_aggregations
-        structure.aggregations do
-          aggregations.each do |agg|
-            structure.set! agg[:name], aggregation(agg[:name], keyword_transformer.transform(agg[:args], :name))
+      def build_aggregations(aggregation_args, aggregation_structure)
+        aggregation_structure.aggregations do
+          aggregation_args.each do |agg|
+            aggregation_structure.set! agg[:name], aggregation(agg[:name], keyword_transformer.transform(agg[:args], :aggs, :aggregations))
           end
         end
       end
@@ -335,8 +335,6 @@ def as_query_string(q)
         _and.join(" AND ")
       end
 
-
-
       def extract_highlighter(highlighter)
         Jbuilder.new do |highlight|
           highlight.extract! highlighter
@@ -357,12 +355,17 @@ def extract_filters(name,opts = {})
       end
 
       def aggregation(name, opts = {})
-        Jbuilder.new do |agg|
+        Jbuilder.new do |agg_structure|
           case
           when opts.is_a?(Hash)
-              agg.extract! opts, *opts.keys
+              nested_agg = opts.delete(:aggs) || opts.delete(:aggregations)
+
+              agg_structure.extract! opts, *opts.keys
+
+              build_aggregations(nested_agg.map {|d| {:name => d.first, :args => d.last } }, agg_structure) if nested_agg
+
           when opts.is_a?(Array)
-              extract_filter_arguments_from_array(agg, opts)
+              extract_filter_arguments_from_array(agg_structure, opts)
           else
             raise "#aggregation only accepts Hash or Array"
           end

lib/stretchy/relations/query_methods.rb

@@ -103,21 +103,6 @@ def build_where(opts, other = [])
 
       private
 
-      KEYWORD_AGGREGATION_FIELDS = [:terms, :rare_terms, :significant_terms, :cardinality, :string_stats]
-
-      def assume_keyword_field(args={}, parent_match=false)
-        if args.is_a?(Hash)
-          args.each do |k, v|
-            if v.is_a?(Hash) 
-              assume_keyword_field(v, KEYWORD_AGGREGATION_FIELDS.include?(k))
-            else
-              next unless v.is_a?(String) || v.is_a?(Symbol)
-              args[k] = ([:field, :fields].include?(k.to_sym) && v !~ /\.keyword$/ && parent_match) ? "#{v}.keyword" : v.to_s
-            end
-          end
-        end
-      end
-
       def check_if_method_has_arguments!(method_name, args)
         if args.blank?
           raise ArgumentError, "The method .#{method_name}() must contain arguments."

spec/models/post.rb

@@ -1,7 +1,7 @@
 class Post < Stretchy::Record
 
 
-    attribute :title,                   :string
+    attribute :title,                   :keyword
     attribute :body,                    :string
     attribute :flagged,                 :boolean,  default: false  
     attribute :actor,                   :hash 

spec/models/resource.rb

@@ -1,11 +1,11 @@
 class Resource < Stretchy::Record
     index_name "resource_test"
 
-    attribute :name,                :keyword
+    attribute :name,                :string
     attribute :email,               :keyword
     attribute :phone,               :string
     attribute :position,            :hash
-    attribute :gender,              :keyword
+    attribute :gender,              :string
     attribute :age,                 :integer
     attribute :income,              :integer
     attribute :income_after_raise,  :integer

spec/models/test_model.rb

@@ -5,5 +5,7 @@ class TestModel < StretchyModel
   attribute :data, :hash
   attribute :published_at, :datetime
   attribute :agreed, :boolean
+  attribute :color, :keyword
+  attribute :text_with_keyword, :text, fields: {slug: {type: :keyword}}
 
 end