From 13ad0fcf772d6e747391d78e14d7d2f81180c99b Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 14 Jul 2022 07:21:38 +0200
Subject: [PATCH 01/42] auto lint  changes

---
 lib/goo/sparql/loader.rb | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index d877d47d3..259b624e0 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -8,7 +8,7 @@ class Loader
       def self.model_load(*options)
         options = options.last
         if options[:models] && options[:models].is_a?(Array) && \
-                             (options[:models].length > Goo.slice_loading_size)
+           (options[:models].length > Goo.slice_loading_size)
           options = options.dup
           models = options[:models]
           include_options = options[:include]
@@ -23,9 +23,9 @@ def self.model_load(*options)
               models_by_id[m.id] = m
             end
           end
-          return models_by_id
+           models_by_id
         else
-          return self.model_load_sliced(options)
+          self.model_load_sliced(options)
         end
       end
 
@@ -147,10 +147,10 @@ def self.get_predicate_map(predicates)
           predicates_map = {}
           uniq_p.each do |p|
             i = 0
-            key = ("var_" + p.last_part + i.to_s).to_sym
+            key = ('var_' + p.last_part + i.to_s).to_sym
             while predicates_map.include?(key)
               i += 1
-              key = ("var_" + p.last_part + i.to_s).to_sym
+              key = ('var_' + p.last_part + i.to_s).to_sym
               break if i > 10
             end
             predicates_map[key] = p
@@ -188,14 +188,14 @@ def self.get_includes(collection, graphs, incl, klass, optional_patterns, query_
       def self.get_binding_as(patterns, predicates_map)
         binding_as = nil
         if predicates_map
-          variables = [:id, :object, :bind_as]
+          variables = %i[id object bind_as]
           binding_as = []
           predicates_map.each do |var, pre|
             binding_as << [[[:id, pre, :object]], var, :bind_as]
           end
         else
-          patterns << [:id, :predicate, :object]
-          variables = [:id, :predicate, :object]
+          patterns << %i[id predicate object]
+          variables = %i[id predicate object]
         end
         unmapped = true
         return binding_as, unmapped, variables
@@ -268,6 +268,7 @@ def self.get_structures(aggregate, count, incl, include_pagination, klass, read_
           if incl_embed
             incl_embed.each do |k, vals|
               next if klass.collection?(k)
+
               attrs_struct = []
               vals.each do |v|
                 attrs_struct << v unless v.kind_of?(Hash)
@@ -278,18 +279,19 @@ def self.get_structures(aggregate, count, incl, include_pagination, klass, read_
           end
           direct_incl.each do |attr|
             next if embed_struct.include?(attr)
+
             embed_struct[attr] = klass.range(attr).struct_object([]) if klass.range(attr)
           end
 
         end
-        return embed_struct, klass_struct
+        [embed_struct, klass_struct]
       end
 
       def self.raise_resource_must_persistent_error(models)
         models.each do |m|
           if (not m.nil?) && !m.respond_to?(:klass) #read only
             raise ArgumentError,
-                  "To load attributes the resource must be persistent" unless m.persistent?
+                  'To load attributes the resource must be persistent' unless m.persistent?
           end
         end
       end
@@ -348,7 +350,7 @@ def self.model_set_collection_attributes(collection, klass, models_by_id, object
               collection_attribute = obj_new[:klass].collection_opts
               obj_new[collection_attribute] = collection_value
             elsif obj_new.class.respond_to?(:collection_opts) &&
-              obj_new.class.collection_opts.instance_of?(Symbol)
+                  obj_new.class.collection_opts.instance_of?(Symbol)
               collection_attribute = obj_new.class.collection_opts
               obj_new.send("#{collection_attribute}=", collection_value)
             end
@@ -399,7 +401,7 @@ def self.model_map_attributes_values(attr_to_load_if_empty, id, main_lang_hash,
                 if Goo.main_lang.nil?
                   models_by_id[id].send("#{v}=", object, on_load: true)
 
-                elsif (v.to_s.eql?("prefLabel"))
+                elsif (v.to_s.eql?('prefLabel'))
                   # Special treatment for prefLabel where we want to extract the main_lang first, or anything else
                   unless main_lang_hash[key]
 

From fac2795dc12634503596194a436d312da09f2c47 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 14 Jul 2022 07:23:30 +0200
Subject: [PATCH 02/42] remove duplicated method

include_bnodes can also be found in the solution mapper
---
 lib/goo/sparql/loader.rb | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index 259b624e0..b2bb69cc2 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -296,25 +296,6 @@ def self.raise_resource_must_persistent_error(models)
         end
       end
 
-      def self.include_bnodes(bnodes, collection, klass, models_by_id)
-        #group by attribute
-        attrs = bnodes.map { |x, y| y.attribute }.uniq
-        attrs.each do |attr|
-          struct = klass.range(attr)
-
-          #bnodes that are in a range of goo ground models
-          #for example parents and children in LD class models
-          #we skip this cases for the moment
-          next if struct.respond_to?(:model_name)
-
-          bnode_attrs = struct.new.to_h.keys
-          ids = bnodes.select { |x, y| y.attribute == attr }.map { |x, y| y.id }
-          klass.where.models(models_by_id.select { |x, y| ids.include?(x) }.values)
-               .in(collection)
-               .include(bnode: { attr => bnode_attrs }).all
-        end
-      end
-
       def self.include_embed_attributes(collection, incl_embed, klass, objects_new)
         incl_embed.each do |attr, next_attrs|
           #anything to join ?

From 8e52ba88d2f05c9f343f6fe7b30dfb9cbfa38006 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 14 Jul 2022 07:25:02 +0200
Subject: [PATCH 03/42] add the new function
 expand_equivalent_predicates_filter

---
 lib/goo/sparql/loader.rb | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index b2bb69cc2..9f0f1d46a 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -97,6 +97,31 @@ def self.model_load_sliced(*options)
         solution_mapper.map_each_solutions(select)
       end
 
+      # Expand equivalent predicate for attribute that are retrieved using filter (the new way to retrieve...)
+      # i.e.: prefLabel can also be retrieved using the "http://data.bioontology.org/metadata/def/prefLabel" URI
+      # so we add "http://data.bioontology.org/metadata/def/prefLabel" to the array_includes_filter that will generates a filter on property for meta:prefLabel
+      # and we add the following entry to the uri_properties_hash: "http://data.bioontology.org/metadata/def/prefLabel" => "prefLabel"
+      # So the object of http://data.bioontology.org/metadata/def/prefLabel will be retrieved and added to this attribute
+      def self.expand_equivalent_predicates_filter(eq_p, array_includes_filter, uri_properties_hash)
+        array_includes_filter_out = array_includes_filter.dup
+        if eq_p && eq_p.length > 0
+          if array_includes_filter
+            array_includes_filter.each do |predicate_filter|
+              if predicate_filter && predicate_filter.is_a?(RDF::URI)
+                if eq_p.include?(predicate_filter.to_s)
+                  eq_p[predicate_filter.to_s].each do |predicate_mapping|
+                    pred_map_uri = RDF::URI.new(predicate_mapping)
+                    array_includes_filter_out << pred_map_uri
+                    uri_properties_hash[pred_map_uri] = uri_properties_hash[predicate_filter]
+                  end
+                end
+              end
+            end
+          end
+        end
+        return array_includes_filter_out, uri_properties_hash
+      end
+
       def self.expand_equivalent_predicates(query, eq_p)
         attribute_mappings = {}
         if eq_p && eq_p.length.positive?

From f18548ae58400fe21d13ae86cf14905543857686 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 14 Jul 2022 07:28:54 +0200
Subject: [PATCH 04/42] update get_includes to select :id, :attributeProperty,
 :attributeObject

---
 lib/goo/sparql/loader.rb | 36 +++++++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index 9f0f1d46a..fb6ac097c 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -184,30 +184,48 @@ def self.get_predicate_map(predicates)
         predicates_map
       end
 
-
-
-
-      def self.get_includes(collection, graphs, incl, klass, optional_patterns, query_options, variables)
+      def self.get_includes(collection, graphs, incl, klass, query_options, variables)
         incl = incl.to_a
         incl_direct = incl.select { |a| a.instance_of?(Symbol) }
-        variables.concat(incl_direct)
+        #variables.concat(incl_direct)
         incl_embed = incl.select { |a| a.instance_of?(Hash) }
-        raise ArgumentError, "Not supported case for embed" if incl_embed.length > 1
+        raise ArgumentError, 'Not supported case for embed' if incl_embed.length > 1
+
         incl.delete_if { |a| !a.instance_of?(Symbol) }
 
         if incl_embed.length.positive?
           incl_embed = incl_embed.first
           embed_variables = incl_embed.keys.sort
-          variables.concat(embed_variables)
+          #variables.concat(embed_variables)
           incl.concat(embed_variables)
         end
+
+        variables.concat(%i[attributeProperty attributeObject])
+        optional_patterns = [%i[id attributeProperty attributeObject]]
+        array_includes_filter = []
+        uri_properties_hash = {}  # hash that contains "URI of the property => attribute label"
+        inverted = false
+
         incl.each do |attr|
           graph, pattern = query_pattern(klass, attr, collection: collection)
           add_rules(attr, klass, query_options)
-          optional_patterns << pattern if pattern
+          if klass.attributes(:all).include?(attr) && klass.inverse?(attr) && !inverted
+            # In case we have an inverse attribute to retrieve (i.e.: submissions linked to an ontology)
+            inverted = true
+            variables.concat([:inverseAttributeObject])
+            optional_patterns << %i[inverseAttributeObject attributeProperty id]
+          end
+          # When doing a "bring" the poorly written optional patterns come from here
+          #optional_patterns << pattern if pattern
+          array_includes_filter << pattern[1] # just take the URI of the attribute property
+
+          # The URI of the property is added to an hash (i.e.: "http://data.bioontology.org/metadata/def/prefLabel" => "prefLabel")
+          # so we can retrieve the property linked to this URI when retrieving the results
+          uri_properties_hash[pattern[1]] = attr
+
           graphs << graph if graph && (!klass.collection_opts || klass.inverse?(attr))
         end
-        [incl, incl_embed, variables, graphs, optional_patterns]
+        [incl, incl_embed, variables, graphs, optional_patterns, uri_properties_hash, array_includes_filter]
       end
 
       def self.get_binding_as(patterns, predicates_map)

From 320a7edc76dc48d7ab0dc7dca05c966832410696 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 14 Jul 2022 07:29:55 +0200
Subject: [PATCH 05/42] use the new get_includes

---
 lib/goo/sparql/loader.rb | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index fb6ac097c..5b193fda7 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -61,11 +61,13 @@ def self.model_load_sliced(*options)
         query_options = {}
         #TODO: breaks the reasoner
         patterns = [[:id, RDF.type, klass.uri_type(collection)]]
-        optional_patterns = []
 
         incl_embed = nil
         unmapped = nil
         bnode_extraction = nil
+        optional_patterns = []
+        array_includes_filter = []
+        uri_properties_hash = {}  # hash that contains "URI of the property => attribute label"
 
         if incl
           if incl.first && incl.first.is_a?(Hash) && incl.first.include?(:bnode)
@@ -76,14 +78,18 @@ def self.model_load_sliced(*options)
             binding_as, unmapped, variables = get_binding_as(patterns, predicates_map)
           else
             #make it deterministic
-            incl, incl_embed, variables, graphs, optional_patterns = get_includes(collection, graphs, incl, klass, optional_patterns, query_options, variables)
+            incl, incl_embed, variables, graphs, optional_patterns, uri_properties_hash, array_includes_filter =
+              get_includes(collection, graphs, incl, klass, query_options, variables)
+            array_includes_filter, uri_properties_hash = expand_equivalent_predicates_filter(equivalent_predicates,
+                                                                                             array_includes_filter,
+                                                                                             uri_properties_hash)
+            array_includes_filter.uniq!
           end
         end
 
 
 
 
-
         query_builder = Goo::SPARQL::QueryBuilder.new options
         select, aggregate_projections =
           query_builder.build_select_query(ids, binding_as,

From 1e3c1640b96006f6ff25b944dbb45418e78114e7 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 14 Jul 2022 07:31:35 +0200
Subject: [PATCH 06/42] update the query builder to add the ?attributeProperty
 filters

---
 lib/goo/sparql/loader.rb        |  3 ++-
 lib/goo/sparql/query_builder.rb | 13 ++++++++++---
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index 5b193fda7..3c5c8be95 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -94,7 +94,8 @@ def self.model_load_sliced(*options)
         select, aggregate_projections =
           query_builder.build_select_query(ids, binding_as,
                                            klass, graphs, optional_patterns,
-                                           order_by, patterns, query_options, variables)
+                                           order_by, patterns, query_options,
+                                           variables, array_includes_filter)
 
         expand_equivalent_predicates(select, equivalent_predicates)
         solution_mapper = Goo::SPARQL::SolutionMapper.new aggregate_projections, bnode_extraction,
diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index 067c11dc4..d0c1f7872 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -18,8 +18,7 @@ def initialize(options)
       end
 
       def build_select_query(ids, binding_as, klass, graphs, optional_patterns,
-                             order_by, patterns, query_options, variables)
-
+                             order_by, patterns, query_options, variables, array_includes_filter)
 
         internal_variables = graph_match(@collection, @graph_match, graphs, klass, patterns, query_options, @unions)
         aggregate_projections, aggregate_vars, variables, optional_patterns = get_aggregate_vars(@aggregate, @collection,
@@ -51,9 +50,15 @@ def build_select_query(ids, binding_as, klass, graphs, optional_patterns,
           order_by_str = order_by.map { |attr, order| "#{order.to_s.upcase}(?#{attr})" }
           select.order_by(*order_by_str)
         end
-
         select.filter(filter_id_str)
 
+        # Add the included attributes properties to the filter (to retrieve all the attributes we ask for)
+        if !array_includes_filter.nil? && array_includes_filter.length > 0
+          filter_predicates = array_includes_filter.map { |p| "?attributeProperty = #{p.to_ntriples}" }
+          filter_predicates = filter_predicates.join " || "
+          select.filter(filter_predicates)
+        end
+
         #if unmapped && predicates && predicates.length > 0
         #  filter_predicates = predicates.map { |p| "?predicate = #{p.to_ntriples}" }
         #  filter_predicates = filter_predicates.join " || "
@@ -125,6 +130,8 @@ def order_by(count, klass, order_by, patterns, variables)
       end
       [order_by, variables, patterns]
       end
+
+
       def sparql_op_string(op)
         case op
         when :or

From 8a9479311133247516b3758ecb800f7e2a2680dd Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 14 Jul 2022 07:37:44 +0200
Subject: [PATCH 07/42] update solutions_mapper to use the new pattern
 (?s,?p,?v,?inverse_v))

---
 lib/goo/sparql/loader.rb           |   5 +-
 lib/goo/sparql/solutions_mapper.rb | 152 +++++++++++++++++++++++++----
 2 files changed, 138 insertions(+), 19 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index 3c5c8be95..229c12581 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -97,10 +97,13 @@ def self.model_load_sliced(*options)
                                            order_by, patterns, query_options,
                                            variables, array_includes_filter)
 
+        # TODO: remove it? expand_equivalent_predicates_filter does the job now
         expand_equivalent_predicates(select, equivalent_predicates)
         solution_mapper = Goo::SPARQL::SolutionMapper.new aggregate_projections, bnode_extraction,
                                                           embed_struct, incl_embed, klass_struct, models_by_id,
-                                                          predicates_map, unmapped, variables, options
+                                                          predicates_map, unmapped,
+                                                          variables, uri_properties_hash, options
+
         solution_mapper.map_each_solutions(select)
       end
 
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 386b3f189..c8575846c 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -6,7 +6,7 @@ class SolutionMapper
 
       def initialize(aggregate_projections, bnode_extraction, embed_struct,
                      incl_embed, klass_struct, models_by_id,
-                     predicates_map, unmapped, variables, options)
+                     predicates_map, unmapped, variables, uri_properties_hash, options)
 
         @aggregate_projections = aggregate_projections
         @bnode_extraction = bnode_extraction
@@ -18,6 +18,7 @@ def initialize(aggregate_projections, bnode_extraction, embed_struct,
         @unmapped = unmapped
         @variables = variables
         @options = options
+        @uri_properties_hash = uri_properties_hash
 
       end
 
@@ -34,6 +35,7 @@ def map_each_solutions(select)
         var_set_hash = {}
         list_attributes = Set.new(klass.attributes(:list))
         all_attributes = Set.new(klass.attributes(:all))
+        id_array = []
 
         select.each_solution do |sol|
           next if sol[:some_type] && klass.type_uri(collection) != sol[:some_type]
@@ -42,6 +44,7 @@ def map_each_solutions(select)
 
           found.add(sol[:id])
           id = sol[:id]
+          id_array << id ## TODO same as "found"
 
           if @bnode_extraction
             struct = create_struct(@bnode_extraction, klass, @models_by_id, sol, @variables)
@@ -60,34 +63,147 @@ def map_each_solutions(select)
             next
           end
 
-          @variables.each do |v|
-            next if (v == :id) && @models_by_id.include?(id)
-            if (v != :id) && !all_attributes.include?(v)
-              model_add_aggregation(@aggregate_projections, @models_by_id, sol, v)
-              #TODO otther schemaless things
-              next
+          # Retrieve aggregates count
+          @aggregate_projections&.each do |aggregate_key, aggregate_val|
+            if @models_by_id[id].respond_to?(:add_aggregate)
+              @models_by_id[id].add_aggregate(aggregate_val[1], aggregate_val[0], sol[aggregate_key].object)
+            else
+              (@models_by_id[id].aggregates ||= []) << Goo::Base::AGGREGATE_VALUE.new(aggregate_val[1],
+                                                                                      aggregate_val[0],
+                                                                                      sol[aggregate_key].object)
             end
-            #group for multiple values
-            object = sol[v] || nil
+          end
+
+          next if sol[:attributeProperty].nil?
+
+          # Retrieve all included attributes
+          object = if !sol[:attributeObject].nil?
+                     sol[:attributeObject]
+                   elsif !sol[:inverseAttributeObject].nil?
+                     sol[:inverseAttributeObject]
+                   end
+
+          # Get the property label using the hash
+
+          v = @uri_properties_hash[sol[:attributeProperty]]
 
-            #bnodes
-            if object.kind_of?(RDF::Node) && object.anonymous? && incl.include?(v)
-              initialize_object(id, klass, object, objects_new, v)
-              next
+          next if v.nil? || ((v != :id) && !all_attributes.include?(v))
+
+          #group for multiple values
+          #bnodes
+          if object.kind_of?(RDF::Node) && object.anonymous? && incl.include?(v)
+            range = klass.range(v)
+            if range.respond_to?(:new)
+              objects_new[object] = BNODES_TUPLES.new(id, v)
             end
+            next
+          end
 
-            object = object.object if object && !(object.kind_of? RDF::URI)
+          if object and !(object.kind_of? RDF::URI)
+            object = object.object
+          end
 
-            #dependent model creation
-            object = dependent_model_creation(@embed_struct, id, @models_by_id, object, objects_new, v, @options)
+          #dependent model creation
+          if object.kind_of?(RDF::URI) && v != :id
+            range_for_v = klass.range(v)
+            if range_for_v
+              if objects_new.include?(object)
+                object = objects_new[object]
+              else
+                unless range_for_v.inmutable?
+                  pre_val = nil
+                  if @models_by_id[id] &&
+                    ((@models_by_id[id].respond_to?(:klass) && models_by_id[id]) ||
+                      @models_by_id[id].loaded_attributes.include?(v))
+                    pre_val = if !read_only
+                                @models_by_id[id].instance_variable_get("@#{v}")
+                              else
+                                @models_by_id[id][v]
+                              end
+                    if pre_val.is_a?(Array)
+                      pre_val = pre_val.select { |x| x.id == object }.first
+                    end
+                  end
+                  if !read_only
+                    object = pre_val ? pre_val : klass.range_object(v, object)
+                    objects_new[object.id] = object
+                  else
+                    #depedent read only
+                    struct = pre_val ? pre_val : @embed_struct[v].new
+                    struct.id = object
+                    struct.klass = klass.range(v)
+                    objects_new[struct.id] = struct
+                    object = struct
+                  end
+                else
+                  object = range_for_v.find(object).first
+                end
+              end
+            end
+          end
 
-            object = object_to_array(id, @klass_struct, @models_by_id, object, v) if list_attributes.include?(v)
+          if list_attributes.include?(v)
+            # To handle attr that are lists
+            pre = @klass_struct ? @models_by_id[id][v] :
+                    @models_by_id[id].instance_variable_get("@#{v}")
+            if object.nil? && pre.nil?
+              object = []
+            elsif object.nil? && !pre.nil?
+              object = pre
+            elsif object
+              object = !pre ? [object] : (pre.dup << object)
+              object.uniq!
+            end
+          end
+
+          if @models_by_id[id].respond_to?(:klass)
+            unless object.nil? && !@models_by_id[id][v].nil?
+              @models_by_id[id][v] = object
+            end
+          else
+            unless @models_by_id[id].class.handler?(v)
+              unless object.nil? && !@models_by_id[id].instance_variable_get("@#{v.to_s}").nil?
+                if v != :id
+                  # if multiple language values are included for a given property, set the
+                  # corresponding model attribute to the English language value - NCBO-1662
+                  if object.kind_of?(RDF::Literal)
+                    key = "#{v}#__#{id.to_s}"
+                    @models_by_id[id].send("#{v}=", object, on_load: true) unless var_set_hash[key]
+                    lang = object.language
+                    var_set_hash[key] = true if lang == :EN || lang == :en
+                  else
+                    @models_by_id[id].send("#{v}=", object, on_load: true)
+                  end
+                end
+              end
+            end
+          end
+
+        end
+        unless incl.nil?
+          # Here we are setting to nil all attributes that have been included but not found in the triplestore
+          id_array.uniq!
+          incl.each do |attr_to_incl|
+            # Go through all attr we had to include
+            next if attr_to_incl.is_a? Hash
+
+            id_array.each do |model_id|
+              # Go through all models queried
+              if @models_by_id[model_id].respond_to?("loaded_attributes") && !@models_by_id[model_id].loaded_attributes.include?(attr_to_incl) && @models_by_id[model_id].respond_to?(attr_to_incl) && !attr_to_incl.to_s.eql?("unmapped")
+                if list_attributes.include?(attr_to_incl)
+                  # If the asked attr has not been loaded then it is set to nil or to an empty array for list attr
+                  @models_by_id[model_id].send("#{attr_to_incl}=", [], on_load: true)
+                else
+                  @models_by_id[model_id].send("#{attr_to_incl}=", nil, on_load: true)
+                end
+              end
+            end
 
-            model_map_attributes_values(id, var_set_hash, @models_by_id, object, sol, v) unless object.nil?
           end
         end
 
         return @models_by_id if @bnode_extraction
+
         model_set_collection_attributes(collection, klass, @models_by_id, objects_new)
 
         #remove from models_by_id elements that were not touched

From 20222224f62d8392df8c2a355f62abe9819902e5 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 19 Jul 2022 14:23:45 +0200
Subject: [PATCH 08/42] update expand_equivalent_predicates

---
 lib/goo/sparql/loader.rb | 71 +++++-----------------------------------
 1 file changed, 8 insertions(+), 63 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index 229c12581..0a94f90c2 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -108,71 +108,16 @@ def self.model_load_sliced(*options)
       end
 
       # Expand equivalent predicate for attribute that are retrieved using filter (the new way to retrieve...)
-      # i.e.: prefLabel can also be retrieved using the "http://data.bioontology.org/metadata/def/prefLabel" URI
-      # so we add "http://data.bioontology.org/metadata/def/prefLabel" to the array_includes_filter that will generates a filter on property for meta:prefLabel
-      # and we add the following entry to the uri_properties_hash: "http://data.bioontology.org/metadata/def/prefLabel" => "prefLabel"
-      # So the object of http://data.bioontology.org/metadata/def/prefLabel will be retrieved and added to this attribute
-      def self.expand_equivalent_predicates_filter(eq_p, array_includes_filter, uri_properties_hash)
-        array_includes_filter_out = array_includes_filter.dup
-        if eq_p && eq_p.length > 0
-          if array_includes_filter
-            array_includes_filter.each do |predicate_filter|
-              if predicate_filter && predicate_filter.is_a?(RDF::URI)
-                if eq_p.include?(predicate_filter.to_s)
-                  eq_p[predicate_filter.to_s].each do |predicate_mapping|
-                    pred_map_uri = RDF::URI.new(predicate_mapping)
-                    array_includes_filter_out << pred_map_uri
-                    uri_properties_hash[pred_map_uri] = uri_properties_hash[predicate_filter]
-                  end
-                end
-              end
-            end
-          end
-        end
-        return array_includes_filter_out, uri_properties_hash
-      end
 
-      def self.expand_equivalent_predicates(query, eq_p)
-        attribute_mappings = {}
-        if eq_p && eq_p.length.positive?
-          count_rewrites = 0
-          if query.options[:optionals]
-            query.options[:optionals].each do |opt|
-              opt.each do |pattern|
-                if pattern.predicate && pattern.predicate.is_a?(RDF::URI)
-                  if eq_p.include?(pattern.predicate.to_s)
-                    if attribute_mappings.include?(pattern.predicate.to_s)
-                      #reuse filter
-                      pattern.predicate =
-                        RDF::Query::Variable.new(attribute_mappings[pattern.predicate.to_s])
-                    else
-                      query_predicate = pattern.predicate
-                      var_name = "rewrite#{count_rewrites}"
-                      pattern.predicate = RDF::Query::Variable.new(var_name)
-                      expansion = eq_p[query_predicate.to_s]
-                      expansion = expansion.map { |x| "?#{var_name} = <#{x}>" }
-                      expansion = expansion.join " || "
-                      # Instead of applending the filters to the end of the query, as in query.filter(expansion),
-                      # we store them in the options[:filter] attribute. They will be included in the OPTIONAL
-                      # sections when the query is constructed. According to AG, this is the CORRECT way of
-                      # constructing the query.
-                      # Because the FILTERs are _outside_ the OPTIONALs, they are applied to _every_
-                      # row returned. i.e., only rows where ?rewrite0 is in its list _and_ ?rewrite1
-                      # is in its list will be returned. I.e., the query will return NO results where
-                      # ?rewrite0 or ?rewrite1 is NULL.
-                      #
-                      # All you need to do is to make sure that the FILTERS are applied only _inside_
-                      # each OPTIONAL.
-                      pattern.options[:filter] = expansion
-                      count_rewrites += 1
-                      attribute_mappings[query_predicate.to_s] = var_name
-                    end
-                  end
-                end
-              end
-            end
+        def expand_equivalent_predicates(properties_to_include, eq_p)
+
+          return unless eq_p && !eq_p.empty?
+
+          properties_to_include&.each do |property_attr, property|
+            property_uri = property[:uri]
+            property[:equivalents] = eq_p[property_uri.to_s].to_a.map { |p| RDF::URI.new(p) } if eq_p.include?(property_uri.to_s)
           end
-        end
+
       end
 
       def self.get_predicate_map(predicates)

From 7745e1f56bec4ee51f7b5be1258998d76f378f00 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 19 Jul 2022 14:24:55 +0200
Subject: [PATCH 09/42] update get_predicate_map to predicate_map

---
 lib/goo/sparql/loader.rb | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index 0a94f90c2..188fb09bf 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -120,24 +120,24 @@ def expand_equivalent_predicates(properties_to_include, eq_p)
 
       end
 
-      def self.get_predicate_map(predicates)
-        predicates_map = nil
-        if predicates
-          uniq_p = predicates.uniq
-          predicates_map = {}
-          uniq_p.each do |p|
-            i = 0
-            key = ('var_' + p.last_part + i.to_s).to_sym
-            while predicates_map.include?(key)
-              i += 1
+        def predicate_map(predicates)
+          predicates_map = nil
+          if predicates
+            uniq_p = predicates.uniq
+            predicates_map = {}
+            uniq_p.each do |p|
+              i = 0
               key = ('var_' + p.last_part + i.to_s).to_sym
-              break if i > 10
+              while predicates_map.include?(key)
+                i += 1
+                key = ('var_' + p.last_part + i.to_s).to_sym
+                break if i > 10
+              end
+              predicates_map[key] = { uri: p, is_inverse: false }
             end
-            predicates_map[key] = p
           end
+          predicates_map
         end
-        predicates_map
-      end
 
       def self.get_includes(collection, graphs, incl, klass, query_options, variables)
         incl = incl.to_a

From d08550841e9f4f55f085be9f070a607a84d558bd Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 19 Jul 2022 14:26:46 +0200
Subject: [PATCH 10/42] update get_includes

---
 lib/goo/sparql/loader.rb | 53 +++++++++-------------------------------
 1 file changed, 12 insertions(+), 41 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index 188fb09bf..731a85f20 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -139,49 +139,20 @@ def predicate_map(predicates)
           predicates_map
         end
 
-      def self.get_includes(collection, graphs, incl, klass, query_options, variables)
-        incl = incl.to_a
-        incl_direct = incl.select { |a| a.instance_of?(Symbol) }
-        #variables.concat(incl_direct)
-        incl_embed = incl.select { |a| a.instance_of?(Hash) }
-        raise ArgumentError, 'Not supported case for embed' if incl_embed.length > 1
-
-        incl.delete_if { |a| !a.instance_of?(Symbol) }
-
-        if incl_embed.length.positive?
-          incl_embed = incl_embed.first
-          embed_variables = incl_embed.keys.sort
-          #variables.concat(embed_variables)
-          incl.concat(embed_variables)
-        end
-
-        variables.concat(%i[attributeProperty attributeObject])
-        optional_patterns = [%i[id attributeProperty attributeObject]]
-        array_includes_filter = []
-        uri_properties_hash = {}  # hash that contains "URI of the property => attribute label"
-        inverted = false
-
-        incl.each do |attr|
-          graph, pattern = query_pattern(klass, attr, collection: collection)
-          add_rules(attr, klass, query_options)
-          if klass.attributes(:all).include?(attr) && klass.inverse?(attr) && !inverted
-            # In case we have an inverse attribute to retrieve (i.e.: submissions linked to an ontology)
-            inverted = true
-            variables.concat([:inverseAttributeObject])
-            optional_patterns << %i[inverseAttributeObject attributeProperty id]
+        def get_includes(collection, graphs, incl, klass, query_options)
+          incl = incl.to_a
+          incl.delete_if { |a| !a.instance_of?(Symbol) }
+          properties_to_include = {}
+          incl.each do |attr|
+            graph, pattern = query_pattern(klass, attr, collection: collection)
+            add_rules(attr, klass, query_options)
+            if klass.attributes(:all).include?(attr)
+              properties_to_include[attr] = { uri: pattern[1], is_inverse: klass.inverse?(attr) } # [property_attr, property_uri , inverse: true]
+            end
+            graphs << graph if graph && (!klass.collection_opts || klass.inverse?(attr))
           end
-          # When doing a "bring" the poorly written optional patterns come from here
-          #optional_patterns << pattern if pattern
-          array_includes_filter << pattern[1] # just take the URI of the attribute property
-
-          # The URI of the property is added to an hash (i.e.: "http://data.bioontology.org/metadata/def/prefLabel" => "prefLabel")
-          # so we can retrieve the property linked to this URI when retrieving the results
-          uri_properties_hash[pattern[1]] = attr
-
-          graphs << graph if graph && (!klass.collection_opts || klass.inverse?(attr))
+          [graphs, properties_to_include,query_options]
         end
-        [incl, incl_embed, variables, graphs, optional_patterns, uri_properties_hash, array_includes_filter]
-      end
 
       def self.get_binding_as(patterns, predicates_map)
         binding_as = nil

From 95a3fea68d89ba7eb38313bf7b1b7ebb23cc3fc9 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 19 Jul 2022 14:28:18 +0200
Subject: [PATCH 11/42] update get_binding_as

---
 lib/goo/sparql/loader.rb | 39 +++++++++++++--------------------------
 1 file changed, 13 insertions(+), 26 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index 731a85f20..79d1fc5de 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -154,34 +154,21 @@ def get_includes(collection, graphs, incl, klass, query_options)
           [graphs, properties_to_include,query_options]
         end
 
-      def self.get_binding_as(patterns, predicates_map)
-        binding_as = nil
-        if predicates_map
-          variables = %i[id object bind_as]
-          binding_as = []
-          predicates_map.each do |var, pre|
-            binding_as << [[[:id, pre, :object]], var, :bind_as]
+        def get_binding_as(patterns, predicates) end
+
+        def bnode_extraction(collection, incl, klass, patterns)
+          bnode_conf = incl.first[:bnode]
+          klass_attr = bnode_conf.keys.first
+          bnode_extraction = klass_attr
+          bnode = RDF::Node.new
+          variables = [:id]
+          patterns << [:id, klass.attribute_uri(klass_attr, collection), bnode]
+          bnode_conf[klass_attr].each do |in_bnode_attr|
+            variables << in_bnode_attr
+            patterns << [bnode, klass.attribute_uri(in_bnode_attr, collection), in_bnode_attr]
           end
-        else
-          patterns << %i[id predicate object]
-          variables = %i[id predicate object]
-        end
-        unmapped = true
-        return binding_as, unmapped, variables
-      end
-
-      def self.bnode_extraction(collection, incl, klass, patterns, variables)
-        bnode_conf = incl.first[:bnode]
-        klass_attr = bnode_conf.keys.first
-        bnode_extraction = klass_attr
-        bnode = RDF::Node.new
-        patterns << [:id, klass.attribute_uri(klass_attr, collection), bnode]
-        bnode_conf[klass_attr].each do |in_bnode_attr|
-          variables << in_bnode_attr
-          patterns << [bnode, klass.attribute_uri(in_bnode_attr, collection), in_bnode_attr]
+          [bnode_extraction, patterns, variables]
         end
-        [bnode_extraction, patterns, variables]
-      end
 
       def self.get_models_by_id_hash(ids, klass, klass_struct, models)
         models_by_id = {}

From 72c67171aab048f518ce8c74f93f8ca5eb651b4c Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 19 Jul 2022 14:29:50 +0200
Subject: [PATCH 12/42] update get_models_by_id_hash

---
 lib/goo/sparql/loader.rb | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index 79d1fc5de..44480c472 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -170,28 +170,28 @@ def bnode_extraction(collection, incl, klass, patterns)
           [bnode_extraction, patterns, variables]
         end
 
-      def self.get_models_by_id_hash(ids, klass, klass_struct, models)
-        models_by_id = {}
-        if models
-          ids = []
-          models.each do |m|
-            unless m.nil?
-              ids << m.id
-              models_by_id[m.id] = m
+        def get_models_by_id_hash(ids, klass, klass_struct, models)
+          models_by_id = {}
+          if models
+            ids = []
+            models.each do |m|
+              unless m.nil?
+                ids << m.id
+                models_by_id[m.id] = m
+              end
             end
-          end
-        elsif ids
-          ids.each do |id|
-            models_by_id[id] = klass_struct ? klass_struct.new : klass.new
-            models_by_id[id].klass = klass if klass_struct
-            models_by_id[id].id = id
-          end
-        else
-          #a where without models
+          elsif ids
+            ids.each do |id|
+              models_by_id[id] = klass_struct ? klass_struct.new : klass.new
+              models_by_id[id].klass = klass if klass_struct
+              models_by_id[id].id = id
+            end
+          else
+            #a where without models
 
+          end
+          return ids, models_by_id
         end
-        return ids, models_by_id
-      end
 
       def self.get_graphs(collection, klass)
         graphs = [klass.uri_type(collection)]

From 076420c9265c777198e8b9a8de3887fd985a537b Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 19 Jul 2022 14:33:51 +0200
Subject: [PATCH 13/42] remove unused functions in the loader (an omission)

---
 lib/goo/sparql/loader.rb | 142 ---------------------------------------
 1 file changed, 142 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index 44480c472..e7a32e38d 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -267,149 +267,7 @@ def self.include_embed_attributes(collection, incl_embed, klass, objects_new)
         end
       end
 
-      def self.models_set_all_persistent(models_by_id, options)
-        if options[:ids] #newly loaded
-          models_by_id.each do |k, m|
-            m.persistent = true
-          end
-        end
-      end
-
-      def self.model_set_collection_attributes(collection, klass, models_by_id, objects_new)
-        collection_value = get_collection_value(collection, klass)
-        if collection_value
-          collection_attribute = klass.collection_opts
-          models_by_id.each do |id, m|
-            m.send("#{collection_attribute}=", collection_value)
-          end
-          objects_new.each do |id, obj_new|
-            if obj_new.respond_to?(:klass)
-              collection_attribute = obj_new[:klass].collection_opts
-              obj_new[collection_attribute] = collection_value
-            elsif obj_new.class.respond_to?(:collection_opts) &&
-                  obj_new.class.collection_opts.instance_of?(Symbol)
-              collection_attribute = obj_new.class.collection_opts
-              obj_new.send("#{collection_attribute}=", collection_value)
             end
-          end
-        end
-      end
-
-      def self.get_collection_value(collection, klass)
-        collection_value = nil
-        if klass.collection_opts.instance_of?(Symbol)
-          if collection.is_a?(Array) && (collection.length == 1)
-            collection_value = collection.first
-          end
-          if collection.respond_to? :id
-            collection_value = collection
-          end
-        end
-        collection_value
-      end
-
-      def self.initialize_empty_attributes(attr_to_load_if_empty, id, models_by_id)
-        attr_to_load_if_empty.each do |empty_attr|
-          # To avoid bug where the attr is not loaded, we return an empty array (because the data model is really bad)
-          unless models_by_id[id].loaded_attributes.include?(empty_attr.to_sym)
-            models_by_id[id].send("#{empty_attr}=", [], on_load: true)
-          end
-        end
-      end
-
-      def self.model_map_attributes_values(attr_to_load_if_empty, id, main_lang_hash, models_by_id, object, sol, v)
-        if models_by_id[id].respond_to?(:klass)
-          models_by_id[id][v] = object if models_by_id[id][v].nil?
-        else
-          model_attribute_val = models_by_id[id].instance_variable_get("@#{v.to_s}")
-          if !models_by_id[id].class.handler?(v) || model_attribute_val.nil?
-            if v != :id
-              # if multiple language values are included for a given property, set the
-              # corresponding model attribute to the English language value - NCBO-1662
-              if sol[v].kind_of?(RDF::Literal)
-                key = "#{v}#__#{id.to_s}"
-                lang = sol[v].language
-
-                #models_by_id[id].send("#{v}=", object, on_load: true) unless var_set_hash[key]
-                #var_set_hash[key] = true if lang == :EN || lang == :en
-
-                # We add the value only if it's language is in the main languages or if lang is nil
-
-                if Goo.main_lang.nil?
-                  models_by_id[id].send("#{v}=", object, on_load: true)
-
-                elsif (v.to_s.eql?('prefLabel'))
-                  # Special treatment for prefLabel where we want to extract the main_lang first, or anything else
-                  unless main_lang_hash[key]
-
-                    models_by_id[id].send("#{v}=", object, on_load: true)
-                    if Goo.main_lang.include?(lang.to_s.downcase)
-                      # If prefLabel from the main_lang found we stop looking for prefLabel
-                      main_lang_hash[key] = true
-                    end
-                  end
-                elsif (lang.nil? || Goo.main_lang.include?(lang.to_s.downcase))
-                  models_by_id[id].send("#{v}=", object, on_load: true)
-                else
-                  attr_to_load_if_empty << v
-                end
-              else
-                models_by_id[id].send("#{v}=", object, on_load: true)
-              end
-            end
-          end
-        end
-      end
-
-      def self.object_to_array(id, klass_struct, models_by_id, object, v)
-        pre = klass_struct ? models_by_id[id][v] :
-                models_by_id[id].instance_variable_get("@#{v}")
-        if object.nil? && pre.nil?
-          object = []
-        elsif object.nil? && !pre.nil?
-          object = pre
-        elsif object
-          object = !pre ? [object] : (pre.dup << object)
-          object.uniq!
-        end
-        object
-      end
-
-      def self.dependent_model_creation(embed_struct, id, models_by_id, object, objects_new, v, options)
-        klass = options[:klass]
-        read_only = options[:read_only]
-        if object.kind_of?(RDF::URI) && v != :id
-          range_for_v = klass.range(v)
-          if range_for_v
-            if objects_new.include?(object)
-              object = objects_new[object]
-            elsif !range_for_v.inmutable?
-              pre_val = get_pre_val(id, models_by_id, object, v, read_only)
-              object = get_object_from_range(pre_val, embed_struct, object, objects_new, v, options)
-            else
-              object = range_for_v.find(object).first
-            end
-          end
-        end
-        object
-      end
-
-      def self.get_object_from_range(pre_val, embed_struct, object, objects_new, v, options)
-        klass = options[:klass]
-        read_only = options[:read_only]
-        range_for_v = klass.range(v)
-        if !read_only
-          object = pre_val || klass.range_object(v, object)
-          objects_new[object.id] = object
-        else
-          #depedent read only
-          struct = pre_val || embed_struct[v].new
-          struct.id = object
-          struct.klass = range_for_v
-          objects_new[struct.id] = struct
-          object = struct
-        end
-        object
       end
 
     end

From 2f8196e97f295cb93a0b436cb499da0aa1b468a5 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 19 Jul 2022 14:37:31 +0200
Subject: [PATCH 14/42] update get_embed_includes

---
 lib/goo/sparql/loader.rb | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index e7a32e38d..7bc43cd9a 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -252,24 +252,19 @@ def self.raise_resource_must_persistent_error(models)
         end
       end
 
-      def self.include_embed_attributes(collection, incl_embed, klass, objects_new)
-        incl_embed.each do |attr, next_attrs|
-          #anything to join ?
-          attr_range = klass.range(attr)
-          next if attr_range.nil?
-          range_objs = objects_new.select { |id, obj|
-            obj.instance_of?(attr_range) || (obj.respond_to?(:klass) && obj[:klass] == attr_range)
-          }.values
-          unless range_objs.empty?
-            range_objs.uniq!
-            attr_range.where().models(range_objs).in(collection).include(*next_attrs).all
+        def get_embed_includes(incl)
+          incl_embed = incl.select { |a| a.instance_of?(Hash) }
+          raise ArgumentError, 'Not supported case for embed' if incl_embed.length > 1
+          if incl_embed.length.positive?
+            incl_embed = incl_embed.first
+            embed_variables = incl_embed.keys.sort
+            #variables.concat(embed_variables)
+            incl.concat(embed_variables)
           end
+          incl_embed
         end
       end
 
-            end
-      end
-
     end
   end
 end

From e312656fe398663a0a5f3ea4dec65d4510184d7d Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 19 Jul 2022 14:42:38 +0200
Subject: [PATCH 15/42] add to the module loader `class << self to specify
 private functions

---
 lib/goo/sparql/loader.rb | 204 +++++++++++++++++++--------------------
 1 file changed, 100 insertions(+), 104 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index 7bc43cd9a..d5646849f 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -1,70 +1,66 @@
 module Goo
   module SPARQL
-    class Loader
-      extend Goo::SPARQL::QueryPatterns
+    module Loader
+      class << self
+        include Goo::SPARQL::QueryPatterns
 
-
-
-      def self.model_load(*options)
-        options = options.last
-        if options[:models] && options[:models].is_a?(Array) && \
+        def model_load(*options)
+          options = options.last
+          if options[:models] && options[:models].is_a?(Array) && \
            (options[:models].length > Goo.slice_loading_size)
-          options = options.dup
-          models = options[:models]
-          include_options = options[:include]
-          models_by_id = Hash.new
-          models.each_slice(Goo.slice_loading_size) do |model_slice|
-            options[:models] = model_slice
-            unless include_options.nil?
-              options[:include] = include_options.dup
+            options = options.dup
+            models = options[:models]
+            include_options = options[:include]
+            models_by_id = Hash.new
+            models.each_slice(Goo.slice_loading_size) do |model_slice|
+              options[:models] = model_slice
+              unless include_options.nil?
+                options[:include] = include_options.dup
+              end
+              model_load_sliced(options)
+              model_slice.each do |m|
+                models_by_id[m.id] = m
+              end
             end
+            models_by_id
+          else
             model_load_sliced(options)
-            model_slice.each do |m|
-              models_by_id[m.id] = m
-            end
           end
-           models_by_id
-        else
-          self.model_load_sliced(options)
         end
-      end
 
-      ##
-      # always a list of attributes with subject == id
-      ##
-      def self.model_load_sliced(*options)
-        options = options.last
-        ids = options[:ids]
-        klass = options[:klass]
-        incl = options[:include]
-        models = options[:models]
-        aggregate = options[:aggregate]
-        read_only = options[:read_only]
-        order_by = options[:order_by]
-        collection = options[:collection]
-        count = options[:count]
-        include_pagination = options[:include_pagination]
-        equivalent_predicates = options[:equivalent_predicates]
-        predicates = options[:predicates]
-        predicates_map = get_predicate_map predicates
-        binding_as = nil
-
-        embed_struct, klass_struct = get_structures(aggregate, count, incl, include_pagination, klass, read_only)
-
-        raise_resource_must_persistent_error(models) if models
-
-        graphs = get_graphs(collection, klass)
-        ids, models_by_id = get_models_by_id_hash(ids, klass, klass_struct, models)
+        ##
+        # always a list of attributes with subject == id
+        ##
+        def model_load_sliced(*options)
+          options = options.last
+          ids = options[:ids]
+          klass = options[:klass]
+          incl = options[:include]
+          models = options[:models]
+          aggregate = options[:aggregate]
+          read_only = options[:read_only]
+          collection = options[:collection]
+          count = options[:count]
+          include_pagination = options[:include_pagination]
+          equivalent_predicates = options[:equivalent_predicates]
+          predicates = options[:predicates]
+
+          embed_struct, klass_struct = get_structures(aggregate, count, incl, include_pagination, klass, read_only)
+
+          raise_resource_must_persistent_error(models) if models
+
+          graphs = get_graphs(collection, klass)
+          ids, models_by_id = get_models_by_id_hash(ids, klass, klass_struct, models)
 
         variables = [:id]
 
-        query_options = {}
-        #TODO: breaks the reasoner
-        patterns = [[:id, RDF.type, klass.uri_type(collection)]]
+          query_options = {}
+          #TODO: breaks the reasoner
+          patterns = [[:id, RDF.type, klass.uri_type(collection)]]
 
-        incl_embed = nil
-        unmapped = nil
-        bnode_extraction = nil
+          incl_embed = nil
+          unmapped = nil
+          bnode_extraction = nil
         optional_patterns = []
         array_includes_filter = []
         uri_properties_hash = {}  # hash that contains "URI of the property => attribute label"
@@ -90,7 +86,7 @@ def self.model_load_sliced(*options)
 
 
 
-        query_builder = Goo::SPARQL::QueryBuilder.new options
+          query_builder = Goo::SPARQL::QueryBuilder.new options
         select, aggregate_projections =
           query_builder.build_select_query(ids, binding_as,
                                            klass, graphs, optional_patterns,
@@ -104,10 +100,10 @@ def self.model_load_sliced(*options)
                                                           predicates_map, unmapped,
                                                           variables, uri_properties_hash, options
 
-        solution_mapper.map_each_solutions(select)
-      end
+          solution_mapper.map_each_solutions(select)
+        end
 
-      # Expand equivalent predicate for attribute that are retrieved using filter (the new way to retrieve...)
+        private
 
         def expand_equivalent_predicates(properties_to_include, eq_p)
 
@@ -193,64 +189,64 @@ def get_models_by_id_hash(ids, klass, klass_struct, models)
           return ids, models_by_id
         end
 
-      def self.get_graphs(collection, klass)
-        graphs = [klass.uri_type(collection)]
-        if collection
-          if collection.is_a?(Array) && collection.length.positive?
-            graphs = collection.map { |x| x.id }
-          elsif !collection.is_a? Array
-            graphs = [collection.id]
+        def get_graphs(collection, klass)
+          graphs = [klass.uri_type(collection)]
+          if collection
+            if collection.is_a?(Array) && collection.length.positive?
+              graphs = collection.map { |x| x.id }
+            elsif !collection.is_a? Array
+              graphs = [collection.id]
+            end
           end
+          graphs
         end
-        graphs
-      end
 
-      def self.get_structures(aggregate, count, incl, include_pagination, klass, read_only)
-        embed_struct = nil
-        klass_struct = nil
+        def get_structures(aggregate, count, incl, include_pagination, klass, read_only)
+          embed_struct = nil
+          klass_struct = nil
 
-        if read_only && !count && !aggregate
-          include_for_struct = incl
-          if !incl && include_pagination
-            #read only and pagination we do not know the attributes yet
-            include_for_struct = include_pagination
-          end
-          direct_incl = !include_for_struct ? [] :
-                          include_for_struct.select { |a| a.instance_of?(Symbol) }
-          incl_embed = include_for_struct.select { |a| a.instance_of?(Hash) }.first
-          klass_struct = klass.struct_object(direct_incl + (incl_embed ? incl_embed.keys : []))
-
-          embed_struct = {}
-          if incl_embed
-            incl_embed.each do |k, vals|
-              next if klass.collection?(k)
-
-              attrs_struct = []
-              vals.each do |v|
-                attrs_struct << v unless v.kind_of?(Hash)
-                attrs_struct.concat(v.keys) if v.kind_of?(Hash)
+          if read_only && !count && !aggregate
+            include_for_struct = incl
+            if !incl && include_pagination
+              #read only and pagination we do not know the attributes yet
+              include_for_struct = include_pagination
+            end
+            direct_incl = !include_for_struct ? [] :
+                            include_for_struct.select { |a| a.instance_of?(Symbol) }
+            incl_embed = include_for_struct.select { |a| a.instance_of?(Hash) }.first
+            klass_struct = klass.struct_object(direct_incl + (incl_embed ? incl_embed.keys : []))
+
+            embed_struct = {}
+            if incl_embed
+              incl_embed.each do |k, vals|
+                next if klass.collection?(k)
+
+                attrs_struct = []
+                vals.each do |v|
+                  attrs_struct << v unless v.kind_of?(Hash)
+                  attrs_struct.concat(v.keys) if v.kind_of?(Hash)
+                end
+                embed_struct[k] = klass.range(k).struct_object(attrs_struct)
               end
-              embed_struct[k] = klass.range(k).struct_object(attrs_struct)
             end
-          end
-          direct_incl.each do |attr|
-            next if embed_struct.include?(attr)
+            direct_incl.each do |attr|
+              next if embed_struct.include?(attr)
 
-            embed_struct[attr] = klass.range(attr).struct_object([]) if klass.range(attr)
-          end
+              embed_struct[attr] = klass.range(attr).struct_object([]) if klass.range(attr)
+            end
 
+          end
+          [embed_struct, klass_struct]
         end
-        [embed_struct, klass_struct]
-      end
 
-      def self.raise_resource_must_persistent_error(models)
-        models.each do |m|
-          if (not m.nil?) && !m.respond_to?(:klass) #read only
-            raise ArgumentError,
-                  'To load attributes the resource must be persistent' unless m.persistent?
+        def raise_resource_must_persistent_error(models)
+          models.each do |m|
+            if (not m.nil?) && !m.respond_to?(:klass) #read only
+              raise ArgumentError,
+                    'To load attributes the resource must be persistent' unless m.persistent?
+            end
           end
         end
-      end
 
         def get_embed_includes(incl)
           incl_embed = incl.select { |a| a.instance_of?(Hash) }

From b22227088ddb296416af5f058c70e31e1836d591 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 19 Jul 2022 14:44:59 +0200
Subject: [PATCH 16/42] add properties_to_include to save the properties to
 include in the query

---
 lib/goo/sparql/loader.rb | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index d5646849f..9db2069d2 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -52,8 +52,6 @@ def model_load_sliced(*options)
           graphs = get_graphs(collection, klass)
           ids, models_by_id = get_models_by_id_hash(ids, klass, klass_struct, models)
 
-        variables = [:id]
-
           query_options = {}
           #TODO: breaks the reasoner
           patterns = [[:id, RDF.type, klass.uri_type(collection)]]
@@ -61,10 +59,8 @@ def model_load_sliced(*options)
           incl_embed = nil
           unmapped = nil
           bnode_extraction = nil
-        optional_patterns = []
-        array_includes_filter = []
-        uri_properties_hash = {}  # hash that contains "URI of the property => attribute label"
-
+          properties_to_include = []
+          variables = [:id]
         if incl
           if incl.first && incl.first.is_a?(Hash) && incl.first.include?(:bnode)
             #limitation only one level BNODE
@@ -79,7 +75,7 @@ def model_load_sliced(*options)
             array_includes_filter, uri_properties_hash = expand_equivalent_predicates_filter(equivalent_predicates,
                                                                                              array_includes_filter,
                                                                                              uri_properties_hash)
-            array_includes_filter.uniq!
+          end
           end
         end
 

From 149a2e8a86d4c9e7a66e353ed5b5e42de6482a08 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 19 Jul 2022 14:49:16 +0200
Subject: [PATCH 17/42] use the new get_includes, predicate_map and
 get_bnode_extraction

---
 lib/goo/sparql/loader.rb | 36 +++++++++++++++++-------------------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index 9db2069d2..a8c3d39d2 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -61,23 +61,23 @@ def model_load_sliced(*options)
           bnode_extraction = nil
           properties_to_include = []
           variables = [:id]
-        if incl
-          if incl.first && incl.first.is_a?(Hash) && incl.first.include?(:bnode)
-            #limitation only one level BNODE
-            bnode_extraction, patterns, variables = bnode_extraction(collection, incl, klass, patterns, variables)
-          elsif incl.first == :unmapped
-            #a filter with for ?predicate will be included
-            binding_as, unmapped, variables = get_binding_as(patterns, predicates_map)
-          else
-            #make it deterministic
-            incl, incl_embed, variables, graphs, optional_patterns, uri_properties_hash, array_includes_filter =
-              get_includes(collection, graphs, incl, klass, query_options, variables)
-            array_includes_filter, uri_properties_hash = expand_equivalent_predicates_filter(equivalent_predicates,
-                                                                                             array_includes_filter,
-                                                                                             uri_properties_hash)
-          end
+          if incl
+            if incl.first && incl.first.is_a?(Hash) && incl.first.include?(:bnode)
+              #limitation only one level BNODE
+              bnode_extraction, patterns, variables = get_bnode_extraction(collection, incl, klass, patterns)
+            else
+              variables = %i[id attributeProperty attributeObject]
+              if incl.first == :unmapped
+                unmapped = true
+                properties_to_include = predicate_map(predicates)
+              else
+                #make it deterministic
+                incl_embed = get_embed_includes(incl)
+                graphs, properties_to_include, query_options = get_includes(collection, graphs, incl,
+                                                             klass, query_options)
+              end
+            end
           end
-        end
 
 
 
@@ -146,9 +146,7 @@ def get_includes(collection, graphs, incl, klass, query_options)
           [graphs, properties_to_include,query_options]
         end
 
-        def get_binding_as(patterns, predicates) end
-
-        def bnode_extraction(collection, incl, klass, patterns)
+        def get_bnode_extraction(collection, incl, klass, patterns)
           bnode_conf = incl.first[:bnode]
           klass_attr = bnode_conf.keys.first
           bnode_extraction = klass_attr

From 5ea4f092949400c40a8f04e25464bc876226b909 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 19 Jul 2022 14:49:32 +0200
Subject: [PATCH 18/42] use the new expand_equivalent_predicates

---
 lib/goo/sparql/loader.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index a8c3d39d2..a9afbcdc7 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -79,7 +79,7 @@ def model_load_sliced(*options)
             end
           end
 
-
+          expand_equivalent_predicates(properties_to_include, equivalent_predicates)
 
 
           query_builder = Goo::SPARQL::QueryBuilder.new options

From 8028017f0350e89480e092232878a275c2a1d89b Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 19 Jul 2022 14:52:41 +0200
Subject: [PATCH 19/42] add in the query builder initialize instance variables

---
 lib/goo/sparql/query_builder.rb | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index d0c1f7872..058d90cfd 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -6,15 +6,19 @@ class QueryBuilder
       def initialize(options)
         @no_graphs = options[:no_graphs]
         @query_filters = options[:filters]
+        @klass = options[:klass]
         @store = options[:store] || :main
         @page = options[:page]
         @count = options[:count]
         @graph_match = options[:graph_match]
+        @unions = options[:unions] || []
         @aggregate = options[:aggregate]
         @collection = options[:collection]
         @model_query_options = options[:query_options]
         @enable_rules = options[:rules]
-        @unions = []
+        @order_by = options[:order_by]
+
+        @query = get_client
       end
 
       def build_select_query(ids, binding_as, klass, graphs, optional_patterns,
@@ -286,8 +290,10 @@ def get_aggregate_vars(aggregate, collection, graphs, internal_variables, klass,
               optional_patterns.concat(agg_patterns)
             end
           end
-        end
-        return aggregate_projections, aggregate_vars, variables, optional_patterns
+      def get_client
+        Goo.sparql_query_client(@store)
+      end
+
       end
 
       def filter_id_query_strings(collection, graphs, ids, internal_variables, klass, optional_patterns, patterns, query_filters)

From cf1d9ea340d54672cd432f01cda9c8eb5531fe7f Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 19 Jul 2022 14:54:12 +0200
Subject: [PATCH 20/42] update the build_select_query

---
 lib/goo/sparql/loader.rb        |   9 +-
 lib/goo/sparql/query_builder.rb | 454 +++++++++++++++++---------------
 2 files changed, 244 insertions(+), 219 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index a9afbcdc7..a483ef03a 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -81,13 +81,10 @@ def model_load_sliced(*options)
 
           expand_equivalent_predicates(properties_to_include, equivalent_predicates)
 
-
           query_builder = Goo::SPARQL::QueryBuilder.new options
-        select, aggregate_projections =
-          query_builder.build_select_query(ids, binding_as,
-                                           klass, graphs, optional_patterns,
-                                           order_by, patterns, query_options,
-                                           variables, array_includes_filter)
+          select, aggregate_projections = query_builder.build_select_query(ids, variables, graphs,
+                                                                           patterns, query_options,
+                                                                           properties_to_include)
 
         # TODO: remove it? expand_equivalent_predicates_filter does the job now
         expand_equivalent_predicates(select, equivalent_predicates)
diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index 058d90cfd..257e68391 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -21,149 +21,145 @@ def initialize(options)
         @query = get_client
       end
 
-      def build_select_query(ids, binding_as, klass, graphs, optional_patterns,
-                             order_by, patterns, query_options, variables, array_includes_filter)
+      def build_select_query(ids, variables, graphs, patterns,
+                             query_options, properties_to_include)
 
-        internal_variables = graph_match(@collection, @graph_match, graphs, klass, patterns, query_options, @unions)
-        aggregate_projections, aggregate_vars, variables, optional_patterns = get_aggregate_vars(@aggregate, @collection,
-                                                                                                 graphs, internal_variables,
-                                                                                                 klass, optional_patterns,
-                                                                                                 @unions, variables)
-        filter_id_str, query_filter_str = filter_id_query_strings(@collection, graphs, ids, internal_variables,
-                                                                  klass, optional_patterns, patterns, @query_filters)
+        internal_variables = graph_match(@collection, @graph_match, graphs, @klass, patterns, query_options, @unions)
 
+        aggregate_projections, aggregate_vars,
+          variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs,
+                                                            @klass, @unions, variables, internal_variables)
 
-        order_by, variables, patterns = order_by(@count, klass, order_by, patterns, variables)
+        @order_by, variables, patterns = init_order_by(@count, @klass, @order_by, patterns, variables)
+        variables, patterns = add_some_type_to_id(patterns, query_options, variables)
 
-        query_options[:rules] = [:NONE] unless @enable_rules
-        query_options = nil if query_options.empty?
+        query_filter_str, patterns, optional_patterns =
+          filter_query_strings(@collection, graphs, internal_variables, @klass, optional_patterns, patterns, @query_filters)
 
         variables = [] if @count
-
-        variables, patterns = add_some_type_to_id(patterns, query_options, variables)
-
-        select = get_select(aggregate_projections, variables, @store)
         variables.delete :some_type
-        select.where(*patterns)
+
+        select_distinct(variables, aggregate_projections)
+          .from(graphs)
+          .where(patterns)
+          .union_bind_in_where(properties_to_include)
 
         optional_patterns.each do |optional|
-          select.optional(*[optional])
+          @query.optional(*[optional])
         end
-        select.union(*@unions) if @unions.length > 0
-        if order_by
-          order_by_str = order_by.map { |attr, order| "#{order.to_s.upcase}(?#{attr})" }
-          select.order_by(*order_by_str)
-        end
-        select.filter(filter_id_str)
 
-        # Add the included attributes properties to the filter (to retrieve all the attributes we ask for)
-        if !array_includes_filter.nil? && array_includes_filter.length > 0
-          filter_predicates = array_includes_filter.map { |p| "?attributeProperty = #{p.to_ntriples}" }
-          filter_predicates = filter_predicates.join " || "
-          select.filter(filter_predicates)
+        query_filter_str&.each do |filter|
+          @query.filter(filter)
         end
 
-        #if unmapped && predicates && predicates.length > 0
-        #  filter_predicates = predicates.map { |p| "?predicate = #{p.to_ntriples}" }
-        #  filter_predicates = filter_predicates.join " || "
-        #  select.filter(filter_predicates)
-        #end
+        @query.union(*@unions) unless @unions.empty?
 
-        if query_filter_str.length > 0
-          query_filter_str.each do |f|
-            select.filter(f)
-          end
-        end
+        ids_filter(ids) if ids
+        order_by if @order_by # TODO  test if work
 
-        if aggregate_vars
-          select.options[:group_by] = [:id]
-          select.options[:count] = aggregate_vars
-        end
+        put_query_aggregate_vars(aggregate_vars) if aggregate_vars
+        count if @count
+        paginate if @page
 
-        if @count
-          select.options[:count] = [[:id, :count_var, :count]]
-        end
-
-        if @page
-          offset = (@page[:page_i] - 1) * @page[:page_size]
-          select.slice(offset, @page[:page_size])
+        ## TODO see usage of rules and query_options
+        query_options.merge!(@model_query_options) if @model_query_options
+        query_options[:rules] = [:NONE] unless @enable_rules
+        query_options = nil if query_options.empty?
+        if query_options
+          query_options[:rules] = query_options[:rules]&.map { |x| x.to_s }.join('+')
+        else
+          query_options = { rules: ['NONE'] }
         end
+        @query.options[:query_options] = query_options
 
-        select.distinct(true)
+        [@query, aggregate_projections]
+      end
 
-        if query_options && !binding_as
-          query_options[:rules] = query_options[:rules].map { |x| x.to_s }.join("+")
-          select.options[:query_options] = query_options
-        else
-          query_options = { rules: ["NONE"] }
-          select.options[:query_options] = query_options
+      def union_bind_in_where(properties)
+        binding_as = []
+        properties.each do |property_attr, property|
+          predicates = [property[:uri]] + (property[:equivalents] || [])
+          options = {
+            binds: [{ value: property_attr, as: :attributeProperty }]
+          }
+          subject = property[:subject] || :id
+          predicates.uniq.each do |predicate_uri|
+            pattern = if property[:is_inverse]
+                        [:attributeObject, predicate_uri, subject]
+                      else
+                        [subject, predicate_uri, :attributeObject]
+                      end
+            binding_as << [[pattern], options]
+          end
         end
+        @query.optional_union_with_bind_as(*binding_as) unless binding_as.empty?
+        self
+      end
 
-        if !graphs.nil? && graphs.length > 0
-          graphs.select! { |g| g.to_s["owl#Class"].nil? }
-        end
+      def where(patterns)
+        @query.where(*patterns)
+        self
+      end
 
-        unless @no_graphs
-          select.from(graphs.uniq)
-        else
-          select.options[:graphs] = graphs.uniq
-        end
+      def paginate
+        offset = (@page[:page_i] - 1) * @page[:page_size]
+        @query.slice(offset, @page[:page_size])
+        self
+      end
 
-        query_options.merge!(@model_query_options) if @model_query_options
-        if binding_as
-          select.union_with_bind_as(*binding_as)
-        end
-        [select, aggregate_projections]
+      def count
+        @query.options[:count] = [%i[id count_var count]]
+        self
       end
 
+      def put_query_aggregate_vars(aggregate_vars)
+        @query.options[:group_by] = [:id]
+        @query.options[:count] = aggregate_vars
+        self
+      end
 
+      def order_by
+        order_by_str = @order_by.map { |attr, order| "#{order.to_s.upcase}(?#{attr})" }
+        @query.order_by(*order_by_str)
+        self
+      end
 
-      private
+      def from(graphs)
 
-      def order_by(count, klass, order_by, patterns, variables)
-      order_by = nil if count
-      if order_by
-        order_by = order_by.first
-        #simple ordering ... needs to use pattern inspection
-        order_by.each do |attr, direction|
-          quad = query_pattern(klass, attr)
-          patterns << quad[1]
-          #mdorf, 9/22/16 If an ORDER BY clause exists, the columns used in the ORDER BY should be present in the SPARQL select
-          variables << attr unless variables.include?(attr)
+        graphs.select! { |g| g.to_s['owl#Class'].nil? } if !graphs.nil? && !graphs.empty?
+
+        if @no_graphs
+          @query.options[:graphs] = graphs.uniq
+        else
+          @query.from(graphs.uniq)
         end
-      end
-      [order_by, variables, patterns]
+        self
       end
 
+      def select_distinct(variables, aggregate_projections)
 
-      def sparql_op_string(op)
-        case op
-        when :or
-          return "||"
-        when :and
-          return "&&"
-        when :==
-          return "="
-        end
-        return op.to_s
+        select_vars = variables.dup
+        reject_aggregations_from_vars(select_vars, aggregate_projections) if aggregate_projections
+        @query = @query.select(*select_vars).distinct(true)
+        self
       end
 
-      def graph_match(collection, graph_match, graphs, klass, patterns, query_options, unions)
-        internal_variables = []
+      def ids_filter(ids)
+        filter_id = []
 
-        if graph_match
-          #make it deterministic - for caching
-          graph_match_iteration = Goo::Base::PatternIteration.new(graph_match)
-          walk_pattern(klass, graph_match_iteration, graphs, patterns, unions,
-                       internal_variables, in_aggregate = false, query_options, collection)
-          graphs.uniq!
+        ids.each do |id|
+          filter_id << "?id = #{id.to_ntriples.to_s}"
         end
-        internal_variables
+        filter_id_str = filter_id.join ' || '
+        @query.filter filter_id_str
+        self
       end
 
-      def patterns_for_match(klass,attr,value,graphs,patterns,unions,
-                                  internal_variables,subject=:id,in_union=false,
-                                  in_aggregate=false, query_options={}, collection=nil)
+      private
+
+      def patterns_for_match(klass, attr, value, graphs, patterns, unions,
+                             internal_variables, subject = :id, in_union = false,
+                             in_aggregate = false, query_options = {}, collection = nil)
         if value.respond_to?(:each) || value.instance_of?(Symbol)
           next_pattern = value.instance_of?(Array) ? value.first : value
 
@@ -177,9 +173,9 @@ def patterns_for_match(klass,attr,value,graphs,patterns,unions,
           internal_variables << value
         end
 
-        add_rules(attr,klass,query_options)
+        add_rules(attr, klass, query_options)
         graph, pattern =
-          query_pattern(klass,attr,value: value,subject: subject, collection: collection)
+          query_pattern(klass, attr, value: value, subject: subject, collection: collection)
         if pattern
           if !in_union
             patterns << pattern
@@ -190,18 +186,110 @@ def patterns_for_match(klass,attr,value,graphs,patterns,unions,
         graphs << graph if graph
         if next_pattern
           range = klass.range(attr)
-          next_pattern.each do |next_attr,next_value|
+          next_pattern.each do |next_attr, next_value|
             patterns_for_match(range, next_attr, next_value, graphs,
-                               patterns, unions, internal_variables, subject=value,
-                               in_union, in_aggregate, collection=collection)
+                               patterns, unions, internal_variables, subject = value,
+                               in_union, in_aggregate, collection = collection)
+          end
+        end
+      end
+
+      def walk_pattern(klass, match_patterns, graphs, patterns, unions,
+                       internal_variables, in_aggregate = false, query_options = {},
+                       collection)
+        match_patterns.each do |match, in_union|
+          unions << [] if in_union
+          match = match.is_a?(Symbol) ? { match => [] } : match
+          match.each do |attr, value|
+            patterns_for_match(klass, attr, value, graphs, patterns,
+                               unions, internal_variables,
+                               subject = :id, in_union = in_union,
+                               in_aggregate = in_aggregate,
+                               query_options = query_options,
+                               collection)
           end
         end
       end
-      def query_filter_sparql(klass,filter,filter_patterns,filter_graphs,
-                                   filter_operations,
-                                   internal_variables,
-                                   inspected_patterns,
-                                   collection)
+
+      def get_aggregate_vars(aggregate, collection, graphs, klass, unions, variables, internal_variables)
+        # mdorf, 6/03/20 If aggregate projections (sub-SELECT within main SELECT) use an alias, that alias cannot appear in the main SELECT
+        # https://github.com/ncbo/goo/issues/106
+        # See last sentence in https://www.w3.org/TR/sparql11-query/#aggregateExample
+        aggregate_vars = nil
+        aggregate_projections = nil
+        optional_patterns = []
+
+        aggregate&.each do |agg|
+          agg_patterns = []
+          graph_match_iteration =
+            Goo::Base::PatternIteration.new(Goo::Base::Pattern.new(agg.pattern))
+          walk_pattern(klass, graph_match_iteration, graphs, agg_patterns, unions,
+                       internal_variables, in_aggregate = agg.aggregate, collection)
+          unless agg_patterns.empty?
+            projection = "#{internal_variables.last.to_s}_projection".to_sym
+            aggregate_on_attr = internal_variables.last.to_s
+            aggregate_on_attr =
+              aggregate_on_attr[0..aggregate_on_attr.index('_agg_') - 1].to_sym
+            (aggregate_projections ||= {})[projection] = [agg.aggregate, aggregate_on_attr]
+            (aggregate_vars ||= []) << [internal_variables.last,
+                                        projection,
+                                        agg.aggregate]
+            variables << projection
+            optional_patterns.concat(agg_patterns)
+          end
+        end
+        [aggregate_projections, aggregate_vars, variables, optional_patterns]
+      end
+
+      def graph_match(collection, graph_match, graphs, klass, patterns, query_options, unions)
+        internal_variables = []
+
+        if graph_match
+          #make it deterministic - for caching
+          graph_match_iteration = Goo::Base::PatternIteration.new(graph_match)
+          walk_pattern(klass, graph_match_iteration, graphs, patterns, unions,
+                       internal_variables, in_aggregate = false, query_options, collection)
+          graphs.uniq!
+        end
+        internal_variables
+      end
+
+      def get_client
+        Goo.sparql_query_client(@store)
+      end
+
+      def init_order_by(count, klass, order_by, patterns, variables)
+        order_by = nil if count
+        if order_by
+          order_by = order_by.first
+          #simple ordering ... needs to use pattern inspection
+          order_by.each do |attr, direction|
+            quad = query_pattern(klass, attr)
+            patterns << quad[1]
+            #mdorf, 9/22/16 If an ORDER BY clause exists, the columns used in the ORDER BY should be present in the SPARQL select
+            variables << attr unless variables.include?(attr)
+          end
+        end
+        [order_by, variables, patterns]
+      end
+
+      def sparql_op_string(op)
+        case op
+        when :or
+          return '||'
+        when :and
+          return '&&'
+        when :==
+          return '='
+        end
+        op.to_s
+      end
+
+      def query_filter_sparql(klass, filter, filter_patterns, filter_graphs,
+                              filter_operations,
+                              internal_variables,
+                              inspected_patterns,
+                              collection)
         #create a object variable to project the value in the filter
         filter.filter_tree.each do |filter_operation|
           filter_pattern_match = {}
@@ -214,14 +302,22 @@ def query_filter_sparql(klass,filter,filter_patterns,filter_graphs,
             attr = filter_pattern_match.keys.first
             patterns_for_match(klass, attr, filter_pattern_match[attr],
                                filter_graphs, filter_patterns,
-                               [],internal_variables,
-                               subject=:id,in_union=false,in_aggregate=false,
-                               collection=collection)
+                               [], internal_variables,
+                               subject = :id, in_union = false, in_aggregate = false,
+                               collection = collection)
             inspected_patterns[filter_pattern_match] = internal_variables.last
           end
           filter_var = inspected_patterns[filter_pattern_match]
+
           if !filter_operation.value.instance_of?(Goo::Filter)
-            unless filter_operation.operator == :unbound || filter_operation.operator == :bound
+            if filter_operation.operator == :unbound || filter_operation.operator == :bound
+              if filter_operation.operator == :unbound
+                filter_operations << "!BOUND(?#{filter_var.to_s})"
+              else
+                filter_operations << "BOUND(?#{filter_var.to_s})"
+              end
+              return :optional
+            else
               value = RDF::Literal.new(filter_operation.value)
               if filter_operation.value.is_a? String
                 value = RDF::Literal.new(filter_operation.value, :datatype => RDF::XSD.string)
@@ -229,118 +325,50 @@ def query_filter_sparql(klass,filter,filter_patterns,filter_graphs,
               filter_operations << (
                 "?#{filter_var.to_s} #{sparql_op_string(filter_operation.operator)} " +
                   " #{value.to_ntriples}")
-            else
-              if filter_operation.operator == :unbound
-                filter_operations << "!BOUND(?#{filter_var.to_s})"
-              else
-                filter_operations << "BOUND(?#{filter_var.to_s})"
-              end
-              return :optional
             end
           else
             filter_operations << "#{sparql_op_string(filter_operation.operator)}"
-            query_filter_sparql(klass,filter_operation.value,filter_patterns,
-                                filter_graphs,filter_operations,
-                                internal_variables,inspected_patterns,collection)
+            query_filter_sparql(klass, filter_operation.value, filter_patterns,
+                                filter_graphs, filter_operations,
+                                internal_variables, inspected_patterns, collection)
           end
         end
       end
-      def walk_pattern(klass, match_patterns, graphs, patterns, unions,
-                            internal_variables,in_aggregate=false,query_options={},
-                            collection)
-        match_patterns.each do |match,in_union|
-          unions << [] if in_union
-          match = match.is_a?(Symbol) ? { match => [] } : match
-          match.each do |attr,value|
-            patterns_for_match(klass, attr, value, graphs, patterns,
-                               unions,internal_variables,
-                               subject=:id,in_union=in_union,
-                               in_aggregate=in_aggregate,
-                               query_options=query_options,
-                               collection)
-          end
-        end
-      end
-
 
+      def filter_query_strings(collection, graphs, internal_variables, klass,
+                               optional_patterns, patterns,
+                               query_filters)
+        query_filter_str = []
 
-      def get_aggregate_vars(aggregate, collection, graphs, internal_variables, klass, optional_patterns, unions, variables)
-        # mdorf, 6/03/20 If aggregate projections (sub-SELECT within main SELECT) use an alias, that alias cannot appear in the main SELECT
-        # https://github.com/ncbo/goo/issues/106
-        # See last sentence in https://www.w3.org/TR/sparql11-query/#aggregateExample
-        aggregate_vars = nil
-        aggregate_projections = nil
-        if aggregate
-          aggregate.each do |agg|
-            agg_patterns = []
-            graph_match_iteration =
-              Goo::Base::PatternIteration.new(Goo::Base::Pattern.new(agg.pattern))
-            walk_pattern(klass, graph_match_iteration, graphs, agg_patterns, unions,
-                         internal_variables, in_aggregate = agg.aggregate, collection)
-            if agg_patterns.length > 0
-              projection = "#{internal_variables.last.to_s}_projection".to_sym
-              aggregate_on_attr = internal_variables.last.to_s
-              aggregate_on_attr =
-                aggregate_on_attr[0..aggregate_on_attr.index("_agg_") - 1].to_sym
-              (aggregate_projections ||= {})[projection] = [agg.aggregate, aggregate_on_attr]
-              (aggregate_vars ||= []) << [internal_variables.last,
-                                          projection,
-                                          agg.aggregate]
-              variables << projection
-              optional_patterns.concat(agg_patterns)
+        filter_patterns = []
+        filter_graphs = []
+        inspected_patterns = {}
+        query_filters&.each do |query_filter|
+          filter_operations = []
+          type = query_filter_sparql(klass, query_filter, filter_patterns, filter_graphs,
+                                     filter_operations, internal_variables,
+                                     inspected_patterns, collection)
+          query_filter_str << filter_operations.join(' ')
+          graphs.concat(filter_graphs) unless filter_graphs.empty?
+          unless filter_patterns.empty?
+            if type == :optional
+              optional_patterns.concat(filter_patterns)
+            else
+              patterns.concat(filter_patterns)
             end
           end
-      def get_client
-        Goo.sparql_query_client(@store)
-      end
-
-      end
-
-      def filter_id_query_strings(collection, graphs, ids, internal_variables, klass, optional_patterns, patterns, query_filters)
-        filter_id = []
-        if ids
-          ids.each do |id|
-            filter_id << "?id = #{id.to_ntriples.to_s}"
-          end
         end
-        filter_id_str = filter_id.join " || "
 
-        query_filter_str = []
-        if query_filters
-          filter_patterns = []
-          filter_graphs = []
-          inspected_patterns = {}
-          query_filters.each do |query_filter|
-            filter_operations = []
-            type = query_filter_sparql(klass, query_filter, filter_patterns, filter_graphs,
-                                       filter_operations, internal_variables,
-                                       inspected_patterns, collection)
-            query_filter_str << filter_operations.join(" ")
-            graphs.concat(filter_graphs) if filter_graphs.length > 0
-            if filter_patterns.length > 0
-              if type == :optional
-                optional_patterns.concat(filter_patterns)
-              else
-                patterns.concat(filter_patterns)
-              end
-            end
-          end
-        end
-        return filter_id_str, query_filter_str
+        [query_filter_str, patterns, optional_patterns, internal_variables]
       end
 
-
-      def get_select(aggregate_projections, variables, store)
-        client = Goo.sparql_query_client(store)
-        select_vars = variables.dup
-        select_vars.reject! { |var| aggregate_projections.key?(var) } if aggregate_projections
-        client.select(*select_vars).distinct()
+      def reject_aggregations_from_vars(variables, aggregate_projections)
+        variables.reject! { |var| aggregate_projections.key?(var) }
       end
 
-
       def add_some_type_to_id(patterns, query_options, variables)
         #rdf:type <x> breaks the reasoner
-        if query_options && query_options[:rules] != [:NONE]
+        if query_options && !query_options.empty? && query_options[:rules] != [:NONE]
           patterns[0] = [:id, RDF[:type], :some_type]
           variables << :some_type
         end

From a9161b4562991b8c9e8ef135931098a38099da3d Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 19 Jul 2022 14:56:03 +0200
Subject: [PATCH 21/42] add to the solution mapper initialize some instance
 variable

---
 lib/goo/sparql/solutions_mapper.rb | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index c8575846c..01f52cc49 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -6,7 +6,7 @@ class SolutionMapper
 
       def initialize(aggregate_projections, bnode_extraction, embed_struct,
                      incl_embed, klass_struct, models_by_id,
-                     predicates_map, unmapped, variables, uri_properties_hash, options)
+                     properties_to_include, unmapped, variables,ids, options)
 
         @aggregate_projections = aggregate_projections
         @bnode_extraction = bnode_extraction
@@ -14,21 +14,20 @@ def initialize(aggregate_projections, bnode_extraction, embed_struct,
         @incl_embed = incl_embed
         @klass_struct = klass_struct
         @models_by_id = models_by_id
-        @predicates_map = predicates_map
+        @properties_to_include = properties_to_include
         @unmapped = unmapped
         @variables = variables
-        @options = options
-        @uri_properties_hash = uri_properties_hash
-
+        @ids = ids
+        @klass = options[:klass]
+        @klass = options[:klass]
+        @read_only = options[:read_only]
+        @incl = options[:include]
+        @count = options[:count]
+        @collection = options[:collection]
       end
 
       def map_each_solutions(select)
 
-        count = @options[:count]
-        klass = @options[:klass]
-        read_only = @options[:read_only]
-        collection = @options[:collection]
-        incl = @options[:include]
 
         found = Set.new
         objects_new = {}

From f4dc3397e5eebf520d43c9bf7a72a36483b4a3f9 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 19 Jul 2022 14:57:41 +0200
Subject: [PATCH 22/42] update map_each_solutions

---
 lib/goo/sparql/loader.rb           |  12 +-
 lib/goo/sparql/solutions_mapper.rb | 456 +++++++++++++++--------------
 2 files changed, 240 insertions(+), 228 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index a483ef03a..821aba260 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -86,12 +86,10 @@ def model_load_sliced(*options)
                                                                            patterns, query_options,
                                                                            properties_to_include)
 
-        # TODO: remove it? expand_equivalent_predicates_filter does the job now
-        expand_equivalent_predicates(select, equivalent_predicates)
-        solution_mapper = Goo::SPARQL::SolutionMapper.new aggregate_projections, bnode_extraction,
-                                                          embed_struct, incl_embed, klass_struct, models_by_id,
-                                                          predicates_map, unmapped,
-                                                          variables, uri_properties_hash, options
+          solution_mapper = Goo::SPARQL::SolutionMapper.new aggregate_projections, bnode_extraction,
+                                                            embed_struct, incl_embed, klass_struct, models_by_id,
+                                                            properties_to_include, unmapped,
+                                                            variables, ids, options
 
           solution_mapper.map_each_solutions(select)
         end
@@ -107,7 +105,7 @@ def expand_equivalent_predicates(properties_to_include, eq_p)
             property[:equivalents] = eq_p[property_uri.to_s].to_a.map { |p| RDF::URI.new(p) } if eq_p.include?(property_uri.to_s)
           end
 
-      end
+        end
 
         def predicate_map(predicates)
           predicates_map = nil
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 01f52cc49..877a40d88 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -32,215 +32,234 @@ def map_each_solutions(select)
         found = Set.new
         objects_new = {}
         var_set_hash = {}
-        list_attributes = Set.new(klass.attributes(:list))
-        all_attributes = Set.new(klass.attributes(:all))
-        id_array = []
+        list_attributes = Set.new(@klass.attributes(:list))
+        all_attributes = Set.new(@klass.attributes(:all))
 
+        
         select.each_solution do |sol|
-          next if sol[:some_type] && klass.type_uri(collection) != sol[:some_type]
-
-          return sol[:count_var].object if count
+          next if sol[:some_type] && @klass.type_uri(@collection) != sol[:some_type]
+          return sol[:count_var].object if @count
 
           found.add(sol[:id])
           id = sol[:id]
-          id_array << id ## TODO same as "found"
+
+          create_model(id)
 
           if @bnode_extraction
-            struct = create_struct(@bnode_extraction, klass, @models_by_id, sol, @variables)
-            @models_by_id[id].send("#{@bnode_extraction}=", struct)
+            add_bnode_to_model(sol)
             next
           end
 
-          @models_by_id[id] = create_class_model(id, klass, @klass_struct) unless @models_by_id.include?(id)
-
           if @unmapped
-            if @predicates_map.nil?
-              model_set_unmapped(@models_by_id, sol)
-            else
-              model_set_unmapped_with_predicates_map(@models_by_id, @predicates_map, sol)
-            end
+            add_unmapped_to_model(sol)
             next
           end
 
-          # Retrieve aggregates count
-          @aggregate_projections&.each do |aggregate_key, aggregate_val|
-            if @models_by_id[id].respond_to?(:add_aggregate)
-              @models_by_id[id].add_aggregate(aggregate_val[1], aggregate_val[0], sol[aggregate_key].object)
-            else
-              (@models_by_id[id].aggregates ||= []) << Goo::Base::AGGREGATE_VALUE.new(aggregate_val[1],
-                                                                                      aggregate_val[0],
-                                                                                      sol[aggregate_key].object)
-            end
+          if @aggregate_projections
+            add_aggregations_to_model(sol)
+            next
           end
 
-          next if sol[:attributeProperty].nil?
-
-          # Retrieve all included attributes
-          object = if !sol[:attributeObject].nil?
-                     sol[:attributeObject]
-                   elsif !sol[:inverseAttributeObject].nil?
-                     sol[:inverseAttributeObject]
-                   end
-
-          # Get the property label using the hash
+          v = sol[:attributeProperty].to_s.to_sym
 
-          v = @uri_properties_hash[sol[:attributeProperty]]
+          next if v.nil? || !all_attributes.include?(v)
 
-          next if v.nil? || ((v != :id) && !all_attributes.include?(v))
+          object = sol[:attributeObject]
 
-          #group for multiple values
           #bnodes
-          if object.kind_of?(RDF::Node) && object.anonymous? && incl.include?(v)
-            range = klass.range(v)
-            if range.respond_to?(:new)
-              objects_new[object] = BNODES_TUPLES.new(id, v)
-            end
+          if bnode_id?(object, v)
+            objects_new = bnode_id_tuple(id, object, objects_new, v)
             next
           end
 
-          if object and !(object.kind_of? RDF::URI)
-            object = object.object
-          end
+          object, objects_new = get_value_object(id, objects_new, object, list_attributes, v)
+          add_object_to_model(id, object, v, var_set_hash)
+        end
 
-          #dependent model creation
-          if object.kind_of?(RDF::URI) && v != :id
-            range_for_v = klass.range(v)
-            if range_for_v
-              if objects_new.include?(object)
-                object = objects_new[object]
-              else
-                unless range_for_v.inmutable?
-                  pre_val = nil
-                  if @models_by_id[id] &&
-                    ((@models_by_id[id].respond_to?(:klass) && models_by_id[id]) ||
-                      @models_by_id[id].loaded_attributes.include?(v))
-                    pre_val = if !read_only
-                                @models_by_id[id].instance_variable_get("@#{v}")
-                              else
-                                @models_by_id[id][v]
-                              end
-                    if pre_val.is_a?(Array)
-                      pre_val = pre_val.select { |x| x.id == object }.first
-                    end
-                  end
-                  if !read_only
-                    object = pre_val ? pre_val : klass.range_object(v, object)
-                    objects_new[object.id] = object
-                  else
-                    #depedent read only
-                    struct = pre_val ? pre_val : @embed_struct[v].new
-                    struct.id = object
-                    struct.klass = klass.range(v)
-                    objects_new[struct.id] = struct
-                    object = struct
-                  end
-                else
-                  object = range_for_v.find(object).first
-                end
-              end
-            end
-          end
+        init_unloaded_attributes(found, list_attributes)
+
+        return @models_by_id if @bnode_extraction
 
-          if list_attributes.include?(v)
-            # To handle attr that are lists
-            pre = @klass_struct ? @models_by_id[id][v] :
-                    @models_by_id[id].instance_variable_get("@#{v}")
-            if object.nil? && pre.nil?
-              object = []
-            elsif object.nil? && !pre.nil?
-              object = pre
-            elsif object
-              object = !pre ? [object] : (pre.dup << object)
-              object.uniq!
+        model_set_collection_attributes(@models_by_id, objects_new)
+
+        #remove from models_by_id elements that were not touched
+        @models_by_id.select! { |k, m| found.include?(k) }
+
+        models_set_all_persistent(@models_by_id) unless @read_only
+
+        #next level of embed attributes
+        include_embed_attributes(@incl_embed, objects_new) if @incl_embed && !@incl_embed.empty?
+
+        #bnodes
+        blank_nodes = objects_new.select { |id, obj| id.is_a?(RDF::Node) && id.anonymous? }
+        include_bnodes(blank_nodes, @models_by_id) unless blank_nodes.empty?
+
+        models_unmapped_to_array(@models_by_id) if @unmapped
+
+        @models_by_id
+      end
+
+      private
+
+      def init_unloaded_attributes(found, list_attributes)
+        return if @incl.nil?
+
+        # Here we are setting to nil all attributes that have been included but not found in the triplestore
+        found.uniq.each do |model_id|
+          m = @models_by_id[model_id]
+          @incl.each do |attr_to_incl|
+            is_handler = m.respond_to?(:handler?) && m.class.handler?(attr_to_incl)
+            next if attr_to_incl.to_s.eql?('unmapped') || is_handler
+
+            loaded = m.respond_to?('loaded_attributes') && m.loaded_attributes.include?(attr_to_incl)
+            is_list = list_attributes.include?(attr_to_incl)
+            is_struct = m.respond_to?(:klass)
+
+            # Go through all models queried
+            if is_struct
+              m[attr_to_incl] = [] if is_list && m[attr_to_incl].nil?
+            elsif is_list && (!loaded || m.send(attr_to_incl.to_s).nil?)
+              m.send("#{attr_to_incl}=", [], on_load: true)
+            elsif !loaded && !is_list && m.respond_to?("#{attr_to_incl}=")
+              m.send("#{attr_to_incl}=", nil, on_load: true)
             end
           end
+        end
+      end
 
-          if @models_by_id[id].respond_to?(:klass)
-            unless object.nil? && !@models_by_id[id][v].nil?
-              @models_by_id[id][v] = object
-            end
+      def get_value_object(id, objects_new, object, list_attributes, predicate)
+        object = object.object if object && !(object.is_a? RDF::URI)
+        range_for_v = @klass.range(predicate)
+        #binding.pry if v.eql?(:enrolled)
+        #dependent model creation
+
+        if object.is_a?(RDF::URI) && (predicate != :id) && !range_for_v.nil?
+          if objects_new.include?(object)
+            object = objects_new[object]
+          elsif !range_for_v.inmutable?
+            pre_val = get_preload_value(id, object, predicate)
+            object, objects_new = if !@read_only
+                                    preloaded_or_new_object(object, objects_new, pre_val, predicate)
+                                  else
+                                    #depedent read only
+                                    preloaded_or_new_struct(object, objects_new, pre_val, predicate)
+                                  end
           else
-            unless @models_by_id[id].class.handler?(v)
-              unless object.nil? && !@models_by_id[id].instance_variable_get("@#{v.to_s}").nil?
-                if v != :id
-                  # if multiple language values are included for a given property, set the
-                  # corresponding model attribute to the English language value - NCBO-1662
-                  if object.kind_of?(RDF::Literal)
-                    key = "#{v}#__#{id.to_s}"
-                    @models_by_id[id].send("#{v}=", object, on_load: true) unless var_set_hash[key]
-                    lang = object.language
-                    var_set_hash[key] = true if lang == :EN || lang == :en
-                  else
-                    @models_by_id[id].send("#{v}=", object, on_load: true)
-                  end
-                end
-              end
-            end
+            object = range_for_v.find(object).first
           end
-
         end
-        unless incl.nil?
-          # Here we are setting to nil all attributes that have been included but not found in the triplestore
-          id_array.uniq!
-          incl.each do |attr_to_incl|
-            # Go through all attr we had to include
-            next if attr_to_incl.is_a? Hash
-
-            id_array.each do |model_id|
-              # Go through all models queried
-              if @models_by_id[model_id].respond_to?("loaded_attributes") && !@models_by_id[model_id].loaded_attributes.include?(attr_to_incl) && @models_by_id[model_id].respond_to?(attr_to_incl) && !attr_to_incl.to_s.eql?("unmapped")
-                if list_attributes.include?(attr_to_incl)
-                  # If the asked attr has not been loaded then it is set to nil or to an empty array for list attr
-                  @models_by_id[model_id].send("#{attr_to_incl}=", [], on_load: true)
+
+        if list_attributes.include?(predicate)
+          # To handle attr that are lists
+          pre = if @klass_struct
+                  @models_by_id[id][predicate]
                 else
-                  @models_by_id[model_id].send("#{attr_to_incl}=", nil, on_load: true)
+                  @models_by_id[id].instance_variable_get("@#{predicate}")
                 end
-              end
-            end
+          if object.nil? && pre.nil?
+            object = []
+          elsif object.nil? && !pre.nil?
+            object = pre
+          elsif object
+            object = !pre ? [object] : (pre.dup << object)
+            object.uniq!
+          end
+        end
+        [object,objects_new]
+      end
 
+      def add_object_to_model(id, object, predicate, var_set_hash)
+        if @models_by_id[id].respond_to?(:klass)
+          @models_by_id[id][predicate] = object unless object.nil? && !@models_by_id[id][predicate].nil?
+        elsif !@models_by_id[id].class.handler?(predicate) &&
+          !(object.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
+          predicate != :id
+          # if multiple language values are included for a given property, set the
+          # corresponding model attribute to the English language value - NCBO-1662
+          if object.is_a?(RDF::Literal)
+            key = "#{predicate}#__#{id}"
+            @models_by_id[id].send("#{predicate}=", object, on_load: true) unless var_set_hash[key]
+            lang = object.language
+            var_set_hash[key] = true if %i[EN en].include?(lang)
+          else
+            @models_by_id[id].send("#{predicate}=", object, on_load: true)
           end
         end
+      end
 
-        return @models_by_id if @bnode_extraction
+      def get_preload_value(id, object, predicate)
+        pre_val = nil
+        if predicate_preloaded?(id, predicate)
+          pre_val = preloaded_value(id, predicate)
+          pre_val = pre_val.select { |x| x.id == object }.first if pre_val.is_a?(Array)
+        end
+        pre_val
+      end
 
-        model_set_collection_attributes(collection, klass, @models_by_id, objects_new)
+      def preloaded_or_new_object(object, objects_new, pre_val, predicate)
+        object = pre_val || @klass.range_object(predicate, object)
+        objects_new[object.id] = object
+        [object, objects_new]
+      end
 
-        #remove from models_by_id elements that were not touched
-        @models_by_id.select! { |k, m| found.include?(k) }
+      def preloaded_or_new_struct(object, objects_new, pre_val, predicate)
+        struct = pre_val || @embed_struct[predicate].new
+        struct.id = object
+        struct.klass = @klass.range(predicate)
+        objects_new[struct.id] = struct
+        [struct, objects_new]
+      end
 
-        models_set_all_persistent(@models_by_id, @options) unless read_only
+      def preloaded_value(id, predicate)
+        if !@read_only
+          @models_by_id[id].instance_variable_get("@#{predicate}")
+        else
+          @models_by_id[id][predicate]
+        end
+      end
 
-        #next level of embed attributes
-        include_embed_attributes(collection, @incl_embed, klass, objects_new) if @incl_embed && !@incl_embed.empty?
+      def predicate_preloaded?(id, predicate)
+        @models_by_id[id] &&
+          (@models_by_id[id].respond_to?(:klass) || @models_by_id[id].loaded_attributes.include?(predicate))
+      end
 
-        #bnodes
-        bnodes = objects_new.select { |id, obj| id.is_a?(RDF::Node) && id.anonymous? }
-        include_bnodes(bnodes, collection, klass, @models_by_id) unless bnodes.empty?
+      def bnode_id?(object, predicate)
+        object.is_a?(RDF::Node) && object.anonymous? && @incl.include?(predicate)
+      end
 
-        models_unmapped_to_array(@models_by_id) if @unmapped
+      def bnode_id_tuple(id, object, objects_new, predicate)
+        range = @klass.range(predicate)
+        if range.respond_to?(:new)
+          objects_new[object] = BNODES_TUPLES.new(id, predicate)
+        end
+        objects_new
+      end
 
-        @models_by_id
+      def add_bnode_to_model(sol)
+        id = sol[:id]
+        struct = create_struct(@bnode_extraction, @models_by_id, sol, @variables)
+        @models_by_id[id].send("#{@bnode_extraction}=", struct)
       end
 
-      private
+      def create_model(id)
+        @models_by_id[id] = create_class_model(id, @klass, @klass_struct) unless @models_by_id.include?(id)
+      end
 
-      def model_set_unmapped(models_by_id, sol)
-        id = sol[:id]
-        if models_by_id[id].respond_to? :klass #struct
-          models_by_id[id][:unmapped] ||= {}
-          (models_by_id[id][:unmapped][sol[:predicate]] ||= []) << sol[:object]
+      def model_set_unmapped(id, predicate, value)
+
+        if @models_by_id[id].respond_to? :klass #struct
+          @models_by_id[id][:unmapped] ||= {}
+          (@models_by_id[id][:unmapped][predicate] ||= []) << value
         else
-          models_by_id[id].unmapped_set(sol[:predicate], sol[:object])
+          @models_by_id[id].unmapped_set(predicate, value)
         end
       end
 
-
-      def create_struct(bnode_extraction, klass, models_by_id, sol, variables)
-        list_attributes = Set.new(klass.attributes(:list))
-        struct = klass.range(bnode_extraction).new
+      def create_struct(bnode_extraction, models_by_id, sol, variables)
+        list_attributes = Set.new(@klass.attributes(:list))
+        struct = @klass.range(bnode_extraction).new
         variables.each do |v|
           next if v == :id
-
           svalue = sol[v]
           struct[v] = svalue.is_a?(RDF::Node) ? svalue : svalue.object
         end
@@ -259,17 +278,18 @@ def create_class_model(id, klass, klass_struct)
         klass_model.klass = klass if klass_struct
         klass_model
       end
+
       def models_unmapped_to_array(models_by_id)
         models_by_id.each do |idm, m|
           m.unmmaped_to_array
         end
       end
 
-      def include_bnodes(bnodes, collection, klass, models_by_id)
+      def include_bnodes(bnodes, models_by_id)
         #group by attribute
         attrs = bnodes.map { |x, y| y.attribute }.uniq
         attrs.each do |attr|
-          struct = klass.range(attr)
+          struct = @klass.range(attr)
 
           #bnodes that are in a range of goo ground models
           #for example parents and children in LD class models
@@ -278,39 +298,38 @@ def include_bnodes(bnodes, collection, klass, models_by_id)
 
           bnode_attrs = struct.new.to_h.keys
           ids = bnodes.select { |x, y| y.attribute == attr }.map { |x, y| y.id }
-          klass.where.models(models_by_id.select { |x, y| ids.include?(x) }.values)
-               .in(collection)
+          @klass.where.models(models_by_id.select { |x, y| ids.include?(x) }.values)
+               .in(@collection)
                .include(bnode: { attr => bnode_attrs }).all
         end
       end
 
-      def include_embed_attributes(collection, incl_embed, klass, objects_new)
+      def include_embed_attributes(incl_embed, objects_new)
         incl_embed.each do |attr, next_attrs|
           #anything to join ?
-          attr_range = klass.range(attr)
+          attr_range = @klass.range(attr)
           next if attr_range.nil?
           range_objs = objects_new.select { |id, obj|
             obj.instance_of?(attr_range) || (obj.respond_to?(:klass) && obj[:klass] == attr_range)
           }.values
           unless range_objs.empty?
             range_objs.uniq!
-            attr_range.where().models(range_objs).in(collection).include(*next_attrs).all
+            attr_range.where().models(range_objs).in(@collection).include(*next_attrs).all
           end
         end
       end
 
-      def models_set_all_persistent(models_by_id, options)
-        if options[:ids] #newly loaded
-          models_by_id.each do |k, m|
-            m.persistent = true
-          end
+      def models_set_all_persistent(models_by_id)
+        return unless @ids
+        models_by_id.each do |k, m|
+          m.persistent = true
         end
       end
 
-      def model_set_collection_attributes(collection, klass, models_by_id, objects_new)
-        collection_value = get_collection_value(collection, klass)
+      def model_set_collection_attributes(models_by_id, objects_new)
+        collection_value = get_collection_value
         if collection_value
-          collection_attribute = klass.collection_opts
+          collection_attribute = @klass.collection_opts
           models_by_id.each do |id, m|
             m.send("#{collection_attribute}=", collection_value)
           end
@@ -327,14 +346,14 @@ def model_set_collection_attributes(collection, klass, models_by_id, objects_new
         end
       end
 
-      def get_collection_value(collection, klass)
+      def get_collection_value
         collection_value = nil
-        if klass.collection_opts.instance_of?(Symbol)
-          if collection.is_a?(Array) && (collection.length == 1)
-            collection_value = collection.first
+        if @klass.collection_opts.instance_of?(Symbol)
+          if @collection.is_a?(Array) && (@collection.length == 1)
+            collection_value = @collection.first
           end
-          if collection.respond_to? :id
-            collection_value = collection
+          if @collection.respond_to? :id
+            collection_value = @collection
           end
         end
         collection_value
@@ -348,11 +367,11 @@ def model_map_attributes_values(id, var_set_hash, models_by_id, object, sol, v)
           if (!models_by_id[id].class.handler?(v) || model_attribute_val.nil?) && v != :id
             # if multiple language values are included for a given property, set the
             # corresponding model attribute to the English language value - NCBO-1662
-            if sol[v].kind_of?(RDF::Literal)
+            if sol[v].is_a?(RDF::Literal)
               key = "#{v}#__#{id.to_s}"
               models_by_id[id].send("#{v}=", object, on_load: true) unless var_set_hash[key]
               lang = sol[v].language
-              var_set_hash[key] = true if lang == :EN || lang == :en
+              var_set_hash[key] = true if %i[EN en EN en EN en].include?(lang)
             else
               models_by_id[id].send("#{v}=", object, on_load: true)
             end
@@ -360,9 +379,12 @@ def model_map_attributes_values(id, var_set_hash, models_by_id, object, sol, v)
         end
       end
 
-      def object_to_array(id, klass_struct, models_by_id, object, v)
-        pre = klass_struct ? models_by_id[id][v] :
-                models_by_id[id].instance_variable_get("@#{v}")
+      def object_to_array(id, klass_struct, models_by_id, object, predicate)
+        pre = if klass_struct
+                models_by_id[id][predicate]
+              else
+                models_by_id[id].instance_variable_get("@#{predicate}")
+              end
         if object.nil? && pre.nil?
           object = []
         elsif object.nil? && !pre.nil?
@@ -375,10 +397,10 @@ def object_to_array(id, klass_struct, models_by_id, object, v)
       end
 
       def dependent_model_creation(embed_struct, id, models_by_id, object, objects_new, v, options)
-        klass = options[:klass]
+
         read_only = options[:read_only]
-        if object.kind_of?(RDF::URI) && v != :id
-          range_for_v = klass.range(v)
+        if object.is_a?(RDF::URI) && v != :id
+          range_for_v = @klass.range(v)
           if range_for_v
             if objects_new.include?(object)
               object = objects_new[object]
@@ -393,16 +415,15 @@ def dependent_model_creation(embed_struct, id, models_by_id, object, objects_new
         object
       end
 
-      def get_object_from_range(pre_val, embed_struct, object, objects_new, v, options)
-        klass = options[:klass]
-        read_only = options[:read_only]
-        range_for_v = klass.range(v)
-        if !read_only
-          object = pre_val || klass.range_object(v, object)
+      def get_object_from_range(pre_val, embed_struct, object, objects_new, predicate)
+
+        range_for_v = @klass.range(predicate)
+        if !@read_only
+          object = pre_val || @klass.range_object(predicate, object)
           objects_new[object.id] = object
         else
           #depedent read only
-          struct = pre_val || embed_struct[v].new
+          struct = pre_val || embed_struct[predicate].new
           struct.id = object
           struct.klass = range_for_v
           objects_new[struct.id] = struct
@@ -411,53 +432,46 @@ def get_object_from_range(pre_val, embed_struct, object, objects_new, v, options
         object
       end
 
-      def get_pre_val(id, models_by_id, object, v, read_only)
+      def get_pre_val(id, models_by_id, object, predicate)
         pre_val = nil
         if models_by_id[id] &&
           ((models_by_id[id].respond_to?(:klass) && models_by_id[id]) ||
-            models_by_id[id].loaded_attributes.include?(v))
-          if !read_only
-            pre_val = models_by_id[id].instance_variable_get("@#{v}")
-          else
-            pre_val = models_by_id[id][v]
-          end
+            models_by_id[id].loaded_attributes.include?(predicate))
+          pre_val = if !@read_only
+                      models_by_id[id].instance_variable_get("@#{predicate}")
+                    else
+                      models_by_id[id][predicate]
+                    end
 
           pre_val = pre_val.select { |x| x.id == object }.first if pre_val.is_a?(Array)
         end
         pre_val
       end
 
-      def initialize_object(id, klass, object, objects_new, v)
-        range = klass.range(v)
-        objects_new[object] = BNODES_TUPLES.new(id, v) if range.respond_to?(:new)
-      end
 
-      def model_add_aggregation(aggregate_projections, models_by_id, sol, v)
+      def add_unmapped_to_model(sol)
+        predicate = sol[:attributeProperty].to_s.to_sym
+        return unless @properties_to_include[predicate]
+
         id = sol[:id]
-        if aggregate_projections && aggregate_projections.include?(v)
-          conf = aggregate_projections[v]
-          if models_by_id[id].respond_to?(:add_aggregate)
-            models_by_id[id].add_aggregate(conf[1], conf[0], sol[v].object)
-          else
-            (models_by_id[id].aggregates ||= []) <<
-              Goo::Base::AGGREGATE_VALUE.new(conf[1], conf[0], sol[v].object)
-          end
-        end
+        value = sol[:attributeObject]
+
+        model_set_unmapped(id, @properties_to_include[predicate][:uri], value)
       end
 
-      def model_set_unmapped_with_predicates_map(models_by_id, predicates_map, sol)
+      def add_aggregations_to_model(sol)
         id = sol[:id]
-        no_graphs = sol[:bind_as].to_s.to_sym
-        if predicates_map.include?(no_graphs)
-          pred = predicates_map[no_graphs]
-          if models_by_id[id].respond_to? :klass #struct
-            models_by_id[id][:unmapped] ||= {}
-            (models_by_id[id][:unmapped][pred] ||= Set.new) << sol[:object]
+        @aggregate_projections&.each do |aggregate_key, aggregate_val|
+          if @models_by_id[id].respond_to?(:add_aggregate)
+            @models_by_id[id].add_aggregate(aggregate_val[1], aggregate_val[0], sol[aggregate_key].object)
           else
-            models_by_id[id].unmapped_set(pred, sol[:object])
+            (@models_by_id[id].aggregates ||= []) << Goo::Base::AGGREGATE_VALUE.new(aggregate_val[1],
+                                                                                    aggregate_val[0],
+                                                                                    sol[aggregate_key].object)
           end
         end
       end
     end
   end
 end
+

From 6cf7320b3616e0ee765ff24c1624106dc8899792 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 23 Nov 2022 09:26:30 +0100
Subject: [PATCH 23/42] update order by query to use unions so that it can be
 optional

---
 lib/goo/sparql/query_builder.rb | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index 257e68391..b540622c0 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -30,7 +30,7 @@ def build_select_query(ids, variables, graphs, patterns,
           variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs,
                                                             @klass, @unions, variables, internal_variables)
 
-        @order_by, variables, patterns = init_order_by(@count, @klass, @order_by, patterns, variables)
+        @order_by, variables, @unions = init_order_by(@count, @klass, @order_by, @unions, variables)
         variables, patterns = add_some_type_to_id(patterns, query_options, variables)
 
         query_filter_str, patterns, optional_patterns =
@@ -258,19 +258,20 @@ def get_client
         Goo.sparql_query_client(@store)
       end
 
-      def init_order_by(count, klass, order_by, patterns, variables)
+      def init_order_by(count, klass, order_by, unions, variables)
         order_by = nil if count
         if order_by
           order_by = order_by.first
           #simple ordering ... needs to use pattern inspection
           order_by.each do |attr, direction|
             quad = query_pattern(klass, attr)
-            patterns << quad[1]
+            unions << [quad[1]]
+            #patterns << quad[1]
             #mdorf, 9/22/16 If an ORDER BY clause exists, the columns used in the ORDER BY should be present in the SPARQL select
             variables << attr unless variables.include?(attr)
           end
         end
-        [order_by, variables, patterns]
+        [order_by, variables, unions]
       end
 
       def sparql_op_string(op)

From dce68a7be662cf0e3477bf24064be16b1beeb588 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 29 Nov 2022 05:22:18 +0100
Subject: [PATCH 24/42] use optional  statements for order by instead of UNION

---
 lib/goo/sparql/query_builder.rb | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index b540622c0..a3c35a432 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -30,7 +30,7 @@ def build_select_query(ids, variables, graphs, patterns,
           variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs,
                                                             @klass, @unions, variables, internal_variables)
 
-        @order_by, variables, @unions = init_order_by(@count, @klass, @order_by, @unions, variables)
+        @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables)
         variables, patterns = add_some_type_to_id(patterns, query_options, variables)
 
         query_filter_str, patterns, optional_patterns =
@@ -71,7 +71,6 @@ def build_select_query(ids, variables, graphs, patterns,
           query_options = { rules: ['NONE'] }
         end
         @query.options[:query_options] = query_options
-
         [@query, aggregate_projections]
       end
 
@@ -258,20 +257,21 @@ def get_client
         Goo.sparql_query_client(@store)
       end
 
-      def init_order_by(count, klass, order_by, unions, variables)
+      def init_order_by(count, klass, order_by, optional_patterns, variables)
         order_by = nil if count
         if order_by
           order_by = order_by.first
           #simple ordering ... needs to use pattern inspection
           order_by.each do |attr, direction|
             quad = query_pattern(klass, attr)
-            unions << [quad[1]]
+            optional_patterns << quad[1]
             #patterns << quad[1]
             #mdorf, 9/22/16 If an ORDER BY clause exists, the columns used in the ORDER BY should be present in the SPARQL select
-            variables << attr unless variables.include?(attr)
+            #variables << attr unless variables.include?(attr)
           end
+          variables = %i[id attributeProperty attributeObject]
         end
-        [order_by, variables, unions]
+        [order_by, variables, optional_patterns]
       end
 
       def sparql_op_string(op)

From 352f3ca831111069127147b65f34362b110ace38 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 17 Dec 2022 16:02:58 +0100
Subject: [PATCH 25/42] fix embed ready only models

---
 lib/goo/sparql/solutions_mapper.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 877a40d88..3e6954f38 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -314,7 +314,9 @@ def include_embed_attributes(incl_embed, objects_new)
           }.values
           unless range_objs.empty?
             range_objs.uniq!
-            attr_range.where().models(range_objs).in(@collection).include(*next_attrs).all
+            query = attr_range.where().models(range_objs).in(@collection).include(*next_attrs)
+            query = query.read_only if @read_only
+            query.all
           end
         end
       end

From 3162ee8db79516dba8f92c399f3b85b80d2aa422 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 19 Jan 2023 11:14:17 +0100
Subject: [PATCH 26/42] fix requests with multiple filters

---
 lib/goo/sparql/query_builder.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index a3c35a432..7ef72193e 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -341,11 +341,11 @@ def filter_query_strings(collection, graphs, internal_variables, klass,
                                query_filters)
         query_filter_str = []
 
-        filter_patterns = []
         filter_graphs = []
         inspected_patterns = {}
         query_filters&.each do |query_filter|
           filter_operations = []
+          filter_patterns = []
           type = query_filter_sparql(klass, query_filter, filter_patterns, filter_graphs,
                                      filter_operations, internal_variables,
                                      inspected_patterns, collection)

From 1b26d2bdc0fd53ea794efd1ed4806b52cb6183a5 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 19 Jan 2023 11:17:37 +0100
Subject: [PATCH 27/42] use ontoportal-lirmm sparql client version to pass
 tests

---
 Gemfile      |  2 +-
 Gemfile.lock | 48 ++++++++++++++++++++++++++----------------------
 2 files changed, 27 insertions(+), 23 deletions(-)

diff --git a/Gemfile b/Gemfile
index 2ca6a3b78..30167e355 100644
--- a/Gemfile
+++ b/Gemfile
@@ -18,4 +18,4 @@ group :profiling do
   gem 'thin'
 end
 
-gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master'
+gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master'
diff --git a/Gemfile.lock b/Gemfile.lock
index 9265d63a9..2945f1d38 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,6 +1,6 @@
 GIT
-  remote: https://github.com/ncbo/sparql-client.git
-  revision: fb4a89b420f8eb6dda5190a126b6c62e32c4c0c9
+  remote: https://github.com/ontoportal-lirmm/sparql-client.git
+  revision: aed51baf4106fd0f3d0e3f9238f0aad9406aa3f0
   branch: master
   specs:
     sparql-client (1.0.1)
@@ -34,14 +34,15 @@ GEM
     addressable (2.3.5)
     builder (3.2.4)
     coderay (1.1.3)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.1.10)
+    connection_pool (2.3.0)
     cube-ruby (0.0.3)
     daemons (1.4.1)
     docile (1.4.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     eventmachine (1.2.7)
-    faraday (1.10.0)
+    faraday (1.10.3)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
       faraday-excon (~> 1.1)
@@ -57,19 +58,19 @@ GEM
     faraday-em_synchrony (1.0.0)
     faraday-excon (1.1.0)
     faraday-httpclient (1.0.1)
-    faraday-multipart (1.0.3)
-      multipart-post (>= 1.2, < 3)
+    faraday-multipart (1.0.4)
+      multipart-post (~> 2)
     faraday-net_http (1.0.1)
     faraday-net_http_persistent (1.2.0)
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
     faraday-retry (1.0.3)
     http-accept (1.7.0)
-    http-cookie (1.0.4)
+    http-cookie (1.0.5)
       domain_name (~> 0.5)
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
-    json_pure (2.6.1)
+    json_pure (2.6.3)
     macaddr (1.7.2)
       systemu (~> 2.6.5)
     method_source (1.0.0)
@@ -78,25 +79,28 @@ GEM
     mime-types-data (3.2022.0105)
     minitest (4.7.5)
     multi_json (1.15.0)
-    multipart-post (2.1.1)
-    mustermann (1.1.1)
+    multipart-post (2.2.3)
+    mustermann (3.0.0)
       ruby2_keywords (~> 0.0.1)
     net-http-persistent (2.9.4)
     netrc (0.11.0)
-    pry (0.14.1)
+    pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    rack (2.2.3)
+    rack (2.2.6.2)
     rack-accept (0.4.5)
       rack (>= 0.4)
     rack-post-body-to-params (0.1.8)
       activesupport (>= 2.3)
-    rack-protection (2.2.0)
+    rack-protection (3.0.5)
       rack
     rake (13.0.6)
     rdf (1.0.8)
       addressable (>= 2.2)
-    redis (4.6.0)
+    redis (5.0.6)
+      redis-client (>= 0.9.0)
+    redis-client (0.12.1)
+      connection_pool
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
@@ -106,16 +110,16 @@ GEM
       builder (>= 2.1.2)
       faraday (>= 0.9, < 3, != 2.0.0)
     ruby2_keywords (0.0.5)
-    simplecov (0.21.2)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    sinatra (2.2.0)
-      mustermann (~> 1.0)
-      rack (~> 2.2)
-      rack-protection (= 2.2.0)
+    sinatra (3.0.5)
+      mustermann (~> 3.0)
+      rack (~> 2.2, >= 2.2.4)
+      rack-protection (= 3.0.5)
       tilt (~> 2.0)
     systemu (2.6.5)
     thin (1.8.1)
@@ -123,11 +127,11 @@ GEM
       eventmachine (~> 1.0, >= 1.0.4)
       rack (>= 1, < 3)
     thread_safe (0.3.6)
-    tilt (2.0.10)
-    tzinfo (0.3.60)
+    tilt (2.0.11)
+    tzinfo (0.3.61)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.8)
+    unf_ext (0.0.8.2)
     uuid (2.3.9)
       macaddr (~> 1.0)
 

From 87e991b0e720c580a71e2c2c10f9ab44183db322 Mon Sep 17 00:00:00 2001
From: Alex Skrenchuk <alexskr@stanford.edu>
Date: Thu, 14 Mar 2024 21:27:47 -0700
Subject: [PATCH 28/42] fix healthcheck for AG

---
 docker-compose.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 71b2fe700..f66e7feff 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -39,11 +39,11 @@ services:
         ; agtool users grant anonymous root:ontoportal_test:rw
         ; tail -f /agraph/data/agraph.log"
     healthcheck:
-      test: ["CMD-SHELL", "curl -m 1 -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1"]
-      start_period: 60s
+      test: ["CMD-SHELL", "agtool storage-report ontoportal_test || exit 1"]
+      start_period: 30s
       interval: 10s
-      timeout: 5s
-      retries: 5
+      timeout: 10s
+      retries: 10
     profiles:
       - ag
 

From 7215f496c1f6130768867725baa224077f16360a Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 17 Apr 2024 17:15:07 +0200
Subject: [PATCH 29/42] add request store  dependency to save globally the
 language per request

---
 Gemfile                  |  3 ++-
 Gemfile.lock             | 20 +++++++++++++-------
 goo.gemspec              |  1 +
 lib/goo/sparql/loader.rb |  6 ++++++
 4 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/Gemfile b/Gemfile
index bfbe20499..81ef9c339 100644
--- a/Gemfile
+++ b/Gemfile
@@ -6,6 +6,8 @@ gem "activesupport"
 gem "cube-ruby", require: "cube"
 gem "rake"
 gem "uuid"
+gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop'
+
 
 group :test do
   gem "minitest", '< 5.0'
@@ -21,4 +23,3 @@ group :profiling do
   gem "thin"
 end
 
-gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop'
diff --git a/Gemfile.lock b/Gemfile.lock
index b68673448..fdccf0596 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -16,6 +16,7 @@ PATH
       pry
       rdf (= 1.0.8)
       redis
+      request_store
       rest-client
       rsolr
       sparql-client
@@ -42,11 +43,10 @@ GEM
     docile (1.4.0)
     domain_name (0.6.20240107)
     eventmachine (1.2.7)
-    faraday (2.8.1)
-      base64
-      faraday-net_http (>= 2.0, < 3.1)
-      ruby2_keywords (>= 0.0.4)
-    faraday-net_http (3.0.2)
+    faraday (2.9.0)
+      faraday-net_http (>= 2.0, < 3.2)
+    faraday-net_http (3.1.0)
+      net-http
     http-accept (1.7.0)
     http-cookie (1.0.5)
       domain_name (~> 0.5)
@@ -63,6 +63,8 @@ GEM
     multi_json (1.15.0)
     mustermann (3.0.0)
       ruby2_keywords (~> 0.0.1)
+    net-http (0.4.1)
+      uri
     net-http-persistent (2.9.4)
     netrc (0.11.0)
     pry (0.14.2)
@@ -84,6 +86,8 @@ GEM
       redis-client (>= 0.22.0)
     redis-client (0.22.1)
       connection_pool
+    request_store (1.6.0)
+      rack (>= 1.4)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
@@ -116,11 +120,13 @@ GEM
     thread_safe (0.3.6)
     tilt (2.3.0)
     tzinfo (0.3.62)
+    uri (0.13.0)
     uuid (2.3.9)
       macaddr (~> 1.0)
 
 PLATFORMS
-  x86_64-darwin-18
+  x86_64-darwin-23
+  x86_64-linux
 
 DEPENDENCIES
   activesupport
@@ -139,4 +145,4 @@ DEPENDENCIES
   uuid
 
 BUNDLED WITH
-   2.3.15
+   2.2.33
diff --git a/goo.gemspec b/goo.gemspec
index c33867990..4e3461a4e 100644
--- a/goo.gemspec
+++ b/goo.gemspec
@@ -14,4 +14,5 @@ Gem::Specification.new do |s|
   s.add_dependency("rsolr")
   s.add_dependency("sparql-client")
   s.add_dependency("uuid")
+  s.add_dependency("request_store")
 end
diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index 821aba260..094fbba28 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -1,3 +1,4 @@
+require 'request_store'
 module Goo
   module SPARQL
     module Loader
@@ -6,8 +7,10 @@ class << self
 
         def model_load(*options)
           options = options.last
+          set_request_lang(options)
           if options[:models] && options[:models].is_a?(Array) && \
            (options[:models].length > Goo.slice_loading_size)
+
             options = options.dup
             models = options[:models]
             include_options = options[:include]
@@ -96,6 +99,9 @@ def model_load_sliced(*options)
 
         private
 
+        def set_request_lang(options)
+          options[:requested_lang] = RequestStore.store[:requested_lang]
+        end
         def expand_equivalent_predicates(properties_to_include, eq_p)
 
           return unless eq_p && !eq_p.empty?

From d6e8c33e49f4b2168151c822a98eab62339a9f8a Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 17 Apr 2024 17:19:00 +0200
Subject: [PATCH 30/42] and languages settings dsl

---
 lib/goo.rb                        | 44 +++++++++++++++++++++++++++++++
 lib/goo/base/settings/settings.rb |  8 ++++++
 2 files changed, 52 insertions(+)

diff --git a/lib/goo.rb b/lib/goo.rb
index 1d38a1514..cfdf9ea31 100644
--- a/lib/goo.rb
+++ b/lib/goo.rb
@@ -30,6 +30,11 @@ module Goo
 
   @@resource_options = Set.new([:persistent]).freeze
 
+  # Define the languages from which the properties values will be taken
+  # It choose the first language that match otherwise return all the values
+  @@main_languages = %w[en]
+  @@requested_language = nil
+
   @@configure_flag = false
   @@sparql_backends = {}
   @@model_by_name = {}
@@ -47,6 +52,27 @@ module Goo
 
   @@slice_loading_size = 500
 
+
+  def self.main_languages
+    @@main_languages
+  end
+  def self.main_languages=(lang)
+    @@main_languages = lang
+  end
+
+  def self.requested_language
+    @@requested_language
+  end
+
+  def self.requested_language=(lang)
+    @@requested_language = lang
+  end
+
+  def self.language_includes(lang)
+    lang_str = lang.to_s
+    main_languages.index { |l| lang_str.downcase.eql?(l) || lang_str.upcase.eql?(l)}
+  end
+
   def self.add_namespace(shortcut, namespace,default=false)
     if !(namespace.instance_of? RDF::Vocabulary)
       raise ArgumentError, "Namespace must be a RDF::Vocabulary object"
@@ -85,6 +111,24 @@ def self.add_sparql_backend(name, *opts)
     @@sparql_backends.freeze
   end
 
+  def self.test_reset
+    if @@sparql_backends[:main][:query].url.to_s["localhost"].nil?
+      raise Exception, "only for testing"
+    end
+    @@sparql_backends[:main][:query]=Goo::SPARQL::Client.new("http://localhost:9000/sparql/",
+                 {protocol: "1.1", "Content-Type" => "application/x-www-form-urlencoded",
+                   read_timeout: 300,
+                  redis_cache: @@redis_client })
+  end
+
+  def self.main_lang
+    @@main_lang
+  end
+
+  def self.main_lang=(value)
+    @@main_lang = value
+  end
+
   def self.use_cache=(value)
     @@use_cache = value
     set_sparql_cache
diff --git a/lib/goo/base/settings/settings.rb b/lib/goo/base/settings/settings.rb
index 2a2744546..e12ed475f 100644
--- a/lib/goo/base/settings/settings.rb
+++ b/lib/goo/base/settings/settings.rb
@@ -372,6 +372,14 @@ def read_only(attributes)
           instance
         end
 
+        def show_all_languages?(args)
+          args.first.is_a?(Hash) && args.first.keys.include?(:include_languages) && args.first[:include_languages]
+        end
+
+        def not_show_all_languages?(values, args)
+          values.is_a?(Hash) && !show_all_languages?(args)
+        end
+
       end
     end
   end

From 163a3de3ced136ed251735d58a0b58e1da9ed9b0 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 17 Apr 2024 17:20:33 +0200
Subject: [PATCH 31/42] create a module in sparql mixins to handle languages
 filters logic

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 186 ++++++++++++++++++
 lib/goo/sparql/sparql.rb                      |   1 +
 2 files changed, 187 insertions(+)
 create mode 100644 lib/goo/sparql/mixins/solution_lang_filter.rb

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
new file mode 100644
index 000000000..dedc09fb8
--- /dev/null
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -0,0 +1,186 @@
+module Goo
+  module SPARQL
+    module Solution
+      class  LanguageFilter
+
+        attr_reader :requested_lang, :unmapped, :objects_by_lang
+
+        def initialize(requested_lang: RequestStore.store[:requested_lang], unmapped: false, list_attributes: [])
+          @list_attributes = list_attributes
+          @objects_by_lang = {}
+          @unmapped = unmapped
+          @requested_lang = get_language(requested_lang)
+        end
+
+        def fill_models_with_all_languages(models_by_id)
+          objects_by_lang.each do |id, predicates|
+            model = models_by_id[id]
+            predicates.each do |predicate, values|
+
+              if values.values.all? { |v| v.all? { |x| literal?(x) && x.plain?} }
+                pull_stored_values(model, values, predicate, @unmapped)
+              end
+            end
+          end
+        end
+
+
+        def set_model_value(model, predicate, values, value)
+          set_value(model, predicate, value) do
+            model.send("#{predicate}=", values, on_load: true)
+          end
+        end
+
+        def set_unmapped_value(model, predicate, value)
+          set_value(model, predicate, value) do
+            return add_unmapped_to_model(model, predicate, value)
+          end
+        end
+
+        def models_unmapped_to_array(m)
+          if show_all_languages?
+            model_group_by_lang(m)
+          else
+            m.unmmaped_to_array
+          end
+        end
+
+        private
+
+
+        def set_value(model, predicate, value, &block)
+          language = object_language(value)
+          
+          if requested_lang.eql?(:ALL) || !literal?(value) || (language_match?(language) && can_add_new_value(model,predicate, language))
+              block.call
+          end
+
+          if requested_lang.eql?(:ALL) || requested_lang.is_a?(Array)
+            language = "@none" if no_lang?(language)
+            store_objects_by_lang(model.id, predicate, value, language)
+          end
+        end
+
+
+        def  can_add_new_value(model, predicate, new_language)
+          old_val = model.send(predicate) rescue nil
+          list_attributes?(predicate) || old_val.blank? || !no_lang?(new_language)
+        end
+
+        def no_lang?(language)
+          language.nil? || language.eql?(:no_lang)
+        end
+        
+        def model_group_by_lang(model)
+          unmapped = model.unmapped
+          cpy = {}
+
+          unmapped.each do |attr, v|
+            cpy[attr] = group_by_lang(v)
+          end
+
+          model.unmapped = cpy
+        end
+
+        def group_by_lang(values)
+
+          return values.to_a if values.all?{|x| x.is_a?(RDF::URI) || !x.respond_to?(:language) }
+
+          values = values.group_by { |x| x.respond_to?(:language) && x.language ? x.language.to_s.downcase : :none }
+
+          no_lang = values[:none] || []
+          return no_lang if !no_lang.empty? && no_lang.all? { |x| x.respond_to?(:plain?) && !x.plain? }
+
+          values
+        end
+
+
+        def object_language(new_value)
+          new_value.language || :no_lang if new_value.is_a?(RDF::Literal)
+        end
+
+        def language_match?(language)
+          # no_lang means that the object is not a literal
+          return true if language.eql?(:no_lang)
+
+          return requested_lang.include?(language)  if requested_lang.is_a?(Array)
+
+          language.eql?(requested_lang)
+        end
+
+        def literal?(object)
+          !object_language(object).nil?
+        end
+
+        def store_objects_by_lang(id, predicate, object, language)
+          # store objects in this format: [id][predicate][language] = [objects]
+          return if requested_lang.is_a?(Array) && !requested_lang.include?(language)
+
+          language_key = language.downcase
+
+          objects_by_lang[id] ||= {}
+          objects_by_lang[id][predicate] ||= {}
+          objects_by_lang[id][predicate][language_key] ||= []
+
+          objects_by_lang[id][predicate][language_key] << object
+        end
+
+
+        def add_unmapped_to_model(model, predicate, value)
+
+          if model.respond_to? :klass # struct
+            model[:unmapped] ||= {}
+            model[:unmapped][predicate] ||= []
+            model[:unmapped][predicate]  << value unless value.nil?
+          else
+            model.unmapped_set(predicate, value)
+          end
+        end
+
+        def pull_stored_values(model, values, predicate, unmapped)
+          if unmapped
+            add_unmapped_to_model(model, predicate, values)
+          else
+            values = values.map do  |language, values_literals|
+              values_string = values_literals.map{|x| x.object}
+              values_string = values_string.first unless list_attributes?(predicate)
+              [language, values_string]
+            end.to_h
+
+            model.send("#{predicate}=", values, on_load: true)
+          end
+
+        end
+
+        def unmapped_get(model, predicate)
+          if model && model.respond_to?(:klass) # struct
+            model[:unmapped]&.dig(predicate)
+          else
+            model.unmapped_get(predicate)
+          end
+
+        end
+
+        def list_attributes?(predicate)
+          @list_attributes.include?(predicate)
+        end
+
+
+        def show_all_languages?
+          @requested_lang.is_a?(Array) || @requested_lang.eql?(:ALL)
+        end
+
+        def get_language(languages)
+          languages = portal_language if languages.nil? || languages.empty?
+          lang = languages.to_s.split(',').map { |l| l.upcase.to_sym }
+          lang.length == 1 ? lang.first : lang
+        end
+
+        def portal_language
+          Goo.main_languages.first
+        end
+
+      end
+    end
+  end
+end
diff --git a/lib/goo/sparql/sparql.rb b/lib/goo/sparql/sparql.rb
index dfd3d0a67..6fa1d5827 100644
--- a/lib/goo/sparql/sparql.rb
+++ b/lib/goo/sparql/sparql.rb
@@ -1,6 +1,7 @@
 require "sparql/client"
 
 require_relative "mixins/query_pattern"
+require_relative "mixins/solution_lang_filter"
 require_relative "query_builder"
 require_relative "solutions_mapper"
 require_relative "client"

From 43c8459c5a84df53e2d48c45e68b065e9436bc00 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 17 Apr 2024 17:26:49 +0200
Subject: [PATCH 32/42] refactor query_builder to extract internal_variables as
 instance variable

---
 lib/goo/sparql/query_builder.rb | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index 7ef72193e..5ef37adcd 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -17,24 +17,24 @@ def initialize(options)
         @model_query_options = options[:query_options]
         @enable_rules = options[:rules]
         @order_by = options[:order_by]
-
+        @internal_variables_map = {}
         @query = get_client
       end
 
       def build_select_query(ids, variables, graphs, patterns,
                              query_options, properties_to_include)
 
-        internal_variables = graph_match(@collection, @graph_match, graphs, @klass, patterns, query_options, @unions)
+        patterns = graph_match(@collection, @graph_match, graphs, @klass, patterns, query_options, @unions)
 
         aggregate_projections, aggregate_vars,
           variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs,
-                                                            @klass, @unions, variables, internal_variables)
+                                                            @klass, @unions, variables)
 
         @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables)
         variables, patterns = add_some_type_to_id(patterns, query_options, variables)
 
-        query_filter_str, patterns, optional_patterns =
-          filter_query_strings(@collection, graphs, internal_variables, @klass, optional_patterns, patterns, @query_filters)
+        query_filter_str, patterns, optional_patterns, filter_variables =
+          filter_query_strings(@collection, graphs, @klass, optional_patterns, patterns, @query_filters)
 
         variables = [] if @count
         variables.delete :some_type
@@ -170,6 +170,7 @@ def patterns_for_match(klass, attr, value, graphs, patterns, unions,
             value = "#{attr}_agg_#{in_aggregate}".to_sym
           end
           internal_variables << value
+          @internal_variables_map[attr] = value
         end
 
         add_rules(attr, klass, query_options)
@@ -210,7 +211,7 @@ def walk_pattern(klass, match_patterns, graphs, patterns, unions,
         end
       end
 
-      def get_aggregate_vars(aggregate, collection, graphs, klass, unions, variables, internal_variables)
+      def get_aggregate_vars(aggregate, collection, graphs, klass, unions, variables)
         # mdorf, 6/03/20 If aggregate projections (sub-SELECT within main SELECT) use an alias, that alias cannot appear in the main SELECT
         # https://github.com/ncbo/goo/issues/106
         # See last sentence in https://www.w3.org/TR/sparql11-query/#aggregateExample
@@ -241,8 +242,6 @@ def get_aggregate_vars(aggregate, collection, graphs, klass, unions, variables,
       end
 
       def graph_match(collection, graph_match, graphs, klass, patterns, query_options, unions)
-        internal_variables = []
-
         if graph_match
           #make it deterministic - for caching
           graph_match_iteration = Goo::Base::PatternIteration.new(graph_match)
@@ -250,7 +249,7 @@ def graph_match(collection, graph_match, graphs, klass, patterns, query_options,
                        internal_variables, in_aggregate = false, query_options, collection)
           graphs.uniq!
         end
-        internal_variables
+        patterns
       end
 
       def get_client
@@ -336,7 +335,7 @@ def query_filter_sparql(klass, filter, filter_patterns, filter_graphs,
         end
       end
 
-      def filter_query_strings(collection, graphs, internal_variables, klass,
+      def filter_query_strings(collection, graphs, klass,
                                optional_patterns, patterns,
                                query_filters)
         query_filter_str = []
@@ -376,6 +375,9 @@ def add_some_type_to_id(patterns, query_options, variables)
         [variables, patterns]
       end
 
+      def internal_variables
+        @internal_variables_map.values
+      end
     end
   end
 end

From 0dc978a4c2e8cef5f48a86fbbcde1011b8eb15e6 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 17 Apr 2024 17:39:42 +0200
Subject: [PATCH 33/42] add complex oder_by for embed attributes and fix empty
 pagination

---
 lib/goo/base/where.rb              |   5 +
 lib/goo/sparql/query_builder.rb    | 133 +++++++++++------
 lib/goo/sparql/solutions_mapper.rb | 227 ++++++++++++-----------------
 test/test_where.rb                 |  30 +++-
 4 files changed, 218 insertions(+), 177 deletions(-)

diff --git a/lib/goo/base/where.rb b/lib/goo/base/where.rb
index 5bc0fa8ca..81cd26cee 100644
--- a/lib/goo/base/where.rb
+++ b/lib/goo/base/where.rb
@@ -209,6 +209,11 @@ def process_query_intl(count=false)
 
         options_load[:ids] = ids if ids
         models_by_id = {}
+        if @page_i && (options_load[:models].length > 0)
+          options_load.delete(:filters)
+          options_load.delete(:order_by)
+        end
+
         if (@page_i && options_load[:models].length > 0) ||
             (!@page_i && (@count.nil? || @count > 0))
           models_by_id = Goo::SPARQL::Queries.model_load(options_load)
diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index 5ef37adcd..6e268ce50 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -25,21 +25,21 @@ def build_select_query(ids, variables, graphs, patterns,
                              query_options, properties_to_include)
 
         patterns = graph_match(@collection, @graph_match, graphs, @klass, patterns, query_options, @unions)
+        variables, patterns = add_some_type_to_id(patterns, query_options, variables)
+        aggregate_projections, aggregate_vars, variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs, @klass, @unions, variables)
 
-        aggregate_projections, aggregate_vars,
-          variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs,
-                                                            @klass, @unions, variables)
+        @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables,patterns, query_options, graphs)
+        order_by_str, order_variables = order_by_string
 
-        @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables)
-        variables, patterns = add_some_type_to_id(patterns, query_options, variables)
 
         query_filter_str, patterns, optional_patterns, filter_variables =
           filter_query_strings(@collection, graphs, @klass, optional_patterns, patterns, @query_filters)
 
+
         variables = [] if @count
         variables.delete :some_type
 
-        select_distinct(variables, aggregate_projections)
+        select_distinct(variables, aggregate_projections, filter_variables, order_variables)
           .from(graphs)
           .where(patterns)
           .union_bind_in_where(properties_to_include)
@@ -55,7 +55,10 @@ def build_select_query(ids, variables, graphs, patterns,
         @query.union(*@unions) unless @unions.empty?
 
         ids_filter(ids) if ids
-        order_by if @order_by # TODO  test if work
+
+
+        @query.order_by(*order_by_str) if @order_by
+
 
         put_query_aggregate_vars(aggregate_vars) if aggregate_vars
         count if @count
@@ -117,10 +120,17 @@ def put_query_aggregate_vars(aggregate_vars)
         self
       end
 
-      def order_by
-        order_by_str = @order_by.map { |attr, order| "#{order.to_s.upcase}(?#{attr})" }
-        @query.order_by(*order_by_str)
-        self
+      def order_by_string
+        order_variables = []
+        order_str = @order_by&.map do |attr, order|
+          if order.is_a?(Hash)
+            sub_attr, order = order.first
+            attr =  @internal_variables_map.select{ |internal_var, attr_var| attr_var.eql?({attr => sub_attr}) || attr_var.eql?(sub_attr)}.keys.last
+          end
+          order_variables << attr
+          "#{order.to_s.upcase}(?#{attr})"
+        end
+        [order_str,order_variables]
       end
 
       def from(graphs)
@@ -135,10 +145,11 @@ def from(graphs)
         self
       end
 
-      def select_distinct(variables, aggregate_projections)
-
+      def select_distinct(variables, aggregate_variables, filter_variables, order_variables)
         select_vars = variables.dup
-        reject_aggregations_from_vars(select_vars, aggregate_projections) if aggregate_projections
+        reject_aggregations_from_vars(select_vars, aggregate_variables) if aggregate_variables
+        # Fix for 4store pagination with a filter https://github.com/ontoportal-lirmm/ontologies_api/issues/25
+        select_vars = (select_vars + filter_variables + order_variables).uniq  if @page
         @query = @query.select(*select_vars).distinct(true)
         self
       end
@@ -147,7 +158,7 @@ def ids_filter(ids)
         filter_id = []
 
         ids.each do |id|
-          filter_id << "?id = #{id.to_ntriples.to_s}"
+          filter_id << "?id = #{id.to_ntriples.to_s.gsub(' ', '%20')}"
         end
         filter_id_str = filter_id.join ' || '
         @query.filter filter_id_str
@@ -159,23 +170,24 @@ def ids_filter(ids)
       def patterns_for_match(klass, attr, value, graphs, patterns, unions,
                              internal_variables, subject = :id, in_union = false,
                              in_aggregate = false, query_options = {}, collection = nil)
+        new_internal_var = value
         if value.respond_to?(:each) || value.instance_of?(Symbol)
           next_pattern = value.instance_of?(Array) ? value.first : value
 
           #for filters
           next_pattern = { next_pattern => [] } if next_pattern.instance_of?(Symbol)
 
-          value = "internal_join_var_#{internal_variables.length}".to_sym
+          new_internal_var = "internal_join_var_#{internal_variables.length}".to_sym
           if in_aggregate
-            value = "#{attr}_agg_#{in_aggregate}".to_sym
+            new_internal_var = "#{attr}_agg_#{in_aggregate}".to_sym
           end
-          internal_variables << value
-          @internal_variables_map[attr] = value
+          internal_variables << new_internal_var
+          @internal_variables_map[new_internal_var] = value.empty? ? attr : {attr => value}
         end
 
         add_rules(attr, klass, query_options)
         graph, pattern =
-          query_pattern(klass, attr, value: value, subject: subject, collection: collection)
+          query_pattern(klass, attr, value: new_internal_var, subject: subject, collection: collection)
         if pattern
           if !in_union
             patterns << pattern
@@ -188,7 +200,7 @@ def patterns_for_match(klass, attr, value, graphs, patterns, unions,
           range = klass.range(attr)
           next_pattern.each do |next_attr, next_value|
             patterns_for_match(range, next_attr, next_value, graphs,
-                               patterns, unions, internal_variables, subject = value,
+                               patterns, unions, internal_variables, subject = new_internal_var,
                                in_union, in_aggregate, collection = collection)
           end
         end
@@ -256,21 +268,51 @@ def get_client
         Goo.sparql_query_client(@store)
       end
 
-      def init_order_by(count, klass, order_by, optional_patterns, variables)
+      def init_order_by(count, klass, order_by, optional_patterns, variables, patterns, query_options, graphs)
         order_by = nil if count
         if order_by
           order_by = order_by.first
           #simple ordering ... needs to use pattern inspection
           order_by.each do |attr, direction|
-            quad = query_pattern(klass, attr)
-            optional_patterns << quad[1]
+
+            if direction.is_a?(Hash)
+              # TODO this part can be improved/refactored, the complexity was added because order by don't work
+              # if the pattern is in the mandatory ones (variable `patterns`)
+              # and optional (variable `optional_patterns`) at the same time
+              sub_attr, direction = direction.first
+              graph_match_iteration = Goo::Base::PatternIteration.new(Goo::Base::Pattern.new({attr => [sub_attr]}))
+              old_internal = internal_variables.dup
+              old_patterns = optional_patterns.dup
+
+              walk_pattern(klass, graph_match_iteration, graphs, optional_patterns, @unions, internal_variables, in_aggregate = false, query_options, @collection)
+              new_variables = (internal_variables - old_internal)
+              internal_variables.delete(new_variables)
+              new_patterns = optional_patterns - old_patterns
+              already_existent_pattern = patterns.select{|x| x[1].eql?(new_patterns.last[1])}.first
+
+              if already_existent_pattern
+                already_existent_variable = already_existent_pattern[2]
+                optional_patterns = old_patterns
+                key = @internal_variables_map.select{|key, value|  key.eql?(new_variables.last)}.keys.first
+                @internal_variables_map[key] = (already_existent_variable || new_variables.last) if key
+
+                #variables << already_existent_variable
+              else
+                #variables <<  new_variables.last
+              end
+
+            else
+              quad = query_pattern(klass, attr)
+              optional_patterns << quad[1]
+              #variables << attr
+            end
+
             #patterns << quad[1]
             #mdorf, 9/22/16 If an ORDER BY clause exists, the columns used in the ORDER BY should be present in the SPARQL select
             #variables << attr unless variables.include?(attr)
           end
-          variables = %i[id attributeProperty attributeObject]
         end
-        [order_by, variables, optional_patterns]
+        [order_by, variables, optional_patterns, patterns]
       end
 
       def sparql_op_string(op)
@@ -309,14 +351,25 @@ def query_filter_sparql(klass, filter, filter_patterns, filter_graphs,
           end
           filter_var = inspected_patterns[filter_pattern_match]
 
-          if !filter_operation.value.instance_of?(Goo::Filter)
-            if filter_operation.operator == :unbound || filter_operation.operator == :bound
-              if filter_operation.operator == :unbound
-                filter_operations << "!BOUND(?#{filter_var.to_s})"
-              else
-                filter_operations << "BOUND(?#{filter_var.to_s})"
-              end
+          if filter_operation.value.instance_of?(Goo::Filter)
+            filter_operations << "#{sparql_op_string(filter_operation.operator)}"
+            query_filter_sparql(klass, filter_operation.value, filter_patterns,
+                                filter_graphs, filter_operations,
+                                internal_variables, inspected_patterns, collection)
+          else
+            case filter_operation.operator
+            when  :unbound
+              filter_operations << "!BOUND(?#{filter_var.to_s})"
+              return :optional
+
+            when :bound
+              filter_operations << "BOUND(?#{filter_var.to_s})"
               return :optional
+            when :regex
+              if  filter_operation.value.is_a?(String)
+                filter_operations << "REGEX(STR(?#{filter_var.to_s}) , \"#{filter_operation.value.to_s}\", \"i\")"
+              end
+
             else
               value = RDF::Literal.new(filter_operation.value)
               if filter_operation.value.is_a? String
@@ -326,11 +379,7 @@ def query_filter_sparql(klass, filter, filter_patterns, filter_graphs,
                 "?#{filter_var.to_s} #{sparql_op_string(filter_operation.operator)} " +
                   " #{value.to_ntriples}")
             end
-          else
-            filter_operations << "#{sparql_op_string(filter_operation.operator)}"
-            query_filter_sparql(klass, filter_operation.value, filter_patterns,
-                                filter_graphs, filter_operations,
-                                internal_variables, inspected_patterns, collection)
+
           end
         end
       end
@@ -339,8 +388,8 @@ def filter_query_strings(collection, graphs, klass,
                                optional_patterns, patterns,
                                query_filters)
         query_filter_str = []
-
         filter_graphs = []
+        filter_variables = []
         inspected_patterns = {}
         query_filters&.each do |query_filter|
           filter_operations = []
@@ -357,9 +406,9 @@ def filter_query_strings(collection, graphs, klass,
               patterns.concat(filter_patterns)
             end
           end
+          filter_variables << inspected_patterns.values.last
         end
-
-        [query_filter_str, patterns, optional_patterns, internal_variables]
+        [query_filter_str, patterns, optional_patterns, filter_variables]
       end
 
       def reject_aggregations_from_vars(variables, aggregate_projections)
@@ -376,7 +425,7 @@ def add_some_type_to_id(patterns, query_options, variables)
       end
 
       def internal_variables
-        @internal_variables_map.values
+        @internal_variables_map.keys
       end
     end
   end
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 954ceca99..c93b203f7 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -1,12 +1,11 @@
 module Goo
   module SPARQL
     class SolutionMapper
-
       BNODES_TUPLES = Struct.new(:id, :attribute)
 
       def initialize(aggregate_projections, bnode_extraction, embed_struct,
-                     incl_embed, klass_struct, models_by_id,
-                     properties_to_include, unmapped, variables,ids, options)
+                      incl_embed, klass_struct, models_by_id,
+                     properties_to_include, unmapped, variables, ids, options)
 
         @aggregate_projections = aggregate_projections
         @bnode_extraction = bnode_extraction
@@ -19,22 +18,22 @@ def initialize(aggregate_projections, bnode_extraction, embed_struct,
         @variables = variables
         @ids = ids
         @klass = options[:klass]
-        @klass = options[:klass]
         @read_only = options[:read_only]
         @incl = options[:include]
         @count = options[:count]
         @collection = options[:collection]
+        @options = options
       end
-
+      
       def map_each_solutions(select)
-
-
         found = Set.new
         objects_new = {}
-        var_set_hash = {}
         list_attributes = Set.new(@klass.attributes(:list))
         all_attributes = Set.new(@klass.attributes(:all))
 
+        @lang_filter = Goo::SPARQL::Solution::LanguageFilter.new(requested_lang: @options[:requested_lang].to_s, unmapped: @unmapped,
+           list_attributes: list_attributes)
+        
         if @options[:page]
           # for using prefixes before queries
           # mdorf, 7/27/2023, AllegroGraph supplied a patch (rfe17161-7.3.1.fasl.patch)
@@ -53,6 +52,7 @@ def map_each_solutions(select)
         # File.write(debug_file, select.to_s + "\n", mode: 'a') if select.to_s =~ /OFFSET \d+ LIMIT 2500$/
 
         select.each_solution do |sol|
+          
           next if sol[:some_type] && @klass.type_uri(@collection) != sol[:some_type]
           return sol[:count_var].object if @count
 
@@ -79,21 +79,24 @@ def map_each_solutions(select)
             next
           end
 
-          v = sol[:attributeProperty].to_s.to_sym
+          predicate = sol[:attributeProperty].to_s.to_sym
 
-          next if v.nil? || !all_attributes.include?(v)
+          next if predicate.nil? || !all_attributes.include?(predicate)
 
           object = sol[:attributeObject]
 
-          #bnodes
-          if bnode_id?(object, v)
-            objects_new = bnode_id_tuple(id, object, objects_new, v)
+          # bnodes
+          if bnode_id?(object, predicate)
+            objects_new = bnode_id_tuple(id, object, objects_new, predicate)
             next
           end
 
-          object, objects_new = get_value_object(id, objects_new, object, list_attributes, v)
-          add_object_to_model(id, object, v, var_set_hash)
+          objects, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate)
+          add_object_to_model(id, objects, object, predicate)
         end
+      
+        # for this moment we are not going to enrich models , maybe we will use it if the results are empty  
+        @lang_filter.fill_models_with_all_languages(@models_by_id)
 
         # for troubleshooting specific queries (write 3 of 3)
         # File.write(debug_file, "\n\n", mode: 'a') if select.to_s =~ /OFFSET \d+ LIMIT 2500$/
@@ -104,20 +107,21 @@ def map_each_solutions(select)
 
         model_set_collection_attributes(@models_by_id, objects_new)
 
-        #remove from models_by_id elements that were not touched
-        @models_by_id.select! { |k, m| found.include?(k) }
+        # remove from models_by_id elements that were not touched
+        @models_by_id.select! { |k, _m| found.include?(k) }
 
         models_set_all_persistent(@models_by_id) unless @read_only
 
-        #next level of embed attributes
+        # next level of embed attributes
         include_embed_attributes(@incl_embed, objects_new) if @incl_embed && !@incl_embed.empty?
 
-        #bnodes
-        blank_nodes = objects_new.select { |id, obj| id.is_a?(RDF::Node) && id.anonymous? }
+        # bnodes
+        blank_nodes = objects_new.select { |id, _obj| id.is_a?(RDF::Node) && id.anonymous? }
         include_bnodes(blank_nodes, @models_by_id) unless blank_nodes.empty?
 
         models_unmapped_to_array(@models_by_id) if @unmapped
-
+        
+       
         @models_by_id
       end
 
@@ -152,8 +156,7 @@ def init_unloaded_attributes(found, list_attributes)
       def get_value_object(id, objects_new, object, list_attributes, predicate)
         object = object.object if object && !(object.is_a? RDF::URI)
         range_for_v = @klass.range(predicate)
-        #binding.pry if v.eql?(:enrolled)
-        #dependent model creation
+       
 
         if object.is_a?(RDF::URI) && (predicate != :id) && !range_for_v.nil?
           if objects_new.include?(object)
@@ -163,7 +166,7 @@ def get_value_object(id, objects_new, object, list_attributes, predicate)
             object, objects_new = if !@read_only
                                     preloaded_or_new_object(object, objects_new, pre_val, predicate)
                                   else
-                                    #depedent read only
+                                    # depedent read only
                                     preloaded_or_new_struct(object, objects_new, pre_val, predicate)
                                   end
           else
@@ -172,40 +175,27 @@ def get_value_object(id, objects_new, object, list_attributes, predicate)
         end
 
         if list_attributes.include?(predicate)
-          # To handle attr that are lists
-          pre = if @klass_struct
-                  @models_by_id[id][predicate]
-                else
-                  @models_by_id[id].instance_variable_get("@#{predicate}")
-                end
-          if object.nil? && pre.nil?
-            object = []
-          elsif object.nil? && !pre.nil?
-            object = pre
-          elsif object
-            object = !pre ? [object] : (pre.dup << object)
+          pre = @klass_struct ? @models_by_id[id][predicate] : @models_by_id[id].instance_variable_get("@#{predicate}")
+
+          if object.nil?
+            object = pre.nil? ? [] : pre
+          else            
+            object = pre.nil? ? [object] : (Array(pre).dup << object)
             object.uniq!
           end
+
         end
-        [object,objects_new]
+        [object, objects_new]
       end
 
-      def add_object_to_model(id, object, predicate, var_set_hash)
+      def add_object_to_model(id, objects, current_obj, predicate)
+
         if @models_by_id[id].respond_to?(:klass)
-          @models_by_id[id][predicate] = object unless object.nil? && !@models_by_id[id][predicate].nil?
+          @models_by_id[id][predicate] = objects unless objects.nil? && !@models_by_id[id][predicate].nil?
         elsif !@models_by_id[id].class.handler?(predicate) &&
-          !(object.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
-          predicate != :id
-          # if multiple language values are included for a given property, set the
-          # corresponding model attribute to the English language value - NCBO-1662
-          if object.is_a?(RDF::Literal)
-            key = "#{predicate}#__#{id}"
-            @models_by_id[id].send("#{predicate}=", object, on_load: true) unless var_set_hash[key]
-            lang = object.language
-            var_set_hash[key] = true if %i[EN en].include?(lang)
-          else
-            @models_by_id[id].send("#{predicate}=", object, on_load: true)
-          end
+              !(objects.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
+              predicate != :id
+          @lang_filter.set_model_value(@models_by_id[id], predicate, objects, current_obj)
         end
       end
 
@@ -213,7 +203,7 @@ def get_preload_value(id, object, predicate)
         pre_val = nil
         if predicate_preloaded?(id, predicate)
           pre_val = preloaded_value(id, predicate)
-          pre_val = pre_val.select { |x| x.id == object }.first if pre_val.is_a?(Array)
+          pre_val = pre_val.select { |x| x.respond_to?(:id) && (x.id == object) }.first if pre_val.is_a?(Array)
         end
         pre_val
       end
@@ -235,6 +225,7 @@ def preloaded_or_new_struct(object, objects_new, pre_val, predicate)
       def preloaded_value(id, predicate)
         if !@read_only
           @models_by_id[id].instance_variable_get("@#{predicate}")
+
         else
           @models_by_id[id][predicate]
         end
@@ -251,9 +242,7 @@ def bnode_id?(object, predicate)
 
       def bnode_id_tuple(id, object, objects_new, predicate)
         range = @klass.range(predicate)
-        if range.respond_to?(:new)
-          objects_new[object] = BNODES_TUPLES.new(id, predicate)
-        end
+        objects_new[object] = BNODES_TUPLES.new(id, predicate) if range.respond_to?(:new)
         objects_new
       end
 
@@ -267,21 +256,13 @@ def create_model(id)
         @models_by_id[id] = create_class_model(id, @klass, @klass_struct) unless @models_by_id.include?(id)
       end
 
-      def model_set_unmapped(id, predicate, value)
-
-        if @models_by_id[id].respond_to? :klass #struct
-          @models_by_id[id][:unmapped] ||= {}
-          (@models_by_id[id][:unmapped][predicate] ||= []) << value
-        else
-          @models_by_id[id].unmapped_set(predicate, value)
-        end
-      end
 
       def create_struct(bnode_extraction, models_by_id, sol, variables)
         list_attributes = Set.new(@klass.attributes(:list))
         struct = @klass.range(bnode_extraction).new
         variables.each do |v|
           next if v == :id
+
           svalue = sol[v]
           struct[v] = svalue.is_a?(RDF::Node) ? svalue : svalue.object
         end
@@ -302,70 +283,78 @@ def create_class_model(id, klass, klass_struct)
       end
 
       def models_unmapped_to_array(models_by_id)
-        models_by_id.each do |idm, m|
-          m.unmmaped_to_array
+        models_by_id.each do |_idm, m|
+          @lang_filter.models_unmapped_to_array(m)
         end
       end
 
+
+      def is_multiple_langs?
+        return true if @requested_lang.is_a?(Array) || @requested_lang.eql?(:ALL)
+        false
+      end
+
       def include_bnodes(bnodes, models_by_id)
-        #group by attribute
-        attrs = bnodes.map { |x, y| y.attribute }.uniq
+        # group by attribute
+        attrs = bnodes.map { |_x, y| y.attribute }.uniq
         attrs.each do |attr|
           struct = @klass.range(attr)
 
-          #bnodes that are in a range of goo ground models
-          #for example parents and children in LD class models
-          #we skip this cases for the moment
+          # bnodes that are in a range of goo ground models
+          # for example parents and children in LD class models
+          # we skip this cases for the moment
           next if struct.respond_to?(:model_name)
 
           bnode_attrs = struct.new.to_h.keys
-          ids = bnodes.select { |x, y| y.attribute == attr }.map { |x, y| y.id }
-          @klass.where.models(models_by_id.select { |x, y| ids.include?(x) }.values)
-               .in(@collection)
-               .include(bnode: { attr => bnode_attrs }).all
+          ids = bnodes.select { |_x, y| y.attribute == attr }.map { |_x, y| y.id }
+          @klass.where.models(models_by_id.select { |x, _y| ids.include?(x) }.values)
+                .in(@collection)
+                .include(bnode: { attr => bnode_attrs }).all
         end
       end
 
       def include_embed_attributes(incl_embed, objects_new)
         incl_embed.each do |attr, next_attrs|
-          #anything to join ?
+          # anything to join ?
           attr_range = @klass.range(attr)
           next if attr_range.nil?
-          range_objs = objects_new.select { |id, obj|
+
+          range_objs = objects_new.select do |_id, obj|
             obj.instance_of?(attr_range) || (obj.respond_to?(:klass) && obj[:klass] == attr_range)
-          }.values
-          unless range_objs.empty?
-            range_objs.uniq!
-            query = attr_range.where().models(range_objs).in(@collection).include(*next_attrs)
-            query = query.read_only if @read_only
-            query.all
-          end
+          end.values
+          next if range_objs.empty?
+
+          range_objs.uniq!
+          query = attr_range.where.models(range_objs).in(@collection).include(*next_attrs)
+          query = query.read_only if @read_only
+          query.all
         end
       end
 
       def models_set_all_persistent(models_by_id)
         return unless @ids
-        models_by_id.each do |k, m|
+
+        models_by_id.each do |_k, m|
           m.persistent = true
         end
       end
 
       def model_set_collection_attributes(models_by_id, objects_new)
         collection_value = get_collection_value
-        if collection_value
-          collection_attribute = @klass.collection_opts
-          models_by_id.each do |id, m|
-            m.send("#{collection_attribute}=", collection_value)
-          end
-          objects_new.each do |id, obj_new|
-            if obj_new.respond_to?(:klass)
-              collection_attribute = obj_new[:klass].collection_opts
-              obj_new[collection_attribute] = collection_value
-            elsif obj_new.class.respond_to?(:collection_opts) &&
-              obj_new.class.collection_opts.instance_of?(Symbol)
-              collection_attribute = obj_new.class.collection_opts
-              obj_new.send("#{collection_attribute}=", collection_value)
-            end
+        return unless collection_value
+
+        collection_attribute = @klass.collection_opts
+        models_by_id.each do |_id, m|
+          m.send("#{collection_attribute}=", collection_value)
+        end
+        objects_new.each do |_id, obj_new|
+          if obj_new.respond_to?(:klass)
+            collection_attribute = obj_new[:klass].collection_opts
+            obj_new[collection_attribute] = collection_value
+          elsif obj_new.class.respond_to?(:collection_opts) &&
+                obj_new.class.collection_opts.instance_of?(Symbol)
+            collection_attribute = obj_new.class.collection_opts
+            obj_new.send("#{collection_attribute}=", collection_value)
           end
         end
       end
@@ -373,36 +362,12 @@ def model_set_collection_attributes(models_by_id, objects_new)
       def get_collection_value
         collection_value = nil
         if @klass.collection_opts.instance_of?(Symbol)
-          if @collection.is_a?(Array) && (@collection.length == 1)
-            collection_value = @collection.first
-          end
-          if @collection.respond_to? :id
-            collection_value = @collection
-          end
+          collection_value = @collection.first if @collection.is_a?(Array) && (@collection.length == 1)
+          collection_value = @collection if @collection.respond_to? :id
         end
         collection_value
       end
 
-      def model_map_attributes_values(id, var_set_hash, models_by_id, object, sol, v)
-        if models_by_id[id].respond_to?(:klass)
-          models_by_id[id][v] = object if models_by_id[id][v].nil?
-        else
-          model_attribute_val = models_by_id[id].instance_variable_get("@#{v.to_s}")
-          if (!models_by_id[id].class.handler?(v) || model_attribute_val.nil?) && v != :id
-            # if multiple language values are included for a given property, set the
-            # corresponding model attribute to the English language value - NCBO-1662
-            if sol[v].is_a?(RDF::Literal)
-              key = "#{v}#__#{id.to_s}"
-              models_by_id[id].send("#{v}=", object, on_load: true) unless var_set_hash[key]
-              lang = sol[v].language
-              var_set_hash[key] = true if %i[EN en EN en EN en].include?(lang)
-            else
-              models_by_id[id].send("#{v}=", object, on_load: true)
-            end
-          end
-        end
-      end
-
       def object_to_array(id, klass_struct, models_by_id, object, predicate)
         pre = if klass_struct
                 models_by_id[id][predicate]
@@ -421,7 +386,6 @@ def object_to_array(id, klass_struct, models_by_id, object, predicate)
       end
 
       def dependent_model_creation(embed_struct, id, models_by_id, object, objects_new, v, options)
-
         read_only = options[:read_only]
         if object.is_a?(RDF::URI) && v != :id
           range_for_v = @klass.range(v)
@@ -440,13 +404,12 @@ def dependent_model_creation(embed_struct, id, models_by_id, object, objects_new
       end
 
       def get_object_from_range(pre_val, embed_struct, object, objects_new, predicate)
-
         range_for_v = @klass.range(predicate)
         if !@read_only
           object = pre_val || @klass.range_object(predicate, object)
           objects_new[object.id] = object
         else
-          #depedent read only
+          # depedent read only
           struct = pre_val || embed_struct[predicate].new
           struct.id = object
           struct.klass = range_for_v
@@ -459,8 +422,8 @@ def get_object_from_range(pre_val, embed_struct, object, objects_new, predicate)
       def get_pre_val(id, models_by_id, object, predicate)
         pre_val = nil
         if models_by_id[id] &&
-          ((models_by_id[id].respond_to?(:klass) && models_by_id[id]) ||
-            models_by_id[id].loaded_attributes.include?(predicate))
+           ((models_by_id[id].respond_to?(:klass) && models_by_id[id]) ||
+             models_by_id[id].loaded_attributes.include?(predicate))
           pre_val = if !@read_only
                       models_by_id[id].instance_variable_get("@#{predicate}")
                     else
@@ -472,7 +435,6 @@ def get_pre_val(id, models_by_id, object, predicate)
         pre_val
       end
 
-
       def add_unmapped_to_model(sol)
         predicate = sol[:attributeProperty].to_s.to_sym
         return unless @properties_to_include[predicate]
@@ -480,7 +442,7 @@ def add_unmapped_to_model(sol)
         id = sol[:id]
         value = sol[:attributeObject]
 
-        model_set_unmapped(id, @properties_to_include[predicate][:uri], value)
+        @lang_filter.set_unmapped_value(@models_by_id[id], @properties_to_include[predicate][:uri], value)
       end
 
       def add_aggregations_to_model(sol)
@@ -498,4 +460,3 @@ def add_aggregations_to_model(sol)
     end
   end
 end
-
diff --git a/test/test_where.rb b/test/test_where.rb
index f3852e13c..173c66cb5 100644
--- a/test/test_where.rb
+++ b/test/test_where.rb
@@ -256,6 +256,20 @@ def test_embed_two_levels
     end
   end
 
+  def test_paging_with_filter_order
+
+    f = Goo::Filter.new(:birth_date) > DateTime.parse('1978-01-03')
+    total_count = Student.where.filter(f).count
+    page_1 = Student.where.include(:name, :birth_date).page(1, total_count - 1).filter(f).order_by(name: :asc).to_a
+    refute_empty page_1
+    assert page_1.next?
+    page_2 = Student.where.include(:name, :birth_date).page(page_1.next_page, total_count - 1).filter(f).order_by(name: :asc).to_a
+
+
+    refute_empty page_2
+    assert_equal total_count, page_1.size + page_2.size
+  end
+
   def test_unique_object_references
     # NOTE: unique references does not apply across different slice loading
     return if Goo.slice_loading_size < 100
@@ -480,13 +494,13 @@ def test_filter
     # unbound on some non existing property
     f = Goo::Filter.new(enrolled: [:xxx]).unbound
     st = Student.where.filter(f).all
-    assert_equal 7, st.length
+    assert st.length == 7
 
     f = Goo::Filter.new(:name).regex("n") # will find all students that contains "n" in there name
     st = Student.where.filter(f).include(:name).all # return "John" , "Daniel"  and  "Susan"
 
     assert_equal 3, st.length
-    assert_equal ["John", "Daniel", "Susan"].sort, st.map { |x| x.name }.sort
+    assert_equal ["John","Daniel","Susan"].sort, st.map { |x| x.name }.sort
   end
 
   def test_aggregated
@@ -578,4 +592,16 @@ def test_include_inverse_with_find
       end
     end
   end
+
+  def test_complex_order_by
+    u = University.where.include(address: [:country]).order_by(address: {country: :asc}).all
+    countries = u.map {|x| x.address.map{|a| a.country}}.flatten
+    assert_equal countries.sort, countries
+
+
+    u = University.where.include(address: [:country]).order_by(address: {country: :desc}).all
+    countries = u.map {|x| x.address.map{|a| a.country}}.flatten
+    assert_equal countries.sort{|a,b| b<=>a }, countries
+  end
+
 end

From e72d6438c47dca99826e6923bb42d9d8798ba2f1 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 17 Apr 2024 20:23:55 +0200
Subject: [PATCH 34/42] add multilingual support tests

---
 test/data/languages.nt         |  9 ++++
 test/test_languages_filters.rb | 90 ++++++++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+)
 create mode 100644 test/data/languages.nt
 create mode 100644 test/test_languages_filters.rb

diff --git a/test/data/languages.nt b/test/data/languages.nt
new file mode 100644
index 000000000..faf464f04
--- /dev/null
+++ b/test/data/languages.nt
@@ -0,0 +1,9 @@
+<http://data.bioontology.org/resource1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://data.bioontology.org/Person> .
+<http://data.bioontology.org/resource1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#label> "John Doe"@en .
+<http://data.bioontology.org/resource1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#label> "Jean Dupont"@fr .
+<http://data.bioontology.org/resource1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#label> "Juan Pérez" .
+<http://data.bioontology.org/resource2> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://data.bioontology.org/Place> .
+<http://data.bioontology.org/resource2> <http://www.w3.org/1999/02/22-rdf-syntax-ns#label> "Paris"@en .
+<http://data.bioontology.org/resource2> <http://www.w3.org/1999/02/22-rdf-syntax-ns#label> "Paris"@fr .
+<http://data.bioontology.org/resource2> <http://www.w3.org/1999/02/22-rdf-syntax-ns#label> "París"@es .
+<http://data.bioontology.org/resource2> <http://www.w3.org/1999/02/22-rdf-syntax-ns#label> "Berlin" .
diff --git a/test/test_languages_filters.rb b/test/test_languages_filters.rb
new file mode 100644
index 000000000..755c5d484
--- /dev/null
+++ b/test/test_languages_filters.rb
@@ -0,0 +1,90 @@
+require_relative "test_case"
+require_relative './app/models'
+
+
+
+class ExamplePerson < Goo::Base::Resource
+  model :person, namespace: :bioportal, name_with: lambda { |k| k.id },
+        collection: :db
+  attribute :db, enforce: [ :database ]
+  attribute :label, namespace: :rdf, enforce: [ :list ]
+end
+
+class ExamplePlace < Goo::Base::Resource
+  model :place, namespace: :bioportal, name_with: lambda { |k| k.id },
+        collection: :db
+  attribute :db, enforce: [ :database ]
+  attribute :label, namespace: :rdf, enforce: [ :list ]
+end
+
+class TestLanguageFilter < MiniTest::Unit::TestCase
+  def self.before_suite
+    RequestStore.store[:requested_lang] = Goo.main_languages.first
+    graph = RDF::URI.new(Test::Models::DATA_ID)
+
+    database = Test::Models::Database.new
+    database.id = graph
+    database.name = "Census tiger 2002"
+    database.save
+
+    @@db = Test::Models::Database.find(RDF::URI.new(Test::Models::DATA_ID)).first
+    @@person_id = RDF::URI.new "http://data.bioontology.org/resource1"
+
+
+    ntriples_file_path = "./test/data/languages.nt"
+
+    Goo.sparql_data_client.put_triples(
+      graph,
+      ntriples_file_path,
+      mime_type = "application/x-turtle")
+  end
+
+  def self.after_suite
+    graph = RDF::URI.new(Test::Models::DATA_ID)
+    Goo.sparql_data_client.delete_graph(graph)
+    database = Test::Models::Database.find(RDF::URI.new(Test::Models::DATA_ID)).first
+    database.delete if database
+    RequestStore.store[:requested_lang] = Goo.main_languages.first
+  end
+
+  def setup
+    RequestStore.store[:requested_lang] = Goo.main_languages.first
+  end
+
+  def test_one_language
+    # by default english and not tagged values
+    person = ExamplePerson.find(@@person_id).in(@@db).include(:label).first
+    assert_equal ["John Doe", "Juan Pérez"].sort, person.label.sort
+
+
+    # select french, return french values and not tagged values
+    RequestStore.store[:requested_lang] = :fr
+    person = ExamplePerson.find(@@person_id).in(@@db).include(:label).first
+    assert_equal ["Jean Dupont", "Juan Pérez"].sort, person.label.sort
+
+  end
+
+  def test_multiple_languages
+    # select all languages
+    RequestStore.store[:requested_lang] = :all
+    expected_result = {:en=>["John Doe"], :fr=>["Jean Dupont"], "@none"=>["Juan Pérez"]}
+    person = ExamplePerson.find(@@person_id).in(@@db).include(:label).first
+    assert_equal expected_result.values.flatten.sort, person.label.sort
+
+    # using include_languages on any attribute returns an hash of {language: values} instead of the array of values
+    assert_equal expected_result, person.label(include_languages: true)
+
+    # filter only french, english and not tagged values
+    RequestStore.store[:requested_lang] = [:fr, :en]
+    person = ExamplePerson.find(@@person_id).in(@@db).include(:label).first
+    assert_equal expected_result.values.flatten.sort.sort, person.label.sort
+  end
+
+
+  def test_language_not_found
+    RequestStore.store[:requested_lang] = :ar
+    person = ExamplePerson.find(@@person_id).in(@@db).include(:label).first
+    # will return only not tagged values if existent
+    assert_equal ["Juan Pérez"], person.label
+  end
+end

From a3d50fbad5dfb712a53240ea3ad66aa6a9fcf0b5 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 17 Apr 2024 20:25:12 +0200
Subject: [PATCH 35/42] update the lang filter module to include not tagged
 values if multiple languages selected

---
 lib/goo/base/settings/settings.rb             | 15 +++++++++++++--
 lib/goo/sparql/mixins/solution_lang_filter.rb | 10 ++++++----
 lib/goo/sparql/solutions_mapper.rb            |  2 +-
 3 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/lib/goo/base/settings/settings.rb b/lib/goo/base/settings/settings.rb
index e12ed475f..1be65e121 100644
--- a/lib/goo/base/settings/settings.rb
+++ b/lib/goo/base/settings/settings.rb
@@ -257,18 +257,29 @@ def shape_attribute(attr)
             self.instance_variable_set("@#{attr}",value)
           end
           define_method("#{attr}") do |*args|
+            attr_value = self.instance_variable_get("@#{attr}")
+
+            if self.class.not_show_all_languages?(attr_value, args)
+              is_array = attr_value.values.first.is_a?(Array)
+              attr_value = attr_value.values.flatten
+              attr_value = attr_value.first unless is_array
+            end
+
+
             if self.class.handler?(attr)
               if @loaded_attributes.include?(attr)
-                return self.instance_variable_get("@#{attr}")
+                return attr_value
               end
               value = self.send("#{self.class.handler(attr)}")
               self.instance_variable_set("@#{attr}",value)
               @loaded_attributes << attr
               return value
             end
+
             if (not @persistent) or @loaded_attributes.include?(attr)
-              return self.instance_variable_get("@#{attr}")
+                return attr_value
             else
+              # TODO: bug here when no labels from one of the main_lang available... (when it is called by ontologies_linked_data ontologies_submission)
               raise Goo::Base::AttributeNotLoaded, "Attribute `#{attr}` is not loaded for #{self.id}. Loaded attributes: #{@loaded_attributes.inspect}."
             end
           end
diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index dedc09fb8..1e9d8d880 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -51,7 +51,7 @@ def models_unmapped_to_array(m)
         def set_value(model, predicate, value, &block)
           language = object_language(value)
           
-          if requested_lang.eql?(:ALL) || !literal?(value) || (language_match?(language) && can_add_new_value(model,predicate, language))
+          if requested_lang.eql?(:ALL) || !literal?(value) || (language_match?(language) && can_add_new_value?(model,predicate, language))
               block.call
           end
 
@@ -62,7 +62,7 @@ def set_value(model, predicate, value, &block)
         end
 
 
-        def  can_add_new_value(model, predicate, new_language)
+        def  can_add_new_value?(model, predicate, new_language)
           old_val = model.send(predicate) rescue nil
           list_attributes?(predicate) || old_val.blank? || !no_lang?(new_language)
         end
@@ -114,7 +114,7 @@ def literal?(object)
 
         def store_objects_by_lang(id, predicate, object, language)
           # store objects in this format: [id][predicate][language] = [objects]
-          return if requested_lang.is_a?(Array) && !requested_lang.include?(language)
+          return if requested_lang.is_a?(Array) && !requested_lang.include?(language) && !language.eql?('@none')
 
           language_key = language.downcase
 
@@ -172,7 +172,9 @@ def show_all_languages?
 
         def get_language(languages)
           languages = portal_language if languages.nil? || languages.empty?
-          lang = languages.to_s.split(',').map { |l| l.upcase.to_sym }
+          lang = languages
+          lang = languages.to_s.split(',') unless lang.is_a?(Array)
+          lang = lang.map { |l| l.upcase.to_sym }
           lang.length == 1 ? lang.first : lang
         end
 
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index c93b203f7..9d9e65e2b 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -31,7 +31,7 @@ def map_each_solutions(select)
         list_attributes = Set.new(@klass.attributes(:list))
         all_attributes = Set.new(@klass.attributes(:all))
 
-        @lang_filter = Goo::SPARQL::Solution::LanguageFilter.new(requested_lang: @options[:requested_lang].to_s, unmapped: @unmapped,
+        @lang_filter = Goo::SPARQL::Solution::LanguageFilter.new(requested_lang: @options[:requested_lang], unmapped: @unmapped,
            list_attributes: list_attributes)
         
         if @options[:page]

From f34f2b08823a88bf0f83ddfb0deda9e7fbca83e8 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 17 Apr 2024 20:26:21 +0200
Subject: [PATCH 36/42] make map_attributes support multilingual

---
 lib/goo/base/resource.rb                      | 71 ++++++++++---------
 lib/goo/sparql/mixins/solution_lang_filter.rb |  6 +-
 test/test_languages_filters.rb                | 31 ++++++--
 3 files changed, 65 insertions(+), 43 deletions(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index e82265d47..e1adc6714 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -15,7 +15,7 @@ class Resource
       attr_reader :modified_attributes
       attr_reader :errors
       attr_reader :aggregates
-      attr_reader :unmapped
+      attr_writer :unmapped
 
       attr_reader :id
 
@@ -123,7 +123,12 @@ def missing_load_attributes
 
       def unmapped_set(attribute,value)
         @unmapped ||= {}
-        (@unmapped[attribute] ||= Set.new) << value
+        @unmapped[attribute] ||= Set.new
+        @unmapped[attribute].merge(Array(value)) unless value.nil?
+      end
+ 
+      def unmapped_get(attribute)
+        @unmapped[attribute]
       end
 
       def unmmaped_to_array
@@ -134,6 +139,12 @@ def unmmaped_to_array
         @unmapped = cpy
       end
 
+      def unmapped(*args)
+        @unmapped&.transform_values do  |language_values|
+          self.class.not_show_all_languages?(language_values, args) ?  language_values.values.flatten: language_values
+        end
+      end
+
       def delete(*args)
         if self.kind_of?(Goo::Base::Enum)
           unless args[0] && args[0][:init_enum]
@@ -211,13 +222,13 @@ def graph
         return col ? col.id : nil
       end
 
-      def self.map_attributes(inst,equivalent_predicates=nil)
+      def self.map_attributes(inst,equivalent_predicates=nil, include_languages: false)
         if (inst.kind_of?(Goo::Base::Resource) && inst.unmapped.nil?) ||
-            (!inst.respond_to?(:unmapped) && inst[:unmapped].nil?)
+          (!inst.respond_to?(:unmapped) && inst[:unmapped].nil?)
           raise ArgumentError, "Resource.map_attributes only works for :unmapped instances"
         end
         klass = inst.respond_to?(:klass) ? inst[:klass] : inst.class
-        unmapped = inst.respond_to?(:klass) ? inst[:unmapped] : inst.unmapped
+        unmapped = inst.respond_to?(:klass) ? inst[:unmapped] : inst.unmapped(include_languages: include_languages)
         list_attrs = klass.attributes(:list)
         unmapped_string_keys = Hash.new
         unmapped.each do |k,v|
@@ -228,36 +239,38 @@ def self.map_attributes(inst,equivalent_predicates=nil)
           next unless inst.respond_to?(attr)
           attr_uri = klass.attribute_uri(attr,inst.collection).to_s
           if unmapped_string_keys.include?(attr_uri.to_s) ||
-              (equivalent_predicates && equivalent_predicates.include?(attr_uri))
-            object = nil
+            (equivalent_predicates && equivalent_predicates.include?(attr_uri))
             if !unmapped_string_keys.include?(attr_uri)
-              equivalent_predicates[attr_uri].each do |eq_attr|
-                if object.nil? and !unmapped_string_keys[eq_attr].nil?
-                  object = unmapped_string_keys[eq_attr].dup
-                else
-                  if object.is_a?Array
-                    if !unmapped_string_keys[eq_attr].nil?
-                      object.concat(unmapped_string_keys[eq_attr])
-                    end
-                  end
+              object = Array(equivalent_predicates[attr_uri].map { |eq_attr| unmapped_string_keys[eq_attr] }).flatten.compact
+              if include_languages && [RDF::URI, Hash].all?{|c| object.map(&:class).include?(c)}
+                object = object.reduce({})  do |all, new_v|
+                  new_v =  { none: [new_v] } if new_v.is_a?(RDF::URI)
+                  all.merge(new_v) {|_, a, b| a + b }
                 end
+              elsif include_languages
+                object = object.first
               end
+
               if object.nil?
-                inst.send("#{attr}=",
-                           list_attrs.include?(attr) ? [] : nil, on_load: true)
+                inst.send("#{attr}=", list_attrs.include?(attr) ? [] : nil, on_load: true)
                 next
               end
             else
               object = unmapped_string_keys[attr_uri]
             end
-            object = object.map { |o| o.is_a?(RDF::URI) ? o : o.object }
+
+            if object.is_a?(Hash)
+              object = object.transform_values{|values| Array(values).map{|o|o.is_a?(RDF::URI) ? o : o.object}}
+            else
+              object = object.map {|o| o.is_a?(RDF::URI) ? o : o.object}
+            end
+
             if klass.range(attr)
               object = object.map { |o|
                 o.is_a?(RDF::URI) ? klass.range_object(attr,o) : o }
             end
-            unless list_attrs.include?(attr)
-              object = object.first
-            end
+
+            object = object.first unless list_attrs.include?(attr) || include_languages
             if inst.respond_to?(:klass)
               inst[attr] = object
             else
@@ -266,16 +279,12 @@ def self.map_attributes(inst,equivalent_predicates=nil)
           else
             inst.send("#{attr}=",
                       list_attrs.include?(attr) ? [] : nil, on_load: true)
-            if inst.id.to_s == "http://purl.obolibrary.org/obo/IAO_0000415"
-              if attr == :definition
-               # binding.pry
-              end
-            end
           end
 
         end
       end
 
+
       def collection
         opts = self.class.collection_opts
         if opts.instance_of?(Symbol)
@@ -414,12 +423,8 @@ def self.range_object(attr,id)
       end
 
       def self.find(id, *options)
-        if !id.instance_of?(RDF::URI) && self.name_with == :id
-          id = RDF::URI.new(id)
-        end
-        unless id.instance_of?(RDF::URI)
-          id = id_from_unique_attribute(name_with(),id)
-        end
+        id = RDF::URI.new(id) if !id.instance_of?(RDF::URI) && self.name_with == :id
+        id = id_from_unique_attribute(name_with(),id) unless id.instance_of?(RDF::URI)
         if self.inmutable? && self.inm_instances && self.inm_instances[id]
           w = Goo::Base::Where.new(self)
           w.instance_variable_set("@result", [self.inm_instances[id]])
diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 1e9d8d880..d93f20c5d 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -83,12 +83,11 @@ def model_group_by_lang(model)
         end
 
         def group_by_lang(values)
-
           return values.to_a if values.all?{|x| x.is_a?(RDF::URI) || !x.respond_to?(:language) }
 
-          values = values.group_by { |x| x.respond_to?(:language) && x.language ? x.language.to_s.downcase : :none }
+          values = values.group_by { |x| x.respond_to?(:language) && x.language ? x.language.to_s.downcase.to_sym : "@none" }
 
-          no_lang = values[:none] || []
+          no_lang = values["@none"] || []
           return no_lang if !no_lang.empty? && no_lang.all? { |x| x.respond_to?(:plain?) && !x.plain? }
 
           values
@@ -139,6 +138,7 @@ def add_unmapped_to_model(model, predicate, value)
 
         def pull_stored_values(model, values, predicate, unmapped)
           if unmapped
+            binding.pry
             add_unmapped_to_model(model, predicate, values)
           else
             values = values.map do  |language, values_literals|
diff --git a/test/test_languages_filters.rb b/test/test_languages_filters.rb
index 755c5d484..a1cc1fd29 100644
--- a/test/test_languages_filters.rb
+++ b/test/test_languages_filters.rb
@@ -10,13 +10,6 @@ class ExamplePerson < Goo::Base::Resource
   attribute :label, namespace: :rdf, enforce: [ :list ]
 end
 
-class ExamplePlace < Goo::Base::Resource
-  model :place, namespace: :bioportal, name_with: lambda { |k| k.id },
-        collection: :db
-  attribute :db, enforce: [ :database ]
-  attribute :label, namespace: :rdf, enforce: [ :list ]
-end
-
 class TestLanguageFilter < MiniTest::Unit::TestCase
   def self.before_suite
     RequestStore.store[:requested_lang] = Goo.main_languages.first
@@ -87,4 +80,28 @@ def test_language_not_found
     # will return only not tagged values if existent
     assert_equal ["Juan Pérez"], person.label
   end
+
+  def test_map_attribute_with_languages
+    RequestStore.store[:requested_lang] = :fr
+    person = ExamplePerson.find(@@person_id).in(@@db).include(:unmapped).first
+    ExamplePerson.map_attributes(person)
+    assert_equal ["Jean Dupont", "Juan Pérez"].sort, person.label.sort
+
+
+    expected_result = {:en=>["John Doe"], :fr=>["Jean Dupont"], "@none"=>["Juan Pérez"]}
+    RequestStore.store[:requested_lang] = [:en, :fr]
+    person = ExamplePerson.find(@@person_id).in(@@db).include(:unmapped).first
+    ExamplePerson.map_attributes(person)
+    assert_equal expected_result.values.flatten.sort.sort, person.label.sort
+
+
+    RequestStore.store[:requested_lang] = :all
+    person = ExamplePerson.find(@@person_id).in(@@db).include(:unmapped).first
+    ExamplePerson.map_attributes(person)
+    assert_equal expected_result.values.flatten.sort.sort, person.label.sort
+
+    person = ExamplePerson.find(@@person_id).in(@@db).include(:unmapped).first
+    ExamplePerson.map_attributes(person, include_languages: true)
+    assert_equal expected_result, person.label(include_languages: true)
+  end
 end

From 27fbe7d18f13078537a234023045d9f9f9caee64 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 17 Apr 2024 20:44:05 +0200
Subject: [PATCH 37/42] unskip test_embed_struct

---
 test/test_read_only.rb | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/test/test_read_only.rb b/test/test_read_only.rb
index 0b139e180..9855decf4 100644
--- a/test/test_read_only.rb
+++ b/test/test_read_only.rb
@@ -37,11 +37,17 @@ def test_struct_find
     end
 
     def test_embed_struct
-      skip "not yet"
+
       students = Student.where(enrolled: [university: [name: "Stanford"]])
                 .include(:name)
-                .include(enrolled: [:name, university: [ :address ]])
+                .include(enrolled: [:name, university: [ :address, :name ]])
                 .read_only.all
+
+      assert_equal 3, students.size
+      students.each do |st|
+        assert st.enrolled.any? {|e| e.is_a?(Struct) && e.university.name.eql?('Stanford')}
+      end
+
     end
   end
 end

From eba6251507c9bdb31885ef908be389a880d5a798 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 17 Apr 2024 20:54:15 +0200
Subject: [PATCH 38/42] add ruby-version file

---
 .ruby-version |  1 +
 Gemfile.lock  | 14 ++++++--------
 2 files changed, 7 insertions(+), 8 deletions(-)
 create mode 100644 .ruby-version

diff --git a/.ruby-version b/.ruby-version
new file mode 100644
index 000000000..6a81b4c83
--- /dev/null
+++ b/.ruby-version
@@ -0,0 +1 @@
+2.7.8
diff --git a/Gemfile.lock b/Gemfile.lock
index fdccf0596..4eae48b57 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -43,10 +43,11 @@ GEM
     docile (1.4.0)
     domain_name (0.6.20240107)
     eventmachine (1.2.7)
-    faraday (2.9.0)
-      faraday-net_http (>= 2.0, < 3.2)
-    faraday-net_http (3.1.0)
-      net-http
+    faraday (2.8.1)
+      base64
+      faraday-net_http (>= 2.0, < 3.1)
+      ruby2_keywords (>= 0.0.4)
+    faraday-net_http (3.0.2)
     http-accept (1.7.0)
     http-cookie (1.0.5)
       domain_name (~> 0.5)
@@ -63,8 +64,6 @@ GEM
     multi_json (1.15.0)
     mustermann (3.0.0)
       ruby2_keywords (~> 0.0.1)
-    net-http (0.4.1)
-      uri
     net-http-persistent (2.9.4)
     netrc (0.11.0)
     pry (0.14.2)
@@ -120,7 +119,6 @@ GEM
     thread_safe (0.3.6)
     tilt (2.3.0)
     tzinfo (0.3.62)
-    uri (0.13.0)
     uuid (2.3.9)
       macaddr (~> 1.0)
 
@@ -145,4 +143,4 @@ DEPENDENCIES
   uuid
 
 BUNDLED WITH
-   2.2.33
+   2.4.22

From fb04e7875974adc0b231596b2f2fb05481dd5d93 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 17 Apr 2024 21:09:25 +0200
Subject: [PATCH 39/42] fix multilingual support for AG backend

---
 lib/goo.rb                                    | 20 +------------------
 lib/goo/sparql/mixins/solution_lang_filter.rb |  8 ++++----
 2 files changed, 5 insertions(+), 23 deletions(-)

diff --git a/lib/goo.rb b/lib/goo.rb
index cfdf9ea31..e85d03d64 100644
--- a/lib/goo.rb
+++ b/lib/goo.rb
@@ -110,25 +110,7 @@ def self.add_sparql_backend(name, *opts)
     @@sparql_backends[name][:backend_name] = opts[:backend_name]
     @@sparql_backends.freeze
   end
-
-  def self.test_reset
-    if @@sparql_backends[:main][:query].url.to_s["localhost"].nil?
-      raise Exception, "only for testing"
-    end
-    @@sparql_backends[:main][:query]=Goo::SPARQL::Client.new("http://localhost:9000/sparql/",
-                 {protocol: "1.1", "Content-Type" => "application/x-www-form-urlencoded",
-                   read_timeout: 300,
-                  redis_cache: @@redis_client })
-  end
-
-  def self.main_lang
-    @@main_lang
-  end
-
-  def self.main_lang=(value)
-    @@main_lang = value
-  end
-
+  
   def self.use_cache=(value)
     @@use_cache = value
     set_sparql_cache
diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index d93f20c5d..80236efee 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -50,7 +50,7 @@ def models_unmapped_to_array(m)
 
         def set_value(model, predicate, value, &block)
           language = object_language(value)
-          
+
           if requested_lang.eql?(:ALL) || !literal?(value) || (language_match?(language) && can_add_new_value?(model,predicate, language))
               block.call
           end
@@ -102,9 +102,9 @@ def language_match?(language)
           # no_lang means that the object is not a literal
           return true if language.eql?(:no_lang)
 
-          return requested_lang.include?(language)  if requested_lang.is_a?(Array)
+          return requested_lang.include?(language.upcase)  if requested_lang.is_a?(Array)
 
-          language.eql?(requested_lang)
+          language.upcase.eql?(requested_lang)
         end
 
         def literal?(object)
@@ -113,7 +113,7 @@ def literal?(object)
 
         def store_objects_by_lang(id, predicate, object, language)
           # store objects in this format: [id][predicate][language] = [objects]
-          return if requested_lang.is_a?(Array) && !requested_lang.include?(language) && !language.eql?('@none')
+          return if requested_lang.is_a?(Array) && !requested_lang.include?(language.upcase) && !language.eql?('@none')
 
           language_key = language.downcase
 

From 563a47fa4329ffb8a774731be210f44a054726c0 Mon Sep 17 00:00:00 2001
From: mdorf <mdorf@stanford.edu>
Date: Wed, 24 Apr 2024 14:17:01 -0700
Subject: [PATCH 40/42] Gemfile.lock update after merging #150

---
 Gemfile.lock | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Gemfile.lock b/Gemfile.lock
index 4eae48b57..7283992d0 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -123,6 +123,7 @@ GEM
       macaddr (~> 1.0)
 
 PLATFORMS
+  x86_64-darwin-21
   x86_64-darwin-23
   x86_64-linux
 

From 5d911f587f22059581ae3e713e5fe9bde9de82e7 Mon Sep 17 00:00:00 2001
From: mdorf <mdorf@stanford.edu>
Date: Wed, 24 Apr 2024 14:24:28 -0700
Subject: [PATCH 41/42] Gemfile.lock update

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 7283992d0..9a8a96087 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/ncbo/sparql-client.git
-  revision: 1657f0dd69fd4b522d3549a6848670175f5e98cc
+  revision: 922874443616859d3af4eca8f818934a27847559
   branch: develop
   specs:
     sparql-client (1.0.1)

From 33583fd1c1d72b449cd6bb91815e3422b8448983 Mon Sep 17 00:00:00 2001
From: mdorf <mdorf@stanford.edu>
Date: Wed, 24 Apr 2024 14:32:04 -0700
Subject: [PATCH 42/42] Gemfile.lock update

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 9a8a96087..7283992d0 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/ncbo/sparql-client.git
-  revision: 922874443616859d3af4eca8f818934a27847559
+  revision: 1657f0dd69fd4b522d3549a6848670175f5e98cc
   branch: develop
   specs:
     sparql-client (1.0.1)