From 89fb1c84a2f72ff6ac6b9b76aa1d438eab9d45fa Mon Sep 17 00:00:00 2001 From: Don Restarone <35935196+donrestarone@users.noreply.github.com> Date: Wed, 15 Mar 2023 16:09:49 -0400 Subject: [PATCH] [feature] Ability to query API resources with KEYWORDS (#1466) Addresses: https://github.com/restarone/violet_rails/issues/1374 **DEMO** https://user-images.githubusercontent.com/25191509/214291392-23e3c962-31ce-4df3-b3fa-1a2565c978be.mov ### **For client-engineers** For http-request to search by `KEYWORDS`, we would need to pass `option: KEYWORDS` as key-value pair for the attributes we want to search by in `properties` payload. In the clip below, we are filtering `cars` by searching in its attributes, namely: `make`, `model` and `type` with `KEYWORDS` option. https://user-images.githubusercontent.com/25191509/224547568-37ff1e9c-51e0-4212-bfa4-dd123c9857ac.mov Payload Example: ``` properties: {"make":{"value":"kia tesla","option":"KEYWORDS"},"model":{"value":"kia tesla","option":"KEYWORDS"},"type":{"value":"kia tesla","option":"KEYWORDS"}}, match: ANY ``` Co-authored-by: Prashant Khadka <25191509+alis-khadka@users.noreply.github.com> --- .../comfy/admin/api_namespaces_controller.rb | 7 - app/helpers/api_namespaces_helper.rb | 23 ++ .../concerns/jsonb_search/query_builder.rb | 54 ++++- .../comfy/admin/api_namespaces/show.html.haml | 10 +- .../comfy/api_namespaces_controller_test.rb | 9 +- .../api/resource_controller_test.rb | 211 ++++++++++++++++++ .../jsonb_search/query_builder_test.rb | 32 +++ 7 files changed, 330 insertions(+), 16 deletions(-) diff --git a/app/controllers/comfy/admin/api_namespaces_controller.rb b/app/controllers/comfy/admin/api_namespaces_controller.rb index f1b78bd8d..228b13415 100755 --- a/app/controllers/comfy/admin/api_namespaces_controller.rb +++ b/app/controllers/comfy/admin/api_namespaces_controller.rb @@ -43,13 +43,6 @@ def show field, direction = params[:q].key?(:s) ? params[:q][:s].split(" ") : [nil, nil] fields_in_properties = @api_namespace.properties.keys - @custom_properties = {} - @api_namespace.properties.values.each_with_index do |obj,index| - if(obj.present? && obj != "nil" && obj != "\"\"") - @custom_properties[fields_in_properties[index]] = obj; - end - end - @custom_properties = JSON.parse(@custom_properties.to_json, object_class: OpenStruct).to_s.gsub(/=/,': ').gsub(/#/,'}').gsub("\\", "'").gsub(/"'"/,'"').gsub(/'""/,'"') @image_options = @api_namespace.non_primitive_properties.select { |non_primitive_property| non_primitive_property.field_type == 'file' }.pluck(:label) # check if we are sorting by a field inside properties jsonb column if field && fields_in_properties.include?(field) diff --git a/app/helpers/api_namespaces_helper.rb b/app/helpers/api_namespaces_helper.rb index 865e435dc..d52746b25 100755 --- a/app/helpers/api_namespaces_helper.rb +++ b/app/helpers/api_namespaces_helper.rb @@ -10,4 +10,27 @@ def graphql_base_url(subdomain, namespace) def system_paths Comfy::Cms::Page.all.pluck(:full_path) end + + def api_html_renderer_dynamic_properties(namespace, search_option = nil) + custom_properties = {} + fields_in_properties = namespace.properties.keys + + namespace.properties.values.each_with_index do |obj,index| + if obj.present? && obj != "nil" && obj != "\"\"" + if search_option.present? + next if !obj.is_a?(Array) && !obj.is_a?(String) + + custom_properties[fields_in_properties[index]] = { + value: obj.is_a?(Array) ? obj.first(1) : obj.split.first, + option: search_option + } + else + custom_properties[fields_in_properties[index]] = obj; + end + end + end + + # sanitize the text to properly display + JSON.parse(custom_properties.to_json, object_class: OpenStruct).to_s.gsub(/=/,': ').gsub(/#/,' }').gsub("\\", "'").gsub(/"'"/,'"').gsub(/'""/,'"') + end end diff --git a/app/models/concerns/jsonb_search/query_builder.rb b/app/models/concerns/jsonb_search/query_builder.rb index 1b59f70b8..7c138698a 100644 --- a/app/models/concerns/jsonb_search/query_builder.rb +++ b/app/models/concerns/jsonb_search/query_builder.rb @@ -2,13 +2,19 @@ module JsonbSearch module QueryBuilder QUERY_OPTION = { EXACT: 'EXACT', - PARTIAL: 'PARTIAL' + PARTIAL: 'PARTIAL', + KEYWORDS: 'KEYWORDS' }.freeze MATCH_OPTION = { ALL: 'ALL', ANY: 'ANY' - }.freeze + }.freeze + + # https://github.com/Altoros/belarus-ruby-on-rails/blob/master/solr/conf/stopwords.txt + STOP_WORDS = [ + 'a', 'an', 'and', 'are', 'as', 'at', 'be', 'but', 'by', 'for', 'if', 'in', 'into', 'is', 'it', 'no', 'not', 'of', 'on', 'or', 's', 'such', 't', 'that', 'the', 'their', 'then', 'there', 'these', 'they', 'this', 'to', 'was', 'will', 'with' + ] class << self def build_jsonb_query(column_name, query_params, match = nil) @@ -23,12 +29,15 @@ def parse_params(query_params) query_params.each do |key, value| if value.is_a?(Hash) && value.key?(:value) + next unless value[:value].present? # { name: { value: 'violet', query }} || { name: { value: 'violet', option: 'EXACT' }} queries << { option: value[:option] || QUERY_OPTION[:EXACT], key: key, value: value[:value], match: value[:match] } elsif value.is_a?(Hash) # { foo: { bar: 'baz', wat: 'up' }} value.each do |k, v| if v.is_a?(Hash) && v.key?(:value) + next unless v[:value].present? + queries << { key: key, value: [{ option: v[:option] || QUERY_OPTION[:EXACT], key: k, value: v[:value], match: v[:match] }] } else queries << { key: key, value: [{ option: QUERY_OPTION[:EXACT], key: k, value: v }]} @@ -75,12 +84,33 @@ def generate_query(param, query_string) end end - # "column ->> 'property' = 'term'" + # "column ->> 'property' = 'term'" def string_query(key, term, option, query) - if option == QUERY_OPTION[:PARTIAL] + if option == QUERY_OPTION[:KEYWORDS] + query_array = [] + operator = 'LIKE' + + terms = term.split(' ') - STOP_WORDS + terms = terms.map { |txt| "%#{txt}%" } + + terms.each do |txt| + query_array << generate_string_sql(query, key, txt, operator) + end + + query_array << generate_string_sql(query, key, "%#{term}%", operator) if terms.size > 1 + + query_array.join(' OR ') + elsif option == QUERY_OPTION[:PARTIAL] term = "%#{term}%" operator = 'LIKE' + + generate_string_sql(query, key, term, operator) + else + generate_string_sql(query, key, term) end + end + + def generate_string_sql(query, key, term, operator = nil) # A ' inside a string quoted with ' may be written as ''. # https://stackoverflow.com/questions/54144340/how-to-query-jsonb-fields-and-values-containing-single-quote-in-rails#comment95120456_54144340 # https://dev.mysql.com/doc/refman/8.0/en/string-literals.html#character-escape-sequences @@ -95,7 +125,21 @@ def hash_query(key, term, option, query) # "column -> 'property' ? '['term']'" def array_query(key, term, option, query, match) - if option == QUERY_OPTION[:PARTIAL] + if option == QUERY_OPTION[:KEYWORDS] + query_array = [] + operator = 'LIKE' + + term.each do |data| + items = data.split(' ') - STOP_WORDS + items = items.map { |txt| "%#{txt}%" } + + items.each do |item| + query_array << "lower(#{query} ->> '#{key}'::text) LIKE lower('#{item.to_s.gsub("'", "''")}')" + end + end + + query_array.join(match == MATCH_OPTION[:ANY] ? ' OR ' : ' AND ') + elsif option == QUERY_OPTION[:PARTIAL] match == MATCH_OPTION[:ANY] ? term.map { |q| "#{query} -> '#{key}' ? '#{q}'" }.join(' OR ') : "#{query} -> '#{key}' @> '#{term.to_json.gsub("'", "''")}'" else "#{query} -> '#{key}' @> '#{term.to_json}' AND #{query} -> '#{key}' <@ '#{term.to_json.gsub("'", "''")}'" diff --git a/app/views/comfy/admin/api_namespaces/show.html.haml b/app/views/comfy/admin/api_namespaces/show.html.haml index 2888d0c79..1f31eef40 100755 --- a/app/views/comfy/admin/api_namespaces/show.html.haml +++ b/app/views/comfy/admin/api_namespaces/show.html.haml @@ -115,10 +115,16 @@ %pre= @api_namespace.snippet %p %b API HTML Renderer index snippet: - %pre= "{{ cms:helper render_api_namespace_resource_index '#{@api_namespace.slug}', scope: { properties: #{@custom_properties} } }}" + %pre= "{{ cms:helper render_api_namespace_resource_index '#{@api_namespace.slug}', scope: { properties: #{api_html_renderer_dynamic_properties(@api_namespace)} } }}" + %p + %b API HTML Renderer index snippet (KEYWORDS - works for array and string data type only): + %pre= "{{ cms:helper render_api_namespace_resource_index '#{@api_namespace.slug}', scope: { properties: #{api_html_renderer_dynamic_properties(@api_namespace, 'KEYWORDS')} } }}" %p %b API HTML Renderer show snippet: - %pre= "{{ cms:helper render_api_namespace_resource '#{@api_namespace.slug}', scope: { properties: #{@custom_properties} } }}" + %pre= "{{ cms:helper render_api_namespace_resource '#{@api_namespace.slug}', scope: { properties: #{api_html_renderer_dynamic_properties(@api_namespace)} } }}" + %p + %b API HTML Renderer show snippet (KEYWORDS - works for array and string data type only): + %pre= "{{ cms:helper render_api_namespace_resource '#{@api_namespace.slug}', scope: { properties: #{api_html_renderer_dynamic_properties(@api_namespace, 'KEYWORDS')} } }}" %p .d-flex.justify-content-between %b Preview (outer border is present in preview only): diff --git a/test/controllers/admin/comfy/api_namespaces_controller_test.rb b/test/controllers/admin/comfy/api_namespaces_controller_test.rb index c7e4529e9..bad0a0c70 100755 --- a/test/controllers/admin/comfy/api_namespaces_controller_test.rb +++ b/test/controllers/admin/comfy/api_namespaces_controller_test.rb @@ -783,9 +783,14 @@ class Comfy::Admin::ApiNamespacesControllerTest < ActionDispatch::IntegrationTes assert_select "b", {count: 1, text: "Form rendering snippet:"} assert_select "pre", {count: 1, text: @api_namespace.snippet} assert_select "b", {count: 1, text: "API HTML Renderer index snippet:"} - assert_select "pre", {count: 1, text: "{{ cms:helper render_api_namespace_resource_index '#{@api_namespace.slug}', scope: { properties: { arr: [1, 2, 3], obj: { a: \"b\", c: \"d\"}, title: \"Hello World\", test_id: 123, alpha_arr: [\"a\", \"b\"], published: true} } }}"} + assert_select "pre", {count: 1, text: "{{ cms:helper render_api_namespace_resource_index '#{@api_namespace.slug}', scope: { properties: { arr: [1, 2, 3], obj: { a: \"b\", c: \"d\" }, title: \"Hello World\", test_id: 123, alpha_arr: [\"a\", \"b\"], published: true } } }}"} assert_select "b", {count: 1, text: "API HTML Renderer show snippet:"} - assert_select "pre", {count: 1, text: "{{ cms:helper render_api_namespace_resource '#{@api_namespace.slug}', scope: { properties: { arr: [1, 2, 3], obj: { a: \"b\", c: \"d\"}, title: \"Hello World\", test_id: 123, alpha_arr: [\"a\", \"b\"], published: true} } }}"} + assert_select "pre", {count: 1, text: "{{ cms:helper render_api_namespace_resource '#{@api_namespace.slug}', scope: { properties: { arr: [1, 2, 3], obj: { a: \"b\", c: \"d\" }, title: \"Hello World\", test_id: 123, alpha_arr: [\"a\", \"b\"], published: true } } }}"} + # Dynamic renderer snippet for KEYWORDS based search + assert_select "b", {count: 1, text: "API HTML Renderer index snippet (KEYWORDS - works for array and string data type only):"} + assert_select "pre", {count: 1, text: "{{ cms:helper render_api_namespace_resource_index '#{@api_namespace.slug}', scope: { properties: { arr: { value: [1], option: \"KEYWORDS\" }, title: { value: \"Hello\", option: \"KEYWORDS\" }, alpha_arr: { value: [\"a\"], option: \"KEYWORDS\" } } } }}"} + assert_select "b", {count: 1, text: "API HTML Renderer show snippet (KEYWORDS - works for array and string data type only):"} + assert_select "pre", {count: 1, text: "{{ cms:helper render_api_namespace_resource '#{@api_namespace.slug}', scope: { properties: { arr: { value: [1], option: \"KEYWORDS\" }, title: { value: \"Hello\", option: \"KEYWORDS\" }, alpha_arr: { value: [\"a\"], option: \"KEYWORDS\" } } } }}"} end ######## API Accessibility Tests - START ######### diff --git a/test/controllers/api/resource_controller_test.rb b/test/controllers/api/resource_controller_test.rb index 2c521d494..b379386db 100644 --- a/test/controllers/api/resource_controller_test.rb +++ b/test/controllers/api/resource_controller_test.rb @@ -300,6 +300,122 @@ class Api::ResourceControllerTest < ActionDispatch::IntegrationTest assert_empty response.parsed_body["data"] end + test '#index search jsonb field - string - KEYWORDS: multi word string' do + @api_resource_1.update(properties: {name: 'Professional Writer'}) + @api_resource_2.update(properties: {name: 'Physical Development'}) + @api_resource_3.update(properties: {name: 'Professional Development'}) + + payload = { + properties: { + name: { + value: 'professional development', + option: 'KEYWORDS' + } + } + } + get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json + assert_response :success + + assert_equal response.parsed_body["data"].pluck("id").map(&:to_i).sort, [@api_resource_1.id, @api_resource_2.id, @api_resource_3.id].sort + end + + test '#index search jsonb field - string - KEYWORDS: multi word string (unhappy)' do + @api_resource_1.update(properties: {name: 'Professional Writer'}) + @api_resource_2.update(properties: {name: 'Physical Development'}) + @api_resource_3.update(properties: {name: 'Professional Development'}) + + payload = { + properties: { + name: { + value: 'hello world', + option: 'KEYWORDS' + } + } + } + get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json + assert_response :success + + assert_empty response.parsed_body["data"] + end + + test '#index search jsonb field - Array - KEYWORDS: match ALL' do + @api_resource_1.update(properties: {tags: ['Professional Writer', 'zebra']}) + @api_resource_2.update(properties: {tags: ['Physical Development', 'cow']}) + @api_resource_3.update(properties: {tags: ['Professional Development', 'animal']}) + + payload = { + properties: { + tags: { + value: ['professional development'], + option: 'KEYWORDS' + } + } + } + get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json + assert_response :success + + assert_equal response.parsed_body["data"].pluck("id").map(&:to_i).sort, [@api_resource_3.id].sort + end + + test '#index search jsonb field - Array - KEYWORDS: match ALL (unhappy)' do + @api_resource_1.update(properties: {tags: ['Professional Writer', 'zebra']}) + @api_resource_2.update(properties: {tags: ['Physical Development', 'cow']}) + @api_resource_3.update(properties: {tags: ['Professional Development', 'animal']}) + + payload = { + properties: { + tags: { + value: ['hello world'], + option: 'KEYWORDS' + } + } + } + get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json + assert_response :success + + assert_empty response.parsed_body["data"] + end + + test '#index search jsonb field - Array - KEYWORDS: match ANY' do + @api_resource_1.update(properties: {tags: ['Professional Writer', 'zebra']}) + @api_resource_2.update(properties: {tags: ['Physical Development', 'cow']}) + @api_resource_3.update(properties: {tags: ['Professional Development', 'animal']}) + + payload = { + properties: { + tags: { + value: ['professional development'], + option: 'KEYWORDS', + match: 'ANY' + } + } + } + get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json + assert_response :success + + assert_equal response.parsed_body["data"].pluck("id").map(&:to_i).sort, [@api_resource_1.id, @api_resource_2.id, @api_resource_3.id].sort + end + + test '#index search jsonb field - Array - KEYWORDS: match ANY (unhappy)' do + @api_resource_1.update(properties: {tags: ['Professional Writer', 'zebra']}) + @api_resource_2.update(properties: {tags: ['Physical Development', 'cow']}) + @api_resource_3.update(properties: {tags: ['Professional Development', 'animal']}) + + payload = { + properties: { + tags: { + value: ['hello world'], + option: 'KEYWORDS', + match: 'ANY' + } + } + } + get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json + assert_response :success + + assert_empty response.parsed_body["data"] + end + test '#index search jsonb field - nested string' do payload = { properties: { @@ -484,4 +600,99 @@ class Api::ResourceControllerTest < ActionDispatch::IntegrationTest assert_empty response.parsed_body["data"] end + + test '#index search jsonb field - array - KEYWORDS match ALL' do + @api_resource_2.update(properties: {interests: ['hello world', 'foo', 'bar']}) + payload = { + properties: { + interests: { + value: ['hello world', 'foo'], + option: 'KEYWORDS' + } + } + } + + get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json + + assert_equal response.parsed_body["data"].pluck("id").map(&:to_i).sort, [@api_resource_2.id].sort + end + + test '#index search jsonb field - array - KEYWORDS match ANY' do + @api_resource_1.update(properties: {interests: ['hello']}) + @api_resource_2.update(properties: {interests: ['hello world', 'foo', 'bar']}) + + payload = { + properties: { + interests: { + value: ['hello world', 'foo'], + option: 'KEYWORDS', + match: 'ANY' + } + } + } + + get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json + + assert_equal response.parsed_body["data"].pluck("id").map(&:to_i).sort, [@api_resource_1.id, @api_resource_2.id].sort + end + + test '#index search jsonb field - string - ignores empty values without throwing exception' do + @api_resource_1.update(properties: {name: 'Professional Writer', age: 11}) + @api_resource_2.update(properties: {name: 'Physical Development', age: 22}) + @api_resource_3.update(properties: {name: 'Professional Development', age: 33}) + + payload = { + properties: { + name: { + value: '', + option: 'KEYWORDS' + }, + age: { + value: '', + option: 'KEYWORDS' + }, + } + } + + get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json + # Does not throw exception + assert_response :success + + response_resource_ids = response.parsed_body["data"].pluck("id").map(&:to_i).sort + + # Ignores empty value parameters and the filters are not applied + assert_includes response_resource_ids, @api_resource_1.id + assert_includes response_resource_ids, @api_resource_2.id + assert_includes response_resource_ids, @api_resource_3.id + end + + test '#index search jsonb field - array - ignores empty values without throwing exception' do + @api_resource_1.update(properties: {name: 'Professional Writer', age: 11}) + @api_resource_2.update(properties: {name: 'Physical Development', age: 22}) + @api_resource_3.update(properties: {name: 'Professional Development', age: 33}) + + payload = { + properties: { + name: { + value: [], + option: 'KEYWORDS' + }, + age: { + value: [], + option: 'KEYWORDS' + }, + } + } + + get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json + # Does not throw exception + assert_response :success + + response_resource_ids = response.parsed_body["data"].pluck("id").map(&:to_i).sort + + # Ignores empty value parameters and the filters are not applied + assert_includes response_resource_ids, @api_resource_1.id + assert_includes response_resource_ids, @api_resource_2.id + assert_includes response_resource_ids, @api_resource_3.id + end end diff --git a/test/models/concerns/jsonb_search/query_builder_test.rb b/test/models/concerns/jsonb_search/query_builder_test.rb index 4664db01f..9817add96 100644 --- a/test/models/concerns/jsonb_search/query_builder_test.rb +++ b/test/models/concerns/jsonb_search/query_builder_test.rb @@ -43,6 +43,24 @@ class JsonbSearch::QueryBuilderTest < ActiveSupport::TestCase assert_equal "lower(properties ->> 'name') = lower('violet') AND lower(properties ->> 'age') = lower('20')", jsonb_query end + test 'query string - KEYWORDS: splits the provided value into words' do + query = { name: { value: 'violet rails development', option: 'KEYWORDS' } } + jsonb_query = JsonbSearch::QueryBuilder.build_jsonb_query(:properties, query) + + expected_query = "lower(properties ->> 'name') LIKE lower('%violet%') OR lower(properties ->> 'name') LIKE lower('%rails%') OR lower(properties ->> 'name') LIKE lower('%development%') OR lower(properties ->> 'name') LIKE lower('%violet rails development%')" + + assert_equal expected_query, jsonb_query + end + + test 'query string - KEYWORDS: splits the provided value into words by skipping the individual query for stopwords' do + query = { name: { value: 'a hope for future', option: 'KEYWORDS' } } + jsonb_query = JsonbSearch::QueryBuilder.build_jsonb_query(:properties, query) + + expected_query = "lower(properties ->> 'name') LIKE lower('%hope%') OR lower(properties ->> 'name') LIKE lower('%future%') OR lower(properties ->> 'name') LIKE lower('%a hope for future%')" + + assert_equal expected_query, jsonb_query + end + test 'query string - nested' do query = { foo: { bar: 'baz' } } jsonb_query = JsonbSearch::QueryBuilder.build_jsonb_query(:properties, query) @@ -92,6 +110,20 @@ class JsonbSearch::QueryBuilderTest < ActiveSupport::TestCase assert_equal "properties -> 'array' ? '#{query[:array][:value][0]}' OR properties -> 'array' ? '#{query[:array][:value][1]}'", jsonb_query end + test 'query array - KEYWORDS match ALL' do + query = { array: { value: ['hello world', 'bar'], option: 'KEYWORDS', match: 'ALL' } } + jsonb_query = JsonbSearch::QueryBuilder.build_jsonb_query(:properties, query) + + assert_equal "lower(properties ->> 'array'::text) LIKE lower('%hello%') AND lower(properties ->> 'array'::text) LIKE lower('%world%') AND lower(properties ->> 'array'::text) LIKE lower('%bar%')", jsonb_query + end + + test 'query array - KEYWORDS match ANY' do + query = { array: { value: ['hello world', 'bar'], option: 'KEYWORDS', match: 'ANY' } } + jsonb_query = JsonbSearch::QueryBuilder.build_jsonb_query(:properties, query) + + assert_equal "lower(properties ->> 'array'::text) LIKE lower('%hello%') OR lower(properties ->> 'array'::text) LIKE lower('%world%') OR lower(properties ->> 'array'::text) LIKE lower('%bar%')", jsonb_query + end + test 'query array - nested' do query = { foo: { array: ['foo', 'bar'] } } jsonb_query = JsonbSearch::QueryBuilder.build_jsonb_query(:properties, query)