diff --git a/config/dev.exs b/config/dev.exs index 62a346395..1bc4fcd53 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -65,6 +65,9 @@ config :code_corps, postmark_project_request_template: "123", postmark_receipt_template: "123" +# Configure elasticsearch +config :code_corps, :elasticsearch_url, "http://0.0.0.0:9200" + # If the dev environment has no CLOUDEX_API_KEY set, we want the app # to still run, with cloudex in test API mode if System.get_env("CLOUDEX_API_KEY") == nil do diff --git a/config/prod.exs b/config/prod.exs index 4058f1e00..8f88cf3dc 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -52,6 +52,9 @@ config :code_corps, :analytics, CodeCorps.Analytics.SegmentAPI config :code_corps, :stripe, Stripe config :code_corps, :stripe_env, :prod +# Configure elasticsearch +config :code_corps, :elasticsearch_url, "http://0.0.0.0:9200" + config :sentry, environment_name: Mix.env || :prod diff --git a/config/test.exs b/config/test.exs index 95d2a1842..57325d3cb 100644 --- a/config/test.exs +++ b/config/test.exs @@ -38,6 +38,9 @@ config :code_corps, :stripe_env, :test config :code_corps, :icon_color_generator, CodeCorps.RandomIconColor.TestGenerator +# Configure elasticsearch +config :code_corps, :elasticsearch_url, "http://0.0.0.0:9200" + # Set Corsica logging to output no console warning when rejecting a request config :code_corps, :corsica_log_level, [rejected: :debug] @@ -54,6 +57,10 @@ config :code_corps, github_app_client_secret: System.get_env("GITHUB_TEST_APP_CLIENT_SECRET"), github_app_pem: pem +# Configure elasticsearch +config :code_corps, :elasticsearch_url, "http://0.0.0.0:9200" +config :code_corps, :elasticsearch_index, "skills" + config :sentry, environment_name: Mix.env || :test diff --git a/lib/code_corps/helpers/elastic_search_helper.ex b/lib/code_corps/helpers/elastic_search_helper.ex new file mode 100644 index 000000000..e9097b98e --- /dev/null +++ b/lib/code_corps/helpers/elastic_search_helper.ex @@ -0,0 +1,94 @@ +defmodule CodeCorps.ElasticSearchHelper do + alias Elastix.Search + alias Elastix.Index + alias Elastix.Document + + def delete(url, index) do + Index.delete(url, index) + end + + def create_index(url, index, type) do + Index.settings(url, index, settings_map()) + Index.settings(url, "#{index}/_mapping/#{type}", field_filter(type)) + end + + def add_documents(url, index, type, documents) when is_list(documents) do + add_documents(url, index, type, documents, []) + end + + def add_documents(url, index, type, documents, query) when is_list(documents) do + Enum.each(documents, fn(x) -> add_document(url, index, type, x, query) end) + end + + def add_document(url, index, type, data) do + add_document(url, index, type, data, []) + end + + def add_document(url, index, type, data, query) do + Document.index_new(url, index, type, data, query) + end + + def search(url, index, type, search_query) do + data = %{ + query: %{ + match: %{"#{type}": search_query} + } + } + Search.search(url, index, [], data) |> process_response(type) + end + + def match_all(url, index, type) do + data = %{ + query: %{ + match_all: %{} + } + } + Search.search(url, index, [], data) |> process_response(type) + end + + def process_response(%HTTPoison.Response{status_code: 200} = response, type) do + response.body["hits"]["hits"] |> Enum.map(fn(x) -> x["_source"] end) + end + + def process_response(_), do: [] + + defp settings_map do + %{ + settings: %{ + number_of_shards: 5, + analysis: %{ + filter: %{ + autocomplete_filter: %{ + type: "edge_ngram", + min_gram: 2, + max_gram: 20 + } + }, + analyzer: %{ + autocomplete: %{ + type: "custom", + tokenizer: "standard", + filter: [ + "lowercase", + "autocomplete_filter" + ] + } + } + } + } + } + end + + def field_filter(type) do + %{ + "#{type}" => %{ + "properties" => %{ + "#{type}" => %{ + "type" => "string", + "analyzer" => "autocomplete" + } + } + } + } + end +end diff --git a/lib/code_corps_web/controllers/skill_controller.ex b/lib/code_corps_web/controllers/skill_controller.ex index e3070b111..d4e23e283 100644 --- a/lib/code_corps_web/controllers/skill_controller.ex +++ b/lib/code_corps_web/controllers/skill_controller.ex @@ -33,6 +33,14 @@ defmodule CodeCorpsWeb.SkillController do end end + @elasticsearch_index "skills" + @elasticsearch_type "title" + @elasticsearch_url Application.get_env(:code_corps, :elasticsearch_url) + + def search(_conn, %{query: query}) do + CodeCorps.ElasticSearchHelper.search(@elasticsearch_url, @elasticsearch_index, @elasticsearch_type, query) + end + @spec load_skills(map) :: list(Skill.t) defp load_skills(%{} = params) do Skill diff --git a/lib/code_corps_web/router.ex b/lib/code_corps_web/router.ex index 25eed5668..1333b912a 100644 --- a/lib/code_corps_web/router.ex +++ b/lib/code_corps_web/router.ex @@ -82,6 +82,7 @@ defmodule CodeCorpsWeb.Router do resources "/role-skills", RoleSkillController, only: [:create, :delete] resources "/roles", RoleController, only: [:create] resources "/skills", SkillController, only: [:create] + resources "/skills/search", SkillController, only: [:show] resources "/stripe-connect-accounts", StripeConnectAccountController, only: [:show, :create, :update] resources "/stripe-connect-plans", StripeConnectPlanController, only: [:show, :create] resources "/stripe-connect-subscriptions", StripeConnectSubscriptionController, only: [:show, :create] diff --git a/mix.exs b/mix.exs index 355912ea6..937970f7f 100644 --- a/mix.exs +++ b/mix.exs @@ -81,7 +81,8 @@ defmodule CodeCorps.Mixfile do {:sweet_xml, "~> 0.5"}, {:timber, "~> 2.0"}, # Logging {:timex, "~> 3.0"}, - {:timex_ecto, "~> 3.0"} + {:timex_ecto, "~> 3.0"}, + {:elastix, git: "https://github.com/paulsullivanjr/elastix.git"} # for elastic search ] end diff --git a/mix.lock b/mix.lock index 04e3ecfee..9f30b7047 100644 --- a/mix.lock +++ b/mix.lock @@ -20,13 +20,14 @@ "earmark": {:hex, :earmark, "1.2.3", "206eb2e2ac1a794aa5256f3982de7a76bf4579ff91cb28d0e17ea2c9491e46a4", [:mix], []}, "ecto": {:hex, :ecto, "2.2.6", "3fd1067661d6d64851a0d4db9acd9e884c00d2d1aa41cc09da687226cf894661", [:mix], [{:db_connection, "~> 1.1", [hex: :db_connection, optional: true]}, {:decimal, "~> 1.2", [hex: :decimal, optional: false]}, {:mariaex, "~> 0.8.0", [hex: :mariaex, optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, optional: true]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: false]}, {:postgrex, "~> 0.13.0", [hex: :postgrex, optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, optional: true]}]}, "ecto_ordered": {:hex, :ecto_ordered, "0.2.0-beta1", "cb066bc608f1c8913cea85af8293261720e6a88e3c99061e6877d7025352f045", [:mix], [{:ecto, "~> 2.0", [hex: :ecto, optional: false]}]}, - "elixir_make": {:hex, :elixir_make, "0.4.0", "992f38fabe705bb45821a728f20914c554b276838433349d4f2341f7a687cddf", [:mix], []}, - "ex_aws": {:hex, :ex_aws, "1.1.4", "4bdc4fff91f8d35c7fe2355b9da54cc51f980c92f1137715d8b2d70d8e8511cc", [:mix], [{:configparser_ex, "~> 0.2.1", [hex: :configparser_ex, optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, optional: true]}]}, - "ex_doc": {:hex, :ex_doc, "0.18.1", "37c69d2ef62f24928c1f4fdc7c724ea04aecfdf500c4329185f8e3649c915baf", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, optional: false]}]}, - "ex_machina": {:hex, :ex_machina, "2.1.0", "4874dc9c78e7cf2d429f24dc3c4005674d4e4da6a08be961ffccc08fb528e28b", [:mix], [{:ecto, "~> 2.1", [hex: :ecto, optional: true]}]}, + "elastix": {:git, "https://github.com/paulsullivanjr/elastix.git", "72441f08d59491ec1101b8bb9afe56463a5cbd75", []}, + "elixir_make": {:hex, :elixir_make, "0.4.0", "992f38fabe705bb45821a728f20914c554b276838433349d4f2341f7a687cddf", [:mix], [], "hexpm"}, + "ex_aws": {:hex, :ex_aws, "1.1.5", "789173f385934f7e27f9ef36692a6c5f7dde06fd6e6f64d4cd92cda613d34bf9", [:mix], [{:configparser_ex, "~> 0.2.1", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"}, + "ex_doc": {:hex, :ex_doc, "0.18.1", "37c69d2ef62f24928c1f4fdc7c724ea04aecfdf500c4329185f8e3649c915baf", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, repo: "hexpm", optional: false]}], "hexpm"}, + "ex_machina": {:hex, :ex_machina, "2.1.0", "4874dc9c78e7cf2d429f24dc3c4005674d4e4da6a08be961ffccc08fb528e28b", [:mix], [{:ecto, "~> 2.1", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "excoveralls": {:hex, :excoveralls, "0.7.5", "339e433e5d3bce09400dc8de7b9040741a409c93917849916c136a0f51fdc183", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: false]}, {:hackney, ">= 0.12.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, "exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm"}, - "file_system": {:hex, :file_system, "0.2.2", "7f1e9de4746f4eb8a4ca8f2fbab582d84a4e40fa394cce7bfcb068b988625b06", [], [], "hexpm"}, + "file_system": {:hex, :file_system, "0.2.2", "7f1e9de4746f4eb8a4ca8f2fbab582d84a4e40fa394cce7bfcb068b988625b06", [:mix], [], "hexpm"}, "fs": {:hex, :fs, "0.9.2", "ed17036c26c3f70ac49781ed9220a50c36775c6ca2cf8182d123b6566e49ec59", [:rebar], []}, "gettext": {:hex, :gettext, "0.13.1", "5e0daf4e7636d771c4c71ad5f3f53ba09a9ae5c250e1ab9c42ba9edccc476263", [:mix], []}, "guardian": {:hex, :guardian, "1.0.0", "21bae2a8c0b4ed5943d9da0c6aeb16e52874c1f675de5d7920ae35471c6263f9", [:mix], [{:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.0 or ~> 1.2 or ~> 1.3", [hex: :phoenix, repo: "hexpm", optional: true]}, {:plug, "~> 1.3.3 or ~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}, {:uuid, ">= 1.1.1", [hex: :uuid, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/integration/skill_controller_search_test.exs b/test/integration/skill_controller_search_test.exs new file mode 100644 index 000000000..62f923900 --- /dev/null +++ b/test/integration/skill_controller_search_test.exs @@ -0,0 +1,59 @@ +defmodule SkillControllerSearchIntegrationTest do + use ExUnit.Case, async: true + alias CodeCorps.ElasticSearchHelper + + @test_url Application.get_env(:code_corps, :elasticsearch_url) + @test_index "skills" + @type_value "title" + + @elixir %{"id" => 1, "description" => "Elixir is an awesome functional language", "title" => "Elixir", "original_row" => 1} + @ruby %{"id" => 2, "description" => "Ruby is an awesome OO language", "title" => "Ruby", "original_row" => 2} + @rails %{"id" => 3, "description" => "Rails is a modern framework", "title" => "Rails", "original_row" => 3} + @css %{"id" => 4, "description" => "CSS is pretty cool too", "title" => "CSS", "original_row" => 4} + @phoenix %{"id" => 5, "description" => "Phoenix is a super framework", "title" => "Phoenix", "original_row" => 5} + + setup do + ElasticSearchHelper.delete(@test_url, @test_index) + ElasticSearchHelper.create_index(@test_url, @test_index, @type_value) + init() + :ok + end + + test "search partial word" do + results = ElasticSearchHelper.search(@test_url, @test_index, "title", "ru") + assert results == [@ruby] + end + + test "fuzzy search partial word" do + results = ElasticSearchHelper.search(@test_url, @test_index, "title", "rj") + # Two lists can be concatenated or subtracted using the ++/2 and --/2 + # see: http://elixir-lang.org/getting-started/basic-types.html#linked-lists + # This allows us to confirm the values we want regardless of the order the values are returned in. + assert results -- ["Ruby", "Rails"] == [] + end + + test "search whole word" do + results = ElasticSearchHelper.search(@test_url, @test_index, "title", "css") + assert results == [@css] + end + + test "fuzzy search whole word" do + results = ElasticSearchHelper.search(@test_url, @test_index, "title", "csw") + assert results == [@css] + end + + test "search no matches" do + results = ElasticSearchHelper.search(@test_url, @test_index, "title", "foo") + assert results == [] + end + + test "match all entries" do + results = ElasticSearchHelper.match_all(@test_url, @test_index, "title") + assert results -- [@elixir, @ruby, @rails, @css] == [] + end + + def init do + ElasticSearchHelper.add_documents(@test_url, @test_index, @type_value, + [@elixir, @css, @ruby], [refresh: true]) + end +end