From 604b8688ef055fd5d979d520a62e872376e51a65 Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Wed, 3 Jul 2024 12:36:51 +0700 Subject: [PATCH] rm httpoison --- mix.exs | 6 +++-- mix.lock | 9 ------- mix_tasks/http.ex | 33 ++++++++++++++++++++++++ mix_tasks/scraper.ex | 4 +-- mix_tasks/update_english_translations.ex | 1 - mix_tasks/update_iso_data.ex | 8 ++---- 6 files changed, 41 insertions(+), 20 deletions(-) create mode 100644 mix_tasks/http.ex diff --git a/mix.exs b/mix.exs index e8170fc..4fc4dce 100644 --- a/mix.exs +++ b/mix.exs @@ -32,10 +32,13 @@ defmodule Location.MixProject do # Run "mix help compile.app" to learn about applications. def application do [ - extra_applications: [:logger] + extra_applications: [:logger | extra_applications(Mix.env())] ] end + defp extra_applications(env) when env in [:dev, :test], do: [:inets, :ssl] + defp extra_applications(_env), do: [] + # Specifies which paths to compile per environment. defp elixirc_paths(env) when env in [:dev, :test], do: ["lib", "mix_tasks"] defp elixirc_paths(_env), do: ["lib"] @@ -46,7 +49,6 @@ defmodule Location.MixProject do {:jason, "~> 1.3"}, {:nimble_csv, "~> 1.1"}, {:floki, "~> 0.36.0", only: [:dev, :test]}, - {:httpoison, "~> 1.8", only: [:dev, :test]}, {:flow, "~> 1.0", only: [:dev, :test]} ] end diff --git a/mix.lock b/mix.lock index 6e8b87e..e12f5d2 100644 --- a/mix.lock +++ b/mix.lock @@ -1,16 +1,7 @@ %{ - "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"}, "floki": {:hex, :floki, "0.36.2", "a7da0193538c93f937714a6704369711998a51a6164a222d710ebd54020aa7a3", [:mix], [], "hexpm", "a8766c0bc92f074e5cb36c4f9961982eda84c5d2b8e979ca67f5c268ec8ed580"}, "flow": {:hex, :flow, "1.2.4", "1dd58918287eb286656008777cb32714b5123d3855956f29aa141ebae456922d", [:mix], [{:gen_stage, "~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}], "hexpm", "874adde96368e71870f3510b91e35bc31652291858c86c0e75359cbdd35eb211"}, "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"}, - "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"}, - "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"}, - "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.4.3", "d3f984eeb96fe53b85d20e0b049f03e57d075b5acda3ac8d465c969a2536c17b", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "9a90e868927f7c777689baa16d86f4d0e086d968db5c05d917ccff6d443e58a3"}, - "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, - "mimerl": {:hex, :mimerl, "1.3.0", "d0cd9fc04b9061f82490f6581e0128379830e78535e017f7780f37fea7545726", [:rebar3], [], "hexpm", "a1e15a50d1887217de95f0b9b0793e32853f7c258a5cd227650889b38839fe9d"}, "nimble_csv": {:hex, :nimble_csv, "1.2.0", "4e26385d260c61eba9d4412c71cea34421f296d5353f914afe3f2e71cce97722", [:mix], [], "hexpm", "d0628117fcc2148178b034044c55359b26966c6eaa8e2ce15777be3bbc91b12a"}, - "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, - "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, } diff --git a/mix_tasks/http.ex b/mix_tasks/http.ex new file mode 100644 index 0000000..f0a6797 --- /dev/null +++ b/mix_tasks/http.ex @@ -0,0 +1,33 @@ +defmodule Location.HTTP do + @moduledoc false + + def get!(url, headers \\ []) do + headers = for {field, val} <- headers, do: {String.to_charlist(field), val} + + http_opts = [ + ssl: [ + verify: :verify_peer, + depth: 4, + customize_hostname_check: [ + match_fun: :public_key.pkix_verify_hostname_match_fun(:https) + ], + cacerts: :public_key.cacerts_get() + ], + timeout: :timer.seconds(15), + connect_timeout: :timer.seconds(15) + ] + + opts = [ + body_format: :binary + ] + + case :httpc.request(:get, {url, headers}, http_opts, opts) do + {:ok, {{_, status, _}, headers, body}} -> + headers = for {field, val} <- headers, do: {List.to_string(field), List.to_string(val)} + {status, headers, body} + + {:error, reason} -> + raise "failed GET #{url} with #{inspect(reason)}" + end + end +end diff --git a/mix_tasks/scraper.ex b/mix_tasks/scraper.ex index 46bd6b7..eff336c 100644 --- a/mix_tasks/scraper.ex +++ b/mix_tasks/scraper.ex @@ -27,8 +27,8 @@ defmodule Location.Scraper do defp scrape_country(country) do url = @subdivision_base_url <> country.alpha_2 - response = HTTPoison.get!(url) - {:ok, document} = Floki.parse_document(response.body) + {200, _headers, body} = Location.HTTP.get!(url) + {:ok, document} = Floki.parse_document(body) rows = Floki.find(document, "table.wikitable.sortable") diff --git a/mix_tasks/update_english_translations.ex b/mix_tasks/update_english_translations.ex index f2908e2..a5b6deb 100644 --- a/mix_tasks/update_english_translations.ex +++ b/mix_tasks/update_english_translations.ex @@ -2,7 +2,6 @@ defmodule Mix.Tasks.UpdateEnglishTranslations do use Mix.Task def run(_) do - HTTPoison.start() Location.Country.load() Location.Scraper.scrape() end diff --git a/mix_tasks/update_iso_data.ex b/mix_tasks/update_iso_data.ex index 7657fb1..a5d7353 100644 --- a/mix_tasks/update_iso_data.ex +++ b/mix_tasks/update_iso_data.ex @@ -7,14 +7,10 @@ defmodule Mix.Tasks.UpdateIsoData do @subdivisions_dest Application.app_dir(:location, "/priv/iso_3166-2.json") def run(_) do - HTTPoison.start() - - %HTTPoison.Response{status_code: 200, body: countries} = HTTPoison.get!(@countries_src) + {200, _headers, countries} = Location.HTTP.get!(@countries_src) File.write!(@countries_dest, countries) - %HTTPoison.Response{status_code: 200, body: new_subdivisions} = - HTTPoison.get!(@subdivisions_src) - + {200, _headers, new_subdivisions} = Location.HTTP.get!(@subdivisions_src) %{"3166-2" => new_subdivisions} = Jason.decode!(new_subdivisions) new_subdivisions =