From f183219fb5550a7dcda4958183cdb5cc00c345a6 Mon Sep 17 00:00:00 2001 From: Ryan Joseph Date: Wed, 26 Jan 2022 22:23:53 -0800 Subject: [PATCH] First major commit of elixir daemon, API version 20220126 Parity functionality with previous go-based daemon, sans all the extra formatting stuff (incl JSON & CSV), as they were unnecessary cruft. --- .gitignore | 3 +- README.md | 5 +- config/config.exs | 16 ++++ config/dev.config.exs | 8 ++ config/prod.config.exs | 9 ++ lib/hlte/application.ex | 18 ++-- lib/hlte/db.ex | 142 ++++++++++++++++++++++++++++ lib/hlte/http/http.ex | 63 +++++++++++- lib/hlte/http/routes/post_hilite.ex | 51 ++++++++++ lib/hlte/http/routes/search.ex | 57 +++++++++++ lib/hlte/http/routes/version.ex | 26 +++++ mix.exs | 27 ++++-- test/hlte_test.exs | 26 ++++- test/test_keyfile | Bin 0 -> 4096 bytes 14 files changed, 422 insertions(+), 29 deletions(-) create mode 100644 config/config.exs create mode 100644 config/dev.config.exs create mode 100644 config/prod.config.exs create mode 100644 lib/hlte/db.ex create mode 100644 lib/hlte/http/routes/post_hilite.ex create mode 100644 lib/hlte/http/routes/search.ex create mode 100644 lib/hlte/http/routes/version.ex create mode 100644 test/test_keyfile diff --git a/.gitignore b/.gitignore index 46d9e1b..18e2634 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ -.keyfile +.keyfile* /data/ +*.sqlite3 ## Mix-defined: diff --git a/README.md b/README.md index 808d313..4a3c3ae 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ # HLTE +Work-in-progress rewrite of the current `go` daemon in elixir, while also removing much of the cruft from an older, now-dead usage model. -``` -dd if=/dev/urandom of=.keyfile bs=1024 count=4 -``` \ No newline at end of file +Use [`tools/keygen`](https://github.com/hlte-net/tools/blob/main/keygen) to create an appropriate key file and get the hexadecimal representation needed for the extension's settings. \ No newline at end of file diff --git a/config/config.exs b/config/config.exs new file mode 100644 index 0000000..4c3ded3 --- /dev/null +++ b/config/config.exs @@ -0,0 +1,16 @@ +import Config + +config :hlte, + api_version: "20220126", + header: "x-hlte", + port: 31337, + db_path: "./data.sqlite3", + key_path: "./.keyfile" + +import_config("#{config_env()}.config.exs") + +config :logger, + utc_log: true, + truncate: :infinity + +config :logger, :console, format: "$time [$level] $levelpad$message\n" diff --git a/config/dev.config.exs b/config/dev.config.exs new file mode 100644 index 0000000..0e17648 --- /dev/null +++ b/config/dev.config.exs @@ -0,0 +1,8 @@ +import Config + +config :hlte, + db_path: "./dev-data.sqlite3" + +config :logger, :console, + format: "$time [$level] $levelpad$message ($metadata)\n", + metadata: [:file, :line] diff --git a/config/prod.config.exs b/config/prod.config.exs new file mode 100644 index 0000000..5523a9c --- /dev/null +++ b/config/prod.config.exs @@ -0,0 +1,9 @@ +import Config + +config :logger, + compile_time_purge_matching: [ + [level_lower_than: :info] + ] + +config :hlte, + port: 56555 diff --git a/lib/hlte/application.ex b/lib/hlte/application.ex index 6acf359..2ac30a5 100644 --- a/lib/hlte/application.ex +++ b/lib/hlte/application.ex @@ -8,13 +8,13 @@ defmodule HLTE.Application do @impl true def start(_type, args) do - IO.puts("START [#{inspect(args)}] <<#{args[:port]}>>") - case load_key(args[:key_path]) do {:ok, key} -> - IO.puts("GOT KEY! #{inspect(key)}") - # persist key! - start_link(args[:port]) + :ok = :persistent_term.put(:key, key) + keyHash = :crypto.hash(:sha256, key) |> :binary.encode_hex() |> :string.lowercase() + Logger.notice("Loaded #{byte_size(key)}-byte key with SHA256 checksum of #{keyHash}") + + start_link(args) {:error, reason, expand_path} -> Logger.emergency(~s/Failed to load key file at #{expand_path}: "#{reason}"/) @@ -32,8 +32,8 @@ defmodule HLTE.Application do {:ok, stat} -> {:error, - "file permissions (#{inspect(stat.mode &&& 0xFFF, base: :octal)}) are too open: it cannot not be readable or writable by anyone but the owner.", - expand_path} + "file mode (#{inspect(stat.mode &&& 0xFFF, base: :octal)}) is too permissive: " <> + "it cannot not be readable or writable by anyone but the owner.", expand_path} {:error, reason} -> {:error, reason, expand_path} @@ -45,7 +45,9 @@ defmodule HLTE.Application do def start_link(args) do children = [ - {HLTE.HTTP, [args]} + {Task.Supervisor, name: HLTE.AsyncSupervisor}, + {HLTE.HTTP, [args[:port], args[:header]]}, + {HLTE.DB, [args[:db_path]]} ] opts = [strategy: :one_for_one, name: HLTE.Supervisor] diff --git a/lib/hlte/db.ex b/lib/hlte/db.ex new file mode 100644 index 0000000..006848d --- /dev/null +++ b/lib/hlte/db.ex @@ -0,0 +1,142 @@ +alias Exqlite.Basic + +defmodule HLTE.DB do + require Logger + + use Task + + def start_link([dbPath]) do + Task.start_link(__MODULE__, :init, [Path.expand(dbPath)]) + end + + def init(dbPath) do + statRes = File.stat(dbPath) + {:ok, conn} = get_conn(dbPath) + + case init_db(statRes, conn) do + {:created} -> + Logger.notice("Created new database at #{dbPath}") + + {:loaded, count} -> + Logger.notice("Loaded existing database with #{count} entries from #{dbPath}") + end + + :persistent_term.put(:db_path, dbPath) + Basic.close(conn) + end + + def init_db({:error, _reason}, conn) do + {:ok, _, _, _} = Basic.exec(conn, "create table hlte ( + checksum text not null, + timestamp integer not null, + primaryURI text not null, + secondaryURI text, + hilite text, + annotation text + )") + + {:created} + end + + def init_db({:ok, _stat}, conn) do + {:ok, [[count]], ["count(*)"]} = + Basic.exec(conn, "SELECT count(*) FROM hlte;") |> Basic.rows() + + {:loaded, count} + end + + def persist(%{"uri" => uri, "secondaryURI" => suri, "data" => data, "annotation" => ann}, hmac) do + rxTime = System.os_time(:nanosecond) + + Task.Supervisor.async_nolink(HLTE.AsyncSupervisor, fn -> + persist_async( + uri, + suri, + data, + ann, + hmac, + rxTime + ) + end) + + rxTime + end + + def search(query, limit, newestFirst) do + t0 = :erlang.monotonic_time(:millisecond) + + searchRes = + Task.await( + Task.Supervisor.async(HLTE.AsyncSupervisor, fn -> + case newestFirst do + "false" -> search_async(query, limit, "asc") + _ -> search_async(query, limit, "desc") + end + end) + ) + + {searchRes, :erlang.monotonic_time(:millisecond) - t0} + end + + defp persist_async( + uri, + suri, + data, + ann, + hmac, + rxTime + ) do + {:ok, conn} = get_conn(:persistent_term.get(:db_path)) + + {:ok, _, _, _} = + Basic.exec(conn, "insert into hlte values(?, ?, ?, ?, ?, ?)", [ + hmac, + rxTime, + uri, + suri, + data, + ann + ]) + + Basic.close(conn) + end + + defp search_async(query, limit, sortDir) do + {:ok, conn} = get_conn(:persistent_term.get(:db_path)) + + {:ok, rows, rowSpec} = + Basic.exec( + conn, + "select * from hlte + where hilite like '%' || ? || '%' + or annotation like '%' || ? || '%' + or primaryURI like '%' || ? || '%' + or secondaryURI like '%' || ? || '%' + order by timestamp #{sortDir} limit ?", + [ + query, + query, + query, + query, + limit + ] + ) + |> Basic.rows() + + Basic.close(conn) + + # transform into a list of maps with key names based on the row names in `rowSpec` + Enum.map(rows, fn ele -> + Enum.reduce(0..(length(rowSpec) - 1), %{}, fn idx, acc -> + Map.put(acc, Enum.at(rowSpec, idx), Enum.at(ele, idx)) + end) + end) + end + + defp get_conn(dbPath) do + case Basic.open(dbPath) do + {:ok, conn} -> {:ok, conn} + _ -> {:err, nil} + end + end +end diff --git a/lib/hlte/http/http.ex b/lib/hlte/http/http.ex index eb90768..36e7264 100644 --- a/lib/hlte/http/http.ex +++ b/lib/hlte/http/http.ex @@ -1,13 +1,66 @@ defmodule HLTE.HTTP do + require Logger + use Task - def start_link(args) do - Task.start_link(__MODULE__, :run, [args]) - end + def start_link(args), do: Task.start_link(__MODULE__, :run, [args]) - def run(args) do - IO.puts("HTTP run #{inspect(args)}") + def run([listen_port, headerName]) when is_number(listen_port) do + {:ok, _} = + :cowboy.start_clear( + :http, + [{:port, listen_port}], + %{:env => %{:dispatch => build_dispatch(headerName)}} + ) + + Logger.notice("HTTP listening on port #{listen_port}") :ok end + + def build_dispatch(headerName) do + :cowboy_router.compile([ + # bind to all interfaces, a la "0.0.0.0" + {:_, + [ + # POST + {"/", HLTE.HTTP.Route.PostHilite, [headerName]}, + + # GET + {"/version", HLTE.HTTP.Route.Version, []}, + {"/search", HLTE.HTTP.Route.Search, [headerName]} + ]} + ]) + end + + @doc """ + Called by route modules to provide the requisite CORS headers in + an OPTIONS pre-flight response. + + Returns a request with the appropriate headers set. + """ + def cors_preflight_options(method, req, headerName) do + r1 = :cowboy_req.set_resp_header("Access-Control-Allow-Methods", "#{method}, OPTIONS", req) + r2 = :cowboy_req.set_resp_header("Access-Control-Allow-Origin", "*", r1) + + :cowboy_req.set_resp_header( + "Access-Control-Allow-Headers", + "Content-Type, content-type, #{headerName}", + r2 + ) + end + + def cors_preflight_options(method, req) do + :cowboy_req.set_resp_header( + "Access-Control-Allow-Origin", + "*", + :cowboy_req.set_resp_header("Access-Control-Allow-Methods", "#{method}, OPTIONS", req) + ) + end + + def calculate_body_hmac(bodyText) do + :crypto.mac(:hmac, :sha256, :persistent_term.get(:key), bodyText) + |> :binary.encode_hex() + |> :string.lowercase() + end end diff --git a/lib/hlte/http/routes/post_hilite.ex b/lib/hlte/http/routes/post_hilite.ex new file mode 100644 index 0000000..3847155 --- /dev/null +++ b/lib/hlte/http/routes/post_hilite.ex @@ -0,0 +1,51 @@ +defmodule HLTE.HTTP.Route.PostHilite do + require Logger + + def init(req, [headerName]) do + {:cowboy_rest, req, [headerName]} + end + + def allowed_methods(req, state) do + {["OPTIONS", "POST"], req, state} + end + + def content_types_accepted(req, state) do + {[ + {"text/json", :post_json}, + {"application/json", :post_json} + ], req, state} + end + + def options(req, [headerName]) do + {:ok, HLTE.HTTP.cors_preflight_options("POST", req, headerName), [headerName]} + end + + def post_json(req, [headerName]) when is_map_key(req.headers, headerName) do + {:ok, bodyText, req2} = :cowboy_req.read_body(req) + hmac = req.headers[headerName] + + {persist(bodyText, hmac, HLTE.HTTP.calculate_body_hmac(bodyText)), + :cowboy_req.set_resp_header("Access-Control-Allow-Origin", "*", req2), [headerName]} + end + + def post_json(req, state) do + Logger.error("POST without header! #{inspect(req)}") + {false, req, state} + end + + def persist(bodyText, bodyHmac, calcHmac) when bodyHmac === calcHmac do + dec = Jason.decode!(bodyText) + rxTime = HLTE.DB.persist(dec, bodyHmac) + + Logger.info( + "Persisted hilite for #{URI.parse(Map.get(dec, "uri")).host} at #{floor(rxTime / 1.0e9)}" + ) + + true + end + + def persist(_bodyText, bodyHmac, calcHmac) do + Logger.critical("Persist failed! HMACS: #{bodyHmac} != #{calcHmac}") + false + end +end diff --git a/lib/hlte/http/routes/search.ex b/lib/hlte/http/routes/search.ex new file mode 100644 index 0000000..400eadc --- /dev/null +++ b/lib/hlte/http/routes/search.ex @@ -0,0 +1,57 @@ +defmodule HLTE.HTTP.Route.Search do + require Logger + + def init(req, [headerName]) do + {:cowboy_rest, req, [headerName]} + end + + def allowed_methods(req, state) do + {["OPTIONS", "GET"], req, state} + end + + def options(req, state) do + {:ok, HLTE.HTTP.cors_preflight_options("GET", req), state} + end + + def content_types_provided(req, state) do + {[ + {"text/json", :get_json}, + {"application/json", :get_json} + ], req, state} + end + + def get_json(req, [headerName]) when is_map_key(req.headers, headerName) do + %{d: newestFirst, l: limit, q: query} = :cowboy_req.match_qs([:d, :l, :q], req) + + case search( + req.headers[headerName], + HLTE.HTTP.calculate_body_hmac(req.qs), + query, + limit, + newestFirst + ) do + {:ok, searchRes, elTime} -> + Logger.info("Executed search '#{query}' (limit=#{limit}) in #{elTime}ms") + {searchRes, req, [headerName]} + + _ -> + Logger.warn("Failed request was:\n#{inspect(req)}") + {"[]", req, [headerName]} + end + end + + def get_json(req, [_headerName]) do + Logger.error("Search request failed! #{inspect(req)}") + false + end + + defp search(bodyHmac, calcHmac, query, limit, newestFirst) when bodyHmac === calcHmac do + {searchRes, elTime} = HLTE.DB.search(query, limit, newestFirst) + {:ok, Jason.encode!(searchRes), elTime} + end + + defp search(bodyHmac, calcHmac, _query, _limit, _newestFirst) do + Logger.critical("Search request (match) failed! HMACS: #{bodyHmac} != #{calcHmac}") + false + end +end diff --git a/lib/hlte/http/routes/version.ex b/lib/hlte/http/routes/version.ex new file mode 100644 index 0000000..9ef0c61 --- /dev/null +++ b/lib/hlte/http/routes/version.ex @@ -0,0 +1,26 @@ +defmodule HLTE.HTTP.Route.Version do + require Logger + + def init(req, opts) do + {:cowboy_rest, req, opts} + end + + def allowed_methods(req, state) do + {["OPTIONS", "GET"], req, state} + end + + def options(req, state) do + {:ok, HLTE.HTTP.cors_preflight_options("GET", req), state} + end + + def content_types_provided(req, state) do + {[ + {"text/plain", :get_version} + ], req, state} + end + + def get_version(req, state) do + {Application.fetch_env!(:hlte, :api_version), + :cowboy_req.set_resp_header("Access-Control-Allow-Origin", "*", req), state} + end +end diff --git a/mix.exs b/mix.exs index 0b5bc67..8986ff7 100644 --- a/mix.exs +++ b/mix.exs @@ -1,11 +1,12 @@ defmodule HLTE.MixProject do + require Logger + use Mix.Project def project do [ - app: :hlte_daemon, + app: :hlte, version: "0.1.0", - api_version: "20220126", elixir: "~> 1.13", start_permanent: Mix.env() == :prod, deps: deps() @@ -13,15 +14,21 @@ defmodule HLTE.MixProject do end def application do + envArgs = [ + header: fe(:header), + port: fe(:port), + db_path: fe(:db_path), + key_path: fe(:key_path) + ] + + Logger.notice("App started with config #{inspect(envArgs)}") + [ extra_applications: [:logger], - mod: - {HLTE.Application, - [ - port: 31337, - local_data_path: "./data", - key_path: "./.keyfile" - ]} + mod: {HLTE.Application, envArgs}, + env: [ + args: envArgs + ] ] end @@ -35,4 +42,6 @@ defmodule HLTE.MixProject do {:exqlite, "~> 0.8.6"} ] end + + defp fe(k), do: Application.fetch_env!(:hlte, k) end diff --git a/test/hlte_test.exs b/test/hlte_test.exs index 7a4bc59..cf3dc6f 100644 --- a/test/hlte_test.exs +++ b/test/hlte_test.exs @@ -1,8 +1,28 @@ defmodule HLTETest do use ExUnit.Case - doctest HLTE + doctest HLTE.Application - test "greets the world" do - assert HLTE.hello() == :world + test "environment specific args are available in Application environment" do + assert Application.fetch_env!(:hlte_daemon, :args)[:key_path] == "./test/test_keyfile" + end + + test "opening permissions on test keyfile causes load failure" do + expand_path = Path.expand(Application.fetch_env!(:hlte_daemon, :args)[:key_path]) + {:ok, %{mode: original_mode}} = File.stat(expand_path) + File.chmod!(expand_path, 0o644) + {:error, reason, ^expand_path} = HLTE.Application.load_key(expand_path) + File.chmod!(expand_path, original_mode) + assert true + end + + test "can load test keyfile" do + {:ok, key} = HLTE.Application.load_key(Application.fetch_env!(:hlte_daemon, :args)[:key_path]) + assert byte_size(key) == 4096 + end + + test "test keyfile contents are as expected" do + {:ok, key} = HLTE.Application.load_key(Application.fetch_env!(:hlte_daemon, :args)[:key_path]) + hash = :crypto.hash(:sha256, key) |> :binary.encode_hex() + assert hash == "01854DDEDC285DE6B39CC1DA6B8BDA00EF4FBDDC2BD1D46F16537AE34E572575" end end diff --git a/test/test_keyfile b/test/test_keyfile new file mode 100644 index 0000000000000000000000000000000000000000..57fb8f278d9696349459560e38d718c79cc59124 GIT binary patch literal 4096 zcmV+b5dZI3C3^{Kf}5>D%D6uPVn}s@iubQ6Y}rXf?OK0#75A-HTYXiH*Nestx}1TY zQ-S~P`^AY%knMpUs5-l*H+sAGPQ9fUbT`1@lV2dw zA2sQ|$f;$+`Pta7kJI!ua#s-|@LfsQHYEP(FyvhZHT_12+Ydd0q33mIkkI{P$kN`Q#iE+}M&56zeLw0%rtYC5{((+eOw)yEV` zEvY@KOjOeLFqsW1RT8?$WB@5HOPy-tUMth~yLhyM&a>_|hCfjkew_U&>Ul>w)Sb+_ zNm8UvF$K@R%#?khiNHzS+nF7B&GtaZ+$Mn;Fo3KJ4XIrI?77w{$?2WkVk0#p_VNhohQc%|Cs5f+@-f8kP3}iX%BbJGbrtS0BvQPxTpez=*+*EfvK>0*gQYiYnM!pl54!gWR_D#gj%tFheG7ZR^OG1VD_1e zkObNUyQv8(ck&rQH;3_yPMW-TS4-SvPRvT~YGt_;-k&g=^kJF*`PIPFdlXZC#z5HzlM!=3DBl_(HlDOtv~jlY zgDBziPq?oT=ar^+8lSMLnkZ?p3lGRP^j|2Wfm&^-2tVu21K1QNg1@}J15<{dJ|~a% z#%d*rb6@qd6gDU|9HT2n#UeJM*EZ$gPl!sUs;X@g+oHj6y8b41b2CD`Q#b z_!jLTPYJ^6eJz*>AA&}v{AxTjB&s<-0t$O_o%8Sa(GDr4`|g<&qofDRSq^==D7TEYK&2+MaTK{2VV~Hk zF!$osw>_JkX$o!v^J5OIp_h% z2gyHG`FrYezV>t&Et^mDeWmGWtg4%f+G3tz1(8)n#f`4#z7L~|U(Ztr*GPF6skNEe zdomC=Ujm+g<;s=H85F&b%(6*+0q=s@=O|Mm7{dh`n!cb6wg802aSY!AfDF>;bxNR4 zuPw+sM`~Q&-UUzmrT++cVA`d?SCAfP-9hJn0l?4(Rn2J_&mUaZt|;XVo_584pc|P; z3-&Tkwva+Tl1!2-w!P?B4Q44+T16vECndSyF zTfNyjA}SjSN`&%}CK59q6Xf1xfrMj+#3PA>ZdBTI8}TtX@6}raV8|?qkpWAZBuOgZ z+C`D32`@{4+zlJ#FDd4mk_9@BhM1WX^{4NoKt&F)%6#HsNX`3%|nw$7<;YHQEhy+1t9> z`q8*PCuWh4Z*N0m(H6d<4fZ8pBH8AN0u2iC;idkYsVwfFtDQO^hjIha;-$o!SpHMw zWGN11ykTBE;^b_blAmwG)!xY3Y$YXL1Rvu(Wz1pRVp!h}R?P~rzlMlX`kv$7&3Su3 za`~8edoDUDgM2N;J6ZkfKA?S_8i2zI2z98&HPg9M&nhUWX(zhFs}UE>A_~|vRF8{n z5KxR7+$)fD7!$n&ivjQf78SJ1lmNO-QXab9@O}9%i*8k>;=QrRok(e|Eilxy}?WFbyeA?W>qYSfQ#E)*y~? z;q^8Wh%)f9j^(=iCzds_n|BV|f{k3*ULbbK@RG|gzkod&D`Y`8tIR1;sp93c^ ze)-eLo>xp{IiP97@=)%HUh~{)sDlbtEFGr8&p9qg60#@gw$lPBg$@5h3doN?*|0e| ze};RPO0PGmwuvxP{u(>;W^`Sp6aSQT(w@gj(CkiQQ`?RVXte5T2WS$7RpTo z(32I1Dfg74^<7_MyAP91L%P;P)&Z5u6KE@CihRuE7B;n0S-?aa_1(PYB|W;OF^D(E zg_wIs3TF;(qSbjpR_JBoQt4RK98heN!h+8D7=o(q*?!&|UcF4@D{ zx~~JJg%RcIwtK0>2$RrVwiLTd)G*(ucH{1bwdj-_n+st)?v<8X$@MFHIDH zL&e>P;)MmMasR#A*O445=}F$zGMK|9GfKN+ZsqbAdsNd~ZEbnguSx>oiZqtWEXSs# zZJxxB)ADYR2|sybRg;%9)vaS(qBS2SWIY}K*^d7Dd1$KQN^2b6EuLhU^&~e!CvG`L z$cj11z)d9E{x}zA?|_ zFN}v4o^b+C*IyKw$E%Vifsa%%F3Qm=nlCk>|5RRieO@&tk16@^R$yuDKcO&Nz3gvk zYuBmwXNxHnd;e5)zU~hZ#VN7Woj=xbGqqb{xFRaj00*Z*Zi^_AAETECecY4P4Z`u> z3TN<{Q>53|-L-}-@~qxFZb<~M&1747_)WTFQ$vIG;WwK zbjIH&+Op`%8_TAH|NV*L%Nzqstj)_jt~CB6TsPdy=F22svFN|wDO9LY)Z)aZaA=XS z&nxKkpq^}P>^$$Z(f6&{CMl(czr4Ii0RvKqTYO? zN3+0g1!)H(-Ieh2x!K16`;j^r@!jh;N1_zvdLhp=vIt7~<}74BsxMtG9w%({J{7_AsG zDy&(e;!1OdzN1i$5s0h5Wh8S0%6j(`Zn41L?yfVAe;TUzjxbs-*~PzWM_}cUUHET& zO?`simx5}saqY)yXo}Vu;-asafYyT*lv#UCEhlyRTNYll2>d~f@X&UH1n)X^5qpg= zAgkc?nxIX?RKAT6&^c3W%)3C@nY0fpM$S(Slr}shdWJyj_4qgD%LyBDi5w?N5-3R< z#H53qMC&02-37{U*BWr%m~K&mQuB2`RBdyJ^Z$l>0;1H@-xSwwGMtzA9>+qA<&39= zDMqBq;ao)oB_+HmuP#5jEr4d2nkMzQ5UsNLor?l&{c~eYF(#wU226SMOo?_bMc0j-}L;Pz1 literal 0 HcmV?d00001