First major commit of elixir daemon, API version 20220126

Parity functionality with previous go-based daemon, sans all the
extra formatting stuff (incl JSON & CSV), as they were unnecessary cruft.

.gitignore

@@ -1,5 +1,6 @@
-.keyfile
+.keyfile*
 /data/
+*.sqlite3
 
 ## Mix-defined:
 

README.md

@@ -1,6 +1,5 @@
 # HLTE
 
+Work-in-progress rewrite of the current `go` daemon in elixir, while also removing much of the cruft from an older, now-dead usage model.
 
-```
-dd if=/dev/urandom of=.keyfile bs=1024 count=4
-```
+Use [`tools/keygen`](https://github.com/hlte-net/tools/blob/main/keygen) to create an appropriate key file and get the hexadecimal representation needed for the extension's settings.

config/config.exs

@@ -0,0 +1,16 @@
+import Config
+
+config :hlte,
+  api_version: "20220126",
+  header: "x-hlte",
+  port: 31337,
+  db_path: "./data.sqlite3",
+  key_path: "./.keyfile"
+
+import_config("#{config_env()}.config.exs")
+
+config :logger,
+  utc_log: true,
+  truncate: :infinity
+
+config :logger, :console, format: "$time [$level] $levelpad$message\n"

config/dev.config.exs

@@ -0,0 +1,8 @@
+import Config
+
+config :hlte,
+  db_path: "./dev-data.sqlite3"
+
+config :logger, :console,
+  format: "$time [$level] $levelpad$message ($metadata)\n",
+  metadata: [:file, :line]

config/prod.config.exs

@@ -0,0 +1,9 @@
+import Config
+
+config :logger,
+  compile_time_purge_matching: [
+    [level_lower_than: :info]
+  ]
+
+config :hlte,
+  port: 56555

lib/hlte/application.ex

@@ -8,13 +8,13 @@ defmodule HLTE.Application do
 
   @impl true
   def start(_type, args) do
-    IO.puts("START [#{inspect(args)}] <<#{args[:port]}>>")
-
     case load_key(args[:key_path]) do
       {:ok, key} ->
-        IO.puts("GOT KEY! #{inspect(key)}")
-        # persist key!
-        start_link(args[:port])
+        :ok = :persistent_term.put(:key, key)
+        keyHash = :crypto.hash(:sha256, key) |> :binary.encode_hex() |> :string.lowercase()
+        Logger.notice("Loaded #{byte_size(key)}-byte key with SHA256 checksum of #{keyHash}")
+
+        start_link(args)
 
       {:error, reason, expand_path} ->
         Logger.emergency(~s/Failed to load key file at #{expand_path}: "#{reason}"/)
@@ -32,8 +32,8 @@ defmodule HLTE.Application do
 
       {:ok, stat} ->
         {:error,
-         "file permissions (#{inspect(stat.mode &&& 0xFFF, base: :octal)}) are too open: it cannot not be readable or writable by anyone but the owner.",
-         expand_path}
+         "file mode (#{inspect(stat.mode &&& 0xFFF, base: :octal)}) is too permissive: " <>
+           "it cannot not be readable or writable by anyone but the owner.", expand_path}
 
       {:error, reason} ->
         {:error, reason, expand_path}
@@ -45,7 +45,9 @@ defmodule HLTE.Application do
 
   def start_link(args) do
     children = [
-      {HLTE.HTTP, [args]}
+      {Task.Supervisor, name: HLTE.AsyncSupervisor},
+      {HLTE.HTTP, [args[:port], args[:header]]},
+      {HLTE.DB, [args[:db_path]]}
     ]
 
     opts = [strategy: :one_for_one, name: HLTE.Supervisor]

lib/hlte/db.ex

@@ -0,0 +1,142 @@
+alias Exqlite.Basic
+
+defmodule HLTE.DB do
+  require Logger
+
+  use Task
+
+  def start_link([dbPath]) do
+    Task.start_link(__MODULE__, :init, [Path.expand(dbPath)])
+  end
+
+  def init(dbPath) do
+    statRes = File.stat(dbPath)
+    {:ok, conn} = get_conn(dbPath)
+
+    case init_db(statRes, conn) do
+      {:created} ->
+        Logger.notice("Created new database at #{dbPath}")
+
+      {:loaded, count} ->
+        Logger.notice("Loaded existing database with #{count} entries from #{dbPath}")
+    end
+
+    :persistent_term.put(:db_path, dbPath)
+    Basic.close(conn)
+  end
+
+  def init_db({:error, _reason}, conn) do
+    {:ok, _, _, _} = Basic.exec(conn, "create table hlte (
+      checksum text not null,
+      timestamp integer not null,
+      primaryURI text not null,
+      secondaryURI text,
+      hilite text,
+      annotation text
+      )")
+
+    {:created}
+  end
+
+  def init_db({:ok, _stat}, conn) do
+    {:ok, [[count]], ["count(*)"]} =
+      Basic.exec(conn, "SELECT count(*) FROM hlte;") |> Basic.rows()
+
+    {:loaded, count}
+  end
+
+  def persist(%{"uri" => uri, "secondaryURI" => suri, "data" => data, "annotation" => ann}, hmac) do
+    rxTime = System.os_time(:nanosecond)
+
+    Task.Supervisor.async_nolink(HLTE.AsyncSupervisor, fn ->
+      persist_async(
+        uri,
+        suri,
+        data,
+        ann,
+        hmac,
+        rxTime
+      )
+    end)
+
+    rxTime
+  end
+
+  def search(query, limit, newestFirst) do
+    t0 = :erlang.monotonic_time(:millisecond)
+
+    searchRes =
+      Task.await(
+        Task.Supervisor.async(HLTE.AsyncSupervisor, fn ->
+          case newestFirst do
+            "false" -> search_async(query, limit, "asc")
+            _ -> search_async(query, limit, "desc")
+          end
+        end)
+      )
+
+    {searchRes, :erlang.monotonic_time(:millisecond) - t0}
+  end
+
+  defp persist_async(
+         uri,
+         suri,
+         data,
+         ann,
+         hmac,
+         rxTime
+       ) do
+    {:ok, conn} = get_conn(:persistent_term.get(:db_path))
+
+    {:ok, _, _, _} =
+      Basic.exec(conn, "insert into hlte values(?, ?, ?, ?, ?, ?)", [
+        hmac,
+        rxTime,
+        uri,
+        suri,
+        data,
+        ann
+      ])
+
+    Basic.close(conn)
+  end
+
+  defp search_async(query, limit, sortDir) do
+    {:ok, conn} = get_conn(:persistent_term.get(:db_path))
+
+    {:ok, rows, rowSpec} =
+      Basic.exec(
+        conn,
+        "select * from hlte
+      where hilite like '%' || ? || '%'
+      or annotation like '%' || ? || '%'
+      or primaryURI like '%' || ? || '%'
+      or secondaryURI like '%' || ? || '%'
+      order by timestamp #{sortDir} limit ?",
+        [
+          query,
+          query,
+          query,
+          query,
+          limit
+        ]
+      )
+      |> Basic.rows()
+
+    Basic.close(conn)
+
+    # transform into a list of maps with key names based on the row names in `rowSpec`
+    Enum.map(rows, fn ele ->
+      Enum.reduce(0..(length(rowSpec) - 1), %{}, fn idx, acc ->
+        Map.put(acc, Enum.at(rowSpec, idx), Enum.at(ele, idx))
+      end)
+    end)
+  end
+
+  defp get_conn(dbPath) do
+    case Basic.open(dbPath) do
+      {:ok, conn} -> {:ok, conn}
+      _ -> {:err, nil}
+    end
+  end
+end

lib/hlte/http/http.ex

@@ -1,13 +1,66 @@
 defmodule HLTE.HTTP do
+  require Logger
+
   use Task
 
-  def start_link(args) do
-    Task.start_link(__MODULE__, :run, [args])
-  end
+  def start_link(args), do: Task.start_link(__MODULE__, :run, [args])
 
-  def run(args) do
-    IO.puts("HTTP run #{inspect(args)}")
+  def run([listen_port, headerName]) when is_number(listen_port) do
+    {:ok, _} =
+      :cowboy.start_clear(
+        :http,
+        [{:port, listen_port}],
+        %{:env => %{:dispatch => build_dispatch(headerName)}}
+      )
+
+    Logger.notice("HTTP listening on port #{listen_port}")
 
     :ok
   end
+
+  def build_dispatch(headerName) do
+    :cowboy_router.compile([
+      # bind to all interfaces, a la "0.0.0.0"
+      {:_,
+       [
+         # POST
+         {"/", HLTE.HTTP.Route.PostHilite, [headerName]},
+
+         # GET
+         {"/version", HLTE.HTTP.Route.Version, []},
+         {"/search", HLTE.HTTP.Route.Search, [headerName]}
+       ]}
+    ])
+  end
+
+  @doc """
+  Called by route modules to provide the requisite CORS headers in
+  an OPTIONS pre-flight response.
+
+  Returns a request with the appropriate headers set.
+  """
+  def cors_preflight_options(method, req, headerName) do
+    r1 = :cowboy_req.set_resp_header("Access-Control-Allow-Methods", "#{method}, OPTIONS", req)
+    r2 = :cowboy_req.set_resp_header("Access-Control-Allow-Origin", "*", r1)
+
+    :cowboy_req.set_resp_header(
+      "Access-Control-Allow-Headers",
+      "Content-Type, content-type, #{headerName}",
+      r2
+    )
+  end
+
+  def cors_preflight_options(method, req) do
+    :cowboy_req.set_resp_header(
+      "Access-Control-Allow-Origin",
+      "*",
+      :cowboy_req.set_resp_header("Access-Control-Allow-Methods", "#{method}, OPTIONS", req)
+    )
+  end
+
+  def calculate_body_hmac(bodyText) do
+    :crypto.mac(:hmac, :sha256, :persistent_term.get(:key), bodyText)
+    |> :binary.encode_hex()
+    |> :string.lowercase()
+  end
 end

lib/hlte/http/routes/post_hilite.ex

@@ -0,0 +1,51 @@
+defmodule HLTE.HTTP.Route.PostHilite do
+  require Logger
+
+  def init(req, [headerName]) do
+    {:cowboy_rest, req, [headerName]}
+  end
+
+  def allowed_methods(req, state) do
+    {["OPTIONS", "POST"], req, state}
+  end
+
+  def content_types_accepted(req, state) do
+    {[
+       {"text/json", :post_json},
+       {"application/json", :post_json}
+     ], req, state}
+  end
+
+  def options(req, [headerName]) do
+    {:ok, HLTE.HTTP.cors_preflight_options("POST", req, headerName), [headerName]}
+  end
+
+  def post_json(req, [headerName]) when is_map_key(req.headers, headerName) do
+    {:ok, bodyText, req2} = :cowboy_req.read_body(req)
+    hmac = req.headers[headerName]
+
+    {persist(bodyText, hmac, HLTE.HTTP.calculate_body_hmac(bodyText)),
+     :cowboy_req.set_resp_header("Access-Control-Allow-Origin", "*", req2), [headerName]}
+  end
+
+  def post_json(req, state) do
+    Logger.error("POST without header! #{inspect(req)}")
+    {false, req, state}
+  end
+
+  def persist(bodyText, bodyHmac, calcHmac) when bodyHmac === calcHmac do
+    dec = Jason.decode!(bodyText)
+    rxTime = HLTE.DB.persist(dec, bodyHmac)
+
+    Logger.info(
+      "Persisted hilite for #{URI.parse(Map.get(dec, "uri")).host} at #{floor(rxTime / 1.0e9)}"
+    )
+
+    true
+  end
+
+  def persist(_bodyText, bodyHmac, calcHmac) do
+    Logger.critical("Persist failed! HMACS: #{bodyHmac} != #{calcHmac}")
+    false
+  end
+end