Separate cache clock

.github/workflows/elixir.yml

@@ -8,13 +8,9 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     name: Build and Test on OTP ${{matrix.otp}} / Elixir ${{matrix.elixir}}
-    strategy:
-      matrix:
-        otp: ['24.1.2'] #, '25.1.2']
-        elixir: ['1.12.3', '1.13.4']
     steps:
 
     - uses: actions/checkout@v2
@@ -25,8 +21,8 @@ jobs:
 
     - uses: erlef/setup-beam@v1
       with:
-        otp-version: ${{matrix.otp}}
-        elixir-version: ${{matrix.elixir}}
+        version-type: strict
+        version-file: .tool-versions
 
     - name: Install dependencies
       run: mix deps.get

.tool-versions

@@ -1,5 +1,4 @@
 erlang 25.1.2
-elixir 1.13.4-otp-24
+elixir 1.14.3-otp-25
 direnv 2.28.0
 nodejs 18.12.0
-rebar 3.15.0

apps/xest/README.md

@@ -16,3 +16,25 @@ Timewise, the Adapter will poll to retrieve information and attempt to maintain
 A cache is in place to avoid spamming servers. Later websockets could be used as well.
 The Agent is there to keep a memory of the past events and expose current state.
 The common Xest structures can extrapolate from past events to estimate current situation, where reasonably safe...
+
+
+## Next Step
+
+After some time, xest has grown into both conflicting conceptual apps/libs:
+- one that is used as the interface to concrete connector implementation (kraken and binance) with behaviours, protocols, etc.
+- one that is used as the "logical backend" for the web UI (and maybe more later)
+
+This is problematic as:
+- the web (and xest app) wants to make sure all connector implementations work as expected...
+- connector implementations depend on xest to know how to communicate with the rest of the software.
+
+There are conflicts in dependencies (although in an umbrella app, these might not be obvious at first),
+but when changing some xest modules used everywhere, sometimes, only part of them are rebuilt by mix.
+
+There are also likely other hidden conflicts brought by this current state of things.
+At a high level, this resemble a Strategy Pattern, and we should therefore separate those two concerns in two apps/libs.
+
+Proposed solution:
+- a xest_connector lib, containing all modules necessary to build a working interface to a crypto exchange API.
+- a xest app as it is now, only to be the client of these connectors (via an adapter, token with protocols, or some other convenient elixir design),
+  and the unique contact point from the web (and other UI apps).

apps/xest/lib/xest/account.ex

@@ -6,12 +6,21 @@ defmodule Xest.Account do
   defmodule Behaviour do
     @moduledoc "Behaviour to allow mocking a xest account for tests"
     @callback balance(atom()) :: %Xest.Account.Balance{}
+    @callback balance(atom(), Keyword.t()) :: %Xest.Account.Balance{}
     @callback transactions(atom()) :: %Xest.Account.TradesHistory{}
     @callback transactions(atom(), String.t()) :: %Xest.Account.TradesHistory{}
   end
 
-  def balance(connector) do
+  def balance(connector, opts \\ [filter: fn x -> Float.round(x, 8) != 0 end]) do
     Xest.Account.Adapter.retrieve(connector, :balance)
+    |> Map.update!(
+      :balances,
+      &Enum.filter(&1, fn b ->
+        {free, ""} = Float.parse(b.free)
+        {locked, ""} = Float.parse(b.locked)
+        opts[:filter].(free) or opts[:filter].(locked)
+      end)
+    )
   end
 
   ## OLD version: all trades (kraken)

apps/xest/lib/xest/account/adapter.ex

@@ -39,6 +39,22 @@ defmodule Xest.Account.Adapter do
     connector_response
   end
 
+  def retrieve(:kraken, :trades, symbol) do
+    connector_response =
+      kraken().trades(
+        # looking for process via its name
+        Process.whereis(kraken()),
+        ""
+      )
+      # TMP to get only one symbol (TODO: better in connector)
+      |> Map.update!(:history, &Enum.filter(&1, fn t -> t.symbol == symbol end))
+
+    connector_response
+  end
+
+  # TODO: retrieve trades for *all* symbol
+  # ...  maybe page by page ??
+
   def retrieve(:binance, :trades, symbol) do
     connector_response =
       binance().trades(

apps/xest/lib/xest/api_server.ex

@@ -41,14 +41,14 @@ defmodule Xest.APIServer do
       def handle_call(request, from, state) do
         {tmap, actual_state} = state
         # Note: actual state should !never! impact request/response
-        case Xest.TransientMap.fetch(tmap, request) do
+        case XestCache.TransientMap.fetch(tmap, request) do
           {:ok, hit} ->
             {:reply, hit, state}
 
           :error ->
             case mockable_impl().handle_cachemiss(request, from, actual_state) do
               {:reply, reply, new_state} ->
-                tmap = Xest.TransientMap.put(tmap, request, reply)
+                tmap = XestCache.TransientMap.put(tmap, request, reply)
                 {:reply, reply, {tmap, new_state}}
 
               {:noreply, new_state} ->
@@ -71,7 +71,7 @@ defmodule Xest.APIServer do
     # Here we add a trasient map to use as cache
     # and prevent call to be handled in client code when possible
     lifetime = Keyword.get(options, :lifetime, nil)
-    tmap = Xest.TransientMap.new(lifetime)
+    tmap = XestCache.TransientMap.new(lifetime)
     # leveraging GenServer behaviour
     GenServer.start_link(module, {tmap, init_arg}, options)
   end

apps/xest/lib/xest/clock.ex

@@ -1,23 +1,41 @@
 defmodule Xest.Clock do
+  # NOT an alias.
+  require XestClock.DateTime
+  # In this module the DateTime.t() type is the core Elixir one.
+
   defmodule Behaviour do
     @moduledoc "Behaviour to allow mocking a xest clock for tests"
     @callback utc_now(atom()) :: DateTime.t()
+    @callback utc_now() :: DateTime.t()
+  end
+
+  @behaviour Behaviour
+
+  @impl true
+  def utc_now() do
+    datetime().utc_now()
   end
 
+  @impl true
   def utc_now(:binance) do
     binance().utc_now(
       # finding the process (or nil if mocked)
       Process.whereis(binance())
     )
   end
 
+  @impl true
   def utc_now(:kraken) do
     kraken().utc_now(
       # finding the process (or nil if mocked)
       Process.whereis(kraken())
     )
   end
 
+  defp datetime() do
+    Application.get_env(:xest_clock, :datetime_module, XestClock.DateTime)
+  end
+
   defp kraken() do
     Application.get_env(:xest, :kraken_clock)
   end

apps/xest/lib/xest/clock/proxy.ex

@@ -40,7 +40,7 @@ defmodule Xest.Clock.Proxy do
 
   def expired?(proxy) do
     # only request local utc_now when actually needed
-    expired?(proxy, Xest.DateTime.utc_now())
+    expired?(proxy, XestClock.DateTime.utc_now())
   end
 
   @spec expired?(t(), DateTime.t()) :: boolean()
@@ -59,7 +59,7 @@ defmodule Xest.Clock.Proxy do
   end
 
   @spec retrieve(t(), DateTime.t()) :: t()
-  def retrieve(proxy, requested_on \\ Xest.DateTime.utc_now())
+  def retrieve(proxy, requested_on \\ XestClock.DateTime.utc_now())
 
   def retrieve(%__MODULE__{remote_clock: nil} = proxy, requested_on) do
     proxy

apps/xest/lib/xest/exchange/adapter.ex

@@ -38,5 +38,6 @@ defmodule Xest.Exchange.Adapter do
 
   def retrieve(:kraken, :symbols) do
     kraken().symbols(Process.whereis(kraken()))
+    |> Map.keys()
   end
 end

apps/xest/lib/xest/exchange/servertime.ex

@@ -3,7 +3,7 @@ defmodule Xest.Exchange.ServerTime do
   import Algae
 
   defdata do
-    servertime :: Xest.DateTime.t()
+    servertime :: XestClock.DateTime.t()
   end
 end
 

apps/xest/lib/xest/portfolio.ex

@@ -0,0 +1,36 @@
+defmodule Xest.Portfolio do
+  @moduledoc false
+
+  # TODO  a way to organise:
+  # - what is displayed and how, no matter the view
+  # - what can be done with what is displayed
+  # - the information that is relevant long term
+
+  # IDEA :
+  # a HODL list : currencies already owned, to hold onto (prevent inadvertent selling)
+  # a SELL list : currencies sellable (ready to trade as quote -default- or base)
+  # a BUY list : currencies that are buyable (ready to trade as quote -default- or base)
+  # Note both sell and buy currencies are "tradable".
+  # the list note the intent, waiting for an opportunity on the market...
+
+  # This creates a NEW list of the list we dont currently HOLD and want to sell
+
+  # Since it is an intent: the user must decide which currency goes into which list.
+  # The list must be remembered between runs... stored into user account/portfolio/bot configuration ?
+
+  # IDEA : 2 levels of user interactions/future bots
+  # - one level waiting for the *best* time to sell / buy (currently the user, but to be replaced when possible)
+  # - one level deciding what to sell / what to buy and on which exchange (currently the user)
+
+  # HELD + KEEP -> HODL list
+  # HELD + TRADE -> SELL list by default (BUY possible) + possible amount adjustment
+  # NEW + TRADE -> BUY list by default (SELL possible) + possible amount adjustment
+  # Stop condition for automatic exiting BUY and SELL list
+  #  -> timeout
+  #  -> market conditions...
+
+  # NOTE :
+  # - basic functionality first (check markets + spot trade).
+  # - wide exchange compatibilty second (need more users !).
+  # - detailed functionality (sub accounts, etc.) third.
+end

apps/xest/mix.exs

@@ -54,6 +54,9 @@ defmodule Xest.MixProject do
   # Type `mix help deps` for examples and options.
   defp deps do
     [
+      {:xest_clock, in_umbrella: true},
+      {:xest_cache, in_umbrella: true},
+
       # Tooling
       {:mix_test_watch, "~> 1.0", only: :dev, runtime: false},
       {:credo, "~> 1.5", only: [:dev, :test], runtime: false},

apps/xest/test/test_helper.exs

@@ -2,13 +2,20 @@ ExUnit.start()
 
 # Datetime configuration for an optional mock,
 # when setting local clock is required.
-Hammox.defmock(Xest.DateTime.Mock, for: Xest.DateTime.Behaviour)
-Hammox.stub_with(Xest.DateTime.Mock, Xest.DateTime.Stub)
+Hammox.defmock(XestClock.DateTime.Mock, for: XestClock.DateTime.Behaviour)
+# Hammox.stub_with(XestClock.DateTime.Mock, XestClock.DateTime.Stub)
 
 Hammox.defmock(TestServerMock, for: Xest.APIServer)
 
 Application.put_env(:xest, :api_test_server, TestServerMock)
 
+# NOTE :here we depend on connector implementations,
+# But only dynamically, and only for tests !!
+# Therefore, the connectors should be built (for test mix env) previously to running the tests here
+# TODO: investigate Protocols for a better design here...
+#   because current design breaks when renaming modules in different apps
+#   and running only some app's tests => dependency not detected -> not recompiled
+
 # Mocking connector using provided behavior
 Hammox.defmock(XestKraken.Exchange.Mock, for: XestKraken.Exchange.Behaviour)
 

apps/xest/test/xest/api_server_test.exs

@@ -1,8 +1,8 @@
 defmodule Xest.APIServer.Test do
-  use ExUnit.Case, async: true
+  use ExUnit.Case, async: false
   use FlowAssertions
 
-  alias Xest.DateTime
+  alias XestClock.DateTime
   alias Xest.APIServer
 
   # cf https://medium.com/genesisblock/elixir-concurrent-testing-architecture-13c5e37374dc
@@ -52,8 +52,15 @@ defmodule Xest.APIServer.Test do
 
   describe "Given time to cache" do
     setup do
-      # setting up datetime mock
-      Application.put_env(:xest, :datetime_module, Xest.DateTime.Mock)
+      # saving XestClock.DateTime implementation
+      previous_datetime = Application.get_env(:xest_clock, :datetime_module)
+      # Setup XestClock.DateTime Mock for these tests
+      Application.put_env(:xest_clock, :datetime_module, XestClock.DateTime.Mock)
+
+      on_exit(fn ->
+        # restoring config
+        Application.put_env(:xest_clock, :datetime_module, previous_datetime)
+      end)
 
       # starts server test process
       server_pid =
@@ -72,7 +79,7 @@ defmodule Xest.APIServer.Test do
         {:reply, state, state}
       end)
 
-      # because we need ot play with time here...
+      # because we need to play with time here...
       DateTime.Mock
       |> allow(self(), server_pid)