Merge pull request #17 from ghivert/refactor/improvements

Upgrade the whole application

Author: ghivert

.github/workflows/test.yml

@@ -16,8 +16,8 @@ jobs:
       - name: Setup BEAM
         uses: erlef/setup-beam@v1
         with:
-          otp-version: '26.0.2'
-          gleam-version: '1.4.1'
+          otp-version: '27.0.0'
+          gleam-version: '1.6.2'
           rebar3-version: '3'
           # elixir-version: "1.15.4"
       - name: Download gleam dependencies
@@ -50,8 +50,8 @@ jobs:
       - name: Setup BEAM
         uses: erlef/setup-beam@v1
         with:
-          otp-version: '26.0.2'
-          gleam-version: '1.4.1'
+          otp-version: '27.0.0'
+          gleam-version: '1.6.2'
           rebar3-version: '3'
       - name: Download gleam dependencies
         run: gleam deps download

.mise.toml

@@ -1,6 +1,6 @@
 # Runtimes used by the stack.
 [tools]
-gleam = "1.4.1"
+gleam = "1.6.2"
 erlang = "27"
 node = "22"
 deno = "latest"

apps/backend/gleam.toml

@@ -2,30 +2,33 @@ name = "backend"
 version = "1.0.0"
 
 [dependencies]
-aws4_request = ">= 0.1.1 and < 1.0.0"
+aws4_request = ">= 1.2.0 and < 2.0.0"
 birl = "~> 1.6"
 chomp = ">= 0.1.0 and < 1.0.0"
-cors_builder = ">= 2.0.0 and < 3.0.0"
+cors_builder = ">= 2.0.2 and < 3.0.0"
 decipher = ">= 1.2.0 and < 2.0.0"
+envoy = ">= 1.0.2 and < 2.0.0"
 gleam_erlang = "~> 0.25"
 gleam_hexpm = "~> 1.0"
 gleam_http = "~> 3.6"
 gleam_httpc = ">= 2.2.0 and < 3.0.0"
-gleam_json = "~> 1.0"
+gleam_json = ">= 2.1.0 and < 3.0.0"
 gleam_otp = "~> 0.10"
 gleam_package_interface = ">= 1.0.0 and < 2.0.0"
-gleam_pgo = {path = "../../packages/pgo"}
-gleam_stdlib = "~> 0.34 or ~> 1.0"
+gleam_regexp = ">= 1.0.0 and < 2.0.0"
+gleam_stdlib = ">= 0.44.0 and < 1.0.0"
+gleam_yielder = ">= 1.1.0 and < 2.0.0"
 glexer = ">= 1.0.1 and < 2.0.0"
-mist = ">= 1.2.0 and < 2.0.0"
-pgo = "~> 0.14"
+interfaces = {path = "../../packages/interfaces"}
+mist = ">= 3.0.0 and < 4.0.0"
+pog = ">= 1.0.1 and < 2.0.0"
 prng = ">= 3.0.3 and < 4.0.0"
 radiate = ">= 0.4.0 and < 1.0.0"
 ranger = ">= 1.2.0 and < 2.0.0"
 simplifile = ">= 2.1.0 and < 3.0.0"
-tom = { path ="../../packages/tom" }
+tom = ">= 1.1.0 and < 2.0.0"
 verl = ">= 1.1.1 and < 2.0.0"
-wisp = ">= 1.1.0 and < 2.0.0"
+wisp = ">= 1.3.0 and < 2.0.0"
 
 [dev-dependencies]
 gleeunit = "~> 1.0"

apps/backend/manifest.toml

@@ -9,6 +9,6 @@
     "backend:db:reset": "yarn backend:db:drop && yarn backend:db:init"
   },
   "dependencies": {
-    "@chouqueth/gleam": "^1.3.2"
+    "@chouqueth/gleam": "^1.6.2"
   }
 }

apps/backend/package.json

@@ -8,7 +8,7 @@ import gleam/httpc
 import gleam/json
 import gleam/list
 import gleam/option.{type Option, Some}
-import gleam/regex
+import gleam/regexp
 import gleam/result
 
 fn query(
@@ -17,32 +17,35 @@ fn query(
   variables: Option(json.Json),
   decoder: dynamic.Decoder(a),
 ) {
-  let body =
-    json.object([
-      #("query", json.string(query)),
-      #("variables", json.nullable(variables, function.identity)),
-    ])
-  use response <- result.try(
+  use response <- result.try({
     request.new()
     |> request.set_header("authorization", "Bearer " <> token)
     |> request.set_header("user-agent", "gloogle / 1.0.0")
     |> request.set_method(http.Post)
     |> request.set_scheme(http.Https)
     |> request.set_host("api.github.com")
     |> request.set_path("/graphql")
-    |> request.set_body(json.to_string(body))
+    |> request.set_body(encode_body(query, variables))
     |> httpc.send()
-    |> result.map_error(error.FetchError),
-  )
+    |> result.map_error(error.FetchError)
+  })
 
   response.body
   |> json.decode(using: decoder)
   |> result.map_error(error.JsonError)
 }
 
+fn encode_body(query: String, variables: Option(json.Json)) -> String {
+  json.object([
+    #("query", json.string(query)),
+    #("variables", json.nullable(variables, function.identity)),
+  ])
+  |> json.to_string
+}
+
 fn match_repository_name(repo_url: String) {
-  let assert Ok(owner_name) = regex.from_string("https://github.com/(.+)/(.+)")
-  regex.scan(with: owner_name, content: repo_url)
+  let assert Ok(owner_name) = regexp.from_string("https://github.com/(.+)/(.+)")
+  regexp.scan(with: owner_name, content: repo_url)
   |> list.first()
   |> result.replace_error(error.UnknownError(
     "No repository match for " <> repo_url,

apps/backend/src/api/github.gleam

@@ -1,3 +1,4 @@
+import aws/s3
 import backend/error
 import gleam/bit_array
 import gleam/erlang/process
@@ -7,23 +8,22 @@ import gleam/httpc
 import gleam/json
 import gleam/package_interface
 import gleam/result
-import s3
 import simplifile
 import tom
 import wisp
 
-@external(erlang, "gloogle_hex_ffi", "get_home")
+@external(erlang, "backend_ffi", "get_home")
 pub fn get_home() -> Result(String, Nil)
 
-@external(erlang, "gloogle_hex_ffi", "extract_tar")
+@external(erlang, "backend_ffi", "extract_tar")
 fn extract_tar(
   tarbin: BitArray,
   base_name: String,
   version: String,
   slug: String,
 ) -> Result(#(String, String, String), Nil)
 
-@external(erlang, "gloogle_hex_ffi", "remove_tar")
+@external(erlang, "backend_ffi", "remove_tar")
 fn remove_tar(slug: String) -> Nil
 
 fn package_slug(name: String, version: String) {

apps/backend/src/api/hex_repo.gleam

@@ -1,26 +1,34 @@
 import aws4_request
-import backend/config
-import birl
+import backend/context
 import gleam/http
 import gleam/http/request
 import gleam/httpc
 import gleam/option.{type Option, None, Some}
 import gleam/result
 
 fn request(url: String, method: http.Method, body: Option(BitArray)) {
-  let date = birl.to_erlang_universal_datetime(birl.now())
-  use bucket_uri <- result.try(config.bucket_uri())
-  use #(access_key, secret_key) <- result.try(config.scaleway_keys())
+  use bucket_uri <- result.try(context.bucket_uri())
+  use #(access_key, secret_key) <- result.try(context.scaleway_keys())
   request.new()
   |> request.set_method(method)
   |> request.set_path(url)
   |> request.set_body(option.unwrap(body, <<>>))
   |> request.set_host(bucket_uri)
   |> request.set_scheme(http.Https)
   |> request.set_header("content-type", "application/octet-stream")
-  |> aws4_request.sign(date, access_key, secret_key, "fr-par", "s3")
+  |> sign(access_key, secret_key)
   |> httpc.send_bits()
-  |> result.nil_error()
+  |> result.replace_error(Nil)
+}
+
+fn sign(request, access_key_id, secret_access_key) {
+  aws4_request.signer(
+    access_key_id:,
+    secret_access_key:,
+    region: "fr-par",
+    service: "s3",
+  )
+  |> aws4_request.sign_bits(request)
 }
 
 pub fn get(name: String) {

apps/backend/src/s3.gleam renamed to apps/backend/src/aws/s3.gleam

@@ -1,73 +1,57 @@
-import backend/config
+import backend/context.{type Context, Context}
 import backend/gleam/type_search/state as type_search
-import backend/postgres/postgres
 import backend/router
+import backend/setup
+import backend/workers
+import envoy
 import gleam/erlang/process
 import gleam/function
+import gleam/int
+import gleam/option.{Some}
 import gleam/otp/supervisor
+import gleam/result
 import mist
-import periodic
-import setup
-import tasks/hex
-import tasks/popularity
-import tasks/ranking
-import tasks/timeseries
 import wisp
 import wisp/logger
 import wisp/wisp_mist
 
 pub fn main() {
-  wisp.configure_logger()
-
-  let secret_key_base = config.get_secret_key_base()
-  let cnf = config.read_config()
-  let ctx = postgres.connect(cnf)
+  configure_logger()
+  let assert Ok(ctx) = context.init()
+  let assert Ok(ctx) = start_type_search_worker(ctx)
+  let assert Ok(_) = start_http_server(ctx)
+  let assert Ok(_) = start_periodic_workers(ctx)
+  process.sleep_forever()
+}
 
-  logger.set_level(cnf.level)
+fn configure_logger() {
+  let level = logger.read_level()
+  wisp.configure_logger()
+  logger.set_level(level)
   setup.radiate()
+}
 
-  let assert Ok(subject) = type_search.init(ctx.db)
-
-  let ctx = ctx |> config.add_type_search_subject(subject)
-
-  let assert Ok(_) =
-    router.handle_request(_, ctx)
-    |> wisp_mist.handler(secret_key_base)
-    |> mist.new()
-    |> mist.port(cnf.port)
-    |> mist.start_http()
-
-  let assert Ok(_) = {
-    use periodic_children <- supervisor.start()
-    use periodic_children <- function.tap(periodic_children)
-    let assert Ok(_) = {
-      use children <- supervisor.start()
-      // Every 10 seconds
-      add_periodic_worker(periodic_children, waiting: 10 * 1000, do: fn() {
-        hex.sync_new_gleam_releases(ctx, children)
-      })
-      // Every day
-      add_periodic_worker(periodic_children, waiting: 86_400_000, do: fn() {
-        ranking.compute_ranking(ctx)
-      })
-      // Every day
-      add_periodic_worker(periodic_children, waiting: 86_400_000, do: fn() {
-        popularity.compute_popularity(ctx)
-      })
-      // Every hour
-      add_periodic_worker(periodic_children, waiting: 3600 * 1000, do: fn() {
-        timeseries.store_timeseries(ctx)
-      })
-    }
-  }
+fn start_type_search_worker(ctx: Context) {
+  use subject <- result.map(type_search.init(ctx.db))
+  Context(..ctx, type_search_subject: Some(subject))
+}
 
-  process.sleep_forever()
+fn start_http_server(ctx) {
+  use port <- result.try(envoy.get("PORT"))
+  use port <- result.map(int.parse(port))
+  let secret_key_base = context.get_secret_key_base()
+  router.handle_request(_, ctx)
+  |> wisp_mist.handler(secret_key_base)
+  |> mist.new()
+  |> mist.port(port)
+  |> mist.start_http()
 }
 
-fn add_periodic_worker(children, waiting delay, do work) {
-  use _ <- function.tap(children)
-  supervisor.add(children, {
-    use _ <- supervisor.worker()
-    periodic.periodically(do: work, waiting: delay)
-  })
+fn start_periodic_workers(ctx) {
+  use children <- supervisor.start()
+  use children <- function.tap(children)
+  workers.sync_new_gleam_releases_ten_secondly(ctx, children)
+  workers.compute_ranking_daily(ctx, children)
+  workers.compute_popularity_daily(ctx, children)
+  workers.store_timeseries_hourly(ctx, children)
 }