feat: add improved analytics

Author: ghivert

apps/backend/src/backend/postgres/queries.gleam

@@ -8,7 +8,6 @@ import gleam/dict.{type Dict}
 import gleam/dynamic
 import gleam/hexpm
 import gleam/int
-import gleam/io
 import gleam/json
 import gleam/list
 import gleam/option.{type Option, None, Some}
@@ -72,6 +71,36 @@ pub fn upsert_search_analytics(db: pgo.Connection, query: String) {
   })
 }
 
+pub fn select_more_popular_packages(db: pgo.Connection) {
+  let decoder =
+    dynamic.tuple4(
+      dynamic.string,
+      dynamic.string,
+      dynamic.int,
+      dynamic.optional(dynamic.int),
+    )
+  use ranked <- result.try({
+    "SELECT name, repository, rank, (popularity -> 'github')::int
+    FROM package
+    ORDER BY rank DESC
+    LIMIT 22"
+    |> pgo.execute(db, [], decoder)
+    |> result.map(fn(r) { r.rows })
+    |> result.map_error(error.DatabaseError)
+  })
+  use popular <- result.try({
+    "SELECT name, repository, rank, (popularity -> 'github')::int
+    FROM package
+    WHERE popularity -> 'github' IS NOT NULL
+    ORDER BY popularity -> 'github' DESC
+    LIMIT 23"
+    |> pgo.execute(db, [], decoder)
+    |> result.map(fn(r) { r.rows })
+    |> result.map_error(error.DatabaseError)
+  })
+  Ok(#(ranked, popular))
+}
+
 pub fn select_last_day_search_analytics(db: pgo.Connection) {
   let #(date, _) = birl.to_erlang_universal_datetime(birl.now())
   let now = birl.from_erlang_universal_datetime(#(date, #(0, 0, 0)))
@@ -113,7 +142,9 @@ pub fn get_timeseries_count(db: pgo.Connection) {
       (SELECT att.occurences
         FROM analytics_timeseries att
         WHERE att.date < at.date
-          AND att.query = at.query),
+          AND att.query = at.query
+        ORDER BY date DESC
+        LIMIT 1),
       0)
     ) searches,
     at.date date

apps/backend/src/backend/router.gleam

@@ -12,7 +12,6 @@ import gleam/int
 import gleam/json
 import gleam/list
 import gleam/option
-import gleam/pair
 import gleam/result
 import gleam/string_builder
 import tasks/hex as syncing
@@ -126,6 +125,16 @@ fn search(query: String, ctx: Context) {
   ])
 }
 
+fn encode_package(package: #(String, String, Int, option.Option(Int))) {
+  let #(name, repository, rank, popularity) = package
+  json.object([
+    #("name", json.string(name)),
+    #("repository", json.string(repository)),
+    #("rank", json.int(rank)),
+    #("popularity", json.nullable(popularity, json.int)),
+  ])
+}
+
 pub fn handle_get(req: Request, ctx: Context) {
   case wisp.path_segments(req) {
     ["healthcheck"] -> wisp.ok()
@@ -149,17 +158,23 @@ pub fn handle_get(req: Request, ctx: Context) {
         use total <- result.try(queries.get_total_searches(ctx.db))
         use signatures <- result.try(queries.get_total_signatures(ctx.db))
         use packages <- result.try(queries.get_total_packages(ctx.db))
+        use #(ranked, popular) <- result.try({
+          queries.select_more_popular_packages(ctx.db)
+        })
         let total = list.first(total) |> result.unwrap(0)
         let signatures = list.first(signatures) |> result.unwrap(0)
         let packages = list.first(packages) |> result.unwrap(0)
-        Ok(#(timeseries, total, signatures, packages))
+        Ok(#(timeseries, total, signatures, packages, ranked, popular))
       }
       |> result.map(fn(content) {
-        let #(timeseries, total, signatures, packages) = content
+        let #(timeseries, total, signatures, packages, ranked, popular) =
+          content
         json.object([
           #("total", json.int(total)),
           #("signatures", json.int(signatures)),
           #("packages", json.int(packages)),
+          #("ranked", json.array(ranked, encode_package)),
+          #("popular", json.array(popular, encode_package)),
           #("timeseries", {
             json.array(timeseries, fn(row) {
               json.object([

apps/frontend/src/bar_chart.element.mjs

@@ -0,0 +1,100 @@
+import { Chart } from 'chart.js/auto'
+
+export class BarChart extends HTMLElement {
+  static observedAttributes = ['datasets']
+
+  #shadow
+  #canvas
+  dataset
+
+  constructor() {
+    super()
+    this.#shadow = this.attachShadow({ mode: 'open' })
+  }
+
+  connectedCallback() {
+    this.#render()
+  }
+
+  #render() {
+    const labels = this.datasets.labels.toArray()
+    const data = this.datasets.data.toArray()
+    const color = this.color
+    const wrapper = document.createElement('div')
+    wrapper.style.position = 'relative'
+    wrapper.style.maxWidth = '850px'
+    // wrapper.style.padding = '12px'
+    // wrapper.style.maxHeight = '150px'
+    this.#canvas = document.createElement('canvas')
+    wrapper.appendChild(this.#canvas)
+    this.#shadow.appendChild(wrapper)
+    Chart.defaults.font.family = 'Lexend'
+    new Chart(this.#canvas, {
+      type: 'bar',
+      data: {
+        labels,
+        datasets: [
+          {
+            data,
+            borderColor: `${color}aa`,
+            backgroundColor: `${color}22`,
+            borderRadius: 5,
+          },
+        ],
+      },
+      options: {
+        aspectRatio: 1,
+        indexAxis: 'y',
+        responsive: true,
+        animation: false,
+        events: [],
+        layout: {
+          padding: {
+            right: 12,
+          },
+        },
+        plugins: {
+          legend: {
+            display: false,
+          },
+        },
+        elements: {
+          bar: {
+            borderWidth: 2,
+            barPercentage: 0.5,
+            barThickness: 6,
+            maxBarThickness: 8,
+            minBarLength: 2,
+          },
+        },
+        scales: {
+          x: {
+            display: true,
+            grid: { drawTicks: false, display: true },
+            ticks: { padding: 0, align: 'inner', padding: 5 },
+          },
+          y: {
+            display: true,
+            grid: { drawTicks: false, display: true },
+            ticks: {
+              padding: 5,
+              mirror: true,
+              includeBounds: false,
+              backdropPadding: 0,
+            },
+          },
+        },
+      },
+    })
+  }
+
+  // Lifecycle functions.
+  disconnectedCallback() {}
+  adoptedCallback() {}
+
+  attributeChangedCallback() {}
+
+  static register() {
+    customElements.define('bar-chart', BarChart)
+  }
+}

apps/frontend/src/line_chart.gleam renamed to apps/frontend/src/chart.gleam

@@ -1,4 +1,3 @@
-import gleam/string
 import lustre/attribute
 import lustre/element
 
@@ -14,3 +13,13 @@ pub fn line_chart(datasets: Dataset) {
     [],
   )
 }
+
+pub fn bar_chart(color: String, datasets: Dataset) {
+  let datasets = attribute.property("datasets", datasets)
+  let color = attribute.property("color", color)
+  element.element(
+    "bar-chart",
+    [attribute.style([#("display", "block")]), datasets, color],
+    [],
+  )
+}

apps/frontend/src/data/model.gleam

@@ -41,6 +41,8 @@ pub type Model {
     total_signatures: Int,
     total_packages: Int,
     timeseries: List(#(Int, birl.Time)),
+    ranked: List(msg.Package),
+    popular: List(msg.Package),
   )
 }
 
@@ -71,6 +73,8 @@ pub fn init() {
     total_signatures: 0,
     total_packages: 0,
     timeseries: [],
+    ranked: [],
+    popular: [],
   )
 }
 
@@ -102,18 +106,15 @@ pub fn update_input(model: Model, content: String) {
   Model(..model, input: content)
 }
 
-pub fn update_analytics(
-  model: Model,
-  analytics: #(Int, Int, Int, List(#(Int, birl.Time))),
-) {
-  let #(total_searches, total_signatures, total_packages, timeseries) =
-    analytics
+pub fn update_analytics(model: Model, analytics: msg.Analytics) {
   Model(
     ..model,
-    timeseries:,
-    total_searches:,
-    total_signatures:,
-    total_packages:,
+    timeseries: analytics.timeseries,
+    total_searches: analytics.total_searches,
+    total_signatures: analytics.total_signatures,
+    total_packages: analytics.total_indexed,
+    ranked: analytics.ranked,
+    popular: analytics.popular,
   )
 }
 
@@ -348,10 +349,12 @@ pub fn reset(model: Model) {
     show_old_packages: False,
     show_documentation_search: False,
     show_vector_search: False,
-    timeseries: [],
-    total_searches: 0,
-    total_signatures: 0,
-    total_packages: 0,
+    timeseries: model.timeseries,
+    total_searches: model.total_searches,
+    total_signatures: model.total_signatures,
+    total_packages: model.total_packages,
+    ranked: model.ranked,
+    popular: model.popular,
   )
 }
 

apps/frontend/src/data/msg.gleam

@@ -2,6 +2,7 @@ import birl
 import data/package
 import data/search_result.{type SearchResults}
 import frontend/router
+import gleam/option
 import lustre_http as http
 import plinth/browser/event.{type Event}
 
@@ -15,6 +16,26 @@ pub type Filter {
   VectorSearch
 }
 
+pub type Package {
+  Package(
+    name: String,
+    repository: String,
+    rank: Int,
+    popularity: option.Option(Int),
+  )
+}
+
+pub type Analytics {
+  Analytics(
+    total_searches: Int,
+    total_signatures: Int,
+    total_indexed: Int,
+    timeseries: List(#(Int, birl.Time)),
+    ranked: List(Package),
+    popular: List(Package),
+  )
+}
+
 pub type Msg {
   None
   OnSearchFocus(event: Event)
@@ -26,7 +47,7 @@ pub type Msg {
   Reset
   ScrollTo(String)
   OnEscape
-  Analytics(Result(#(Int, Int, Int, List(#(Int, birl.Time))), http.HttpError))
+  OnAnalytics(Result(Analytics, http.HttpError))
   OnRouteChange(router.Route)
   OnCheckFilter(Filter, Bool)
 }

apps/frontend/src/frontend.gleam

@@ -74,6 +74,16 @@ pub fn main() {
     |> lustre.start("#app", Nil)
 }
 
+fn decode_package(dyn) {
+  dynamic.decode4(
+    msg.Package,
+    dynamic.field("name", dynamic.string),
+    dynamic.field("repository", dynamic.string),
+    dynamic.field("rank", dynamic.int),
+    dynamic.field("popularity", dynamic.optional(dynamic.int)),
+  )(dyn)
+}
+
 fn init(_) {
   let initial =
     modem.initial_uri()
@@ -90,10 +100,10 @@ fn init(_) {
     |> http.get(config.api_endpoint() <> "/trendings", _),
   )
   |> update.add_effect(
-    msg.Analytics
+    msg.OnAnalytics
     |> http.expect_json(
-      dynamic.decode4(
-        fn(a, b, c, d) { #(a, b, c, d) },
+      dynamic.decode6(
+        msg.Analytics,
         dynamic.field("total", dynamic.int),
         dynamic.field("signatures", dynamic.int),
         dynamic.field("packages", dynamic.int),
@@ -110,6 +120,8 @@ fn init(_) {
             }),
           ))
         }),
+        dynamic.field("ranked", dynamic.list(decode_package)),
+        dynamic.field("popular", dynamic.list(decode_package)),
       ),
       _,
     )
@@ -142,7 +154,7 @@ fn update(model: Model, msg: Msg) {
       handle_search_results(model, input, search_results)
     msg.OnCheckFilter(filter, value) ->
       handle_oncheck_filter(model, filter, value)
-    msg.Analytics(analytics) -> {
+    msg.OnAnalytics(analytics) -> {
       case analytics {
         Error(_) -> #(model, effect.none())
         Ok(analytics) ->