feat: [SPMVP -6521] use typesense to fetch article list in karbon to reduce api server load (#281)

Author: SidStraw

packages/karbon/package.json

@@ -174,6 +174,7 @@
     "tiny-invariant": "^1.3.1",
     "ts-pattern": "^5.0.5",
     "type-fest": "^4.3.1",
+    "typesense": "^1.7.1",
     "typesense-instantsearch-adapter": "^2.7.1",
     "unbuild": "^2.0.0",
     "unenv": "^1.7.4",
@@ -229,4 +230,4 @@
     "access": "public"
   },
   "gitHead": "8df1f4d5837a7e2ddbff6cc79f5fec256c34a394"
-}
+}

packages/karbon/src/runtime/api/article.ts

@@ -5,80 +5,18 @@ import { encrypt } from 'micro-aes-gcm'
 // This file contains global crypto polyfill
 import { CompactEncrypt } from '@storipress/jose-browser'
 import { useStoripressClient } from '../composables/storipress-client'
+import type { TypesenseFilter } from '../composables/typesense-client'
+import { PER_PAGE, getSearchQuery, useTypesenseClient } from '../composables/typesense-client'
 import { splitPaidContent } from '../lib/split-paid-content'
 import type { NormalSegment } from '../lib/split-article'
 import { splitArticle } from '../lib/split-article'
 import { getStoripressConfig } from '../composables/storipress-base-client'
 import { verboseInvariant } from '../utils/verbose-invariant'
-import { getAllWithPagination } from './helper'
 import type { PaidContent, RawArticleLike, _NormalizeArticle } from './normalize-article'
 import { normalizeArticle } from './normalize-article'
 
 export type { NormalizeArticle, PaidContent } from './normalize-article'
 
-export const ListArticles = gql`
-  query ListArticles($page: Int!) {
-    articles(page: $page, sortBy: [{ column: PUBLISHED_AT, order: DESC }], published: true) {
-      paginatorInfo {
-        lastPage
-        hasMorePages
-        count
-      }
-      data {
-        id
-        title
-        blurb
-        slug
-        sid
-        published_at
-        updated_at
-        featured
-        plan
-        cover
-        seo
-        layout {
-          id
-          name
-        }
-        desk {
-          id
-          name
-          slug
-          layout {
-            id
-            name
-          }
-          desk {
-            id
-            name
-            slug
-            layout {
-              id
-              name
-            }
-          }
-        }
-        tags {
-          id
-          slug
-          name
-        }
-        authors {
-          id
-          slug
-          bio
-          socials
-          avatar
-          email
-          location
-          first_name
-          last_name
-          full_name
-        }
-      }
-    }
-  }
-`
 const GetArticle = gql`
   query GetArticle($id: ID!) {
     article(id: $id) {
@@ -315,14 +253,23 @@ const GetArticle = gql`
   }
 `
 
-export async function listArticles(filter?: { desk: string; tag: string; author: string }) {
-  return getAllWithPagination(ListArticles, filter, ({ articles: { paginatorInfo, data } }) => {
-    const res = data.map((data: RawArticleLike) => normalizeArticle(data))
-    return {
-      paginatorInfo,
-      data: res,
-    }
-  })
+export async function listArticles(filter?: TypesenseFilter) {
+  const typesenseClient = useTypesenseClient()
+  const documents = typesenseClient?.collections('articles').documents()
+
+  const articles = []
+  let hasMore = true
+  let page = 1
+  while (hasMore) {
+    const searchResult = await documents?.search(getSearchQuery(page, filter), {})
+    const currentPageArticles =
+      searchResult?.hits?.map(({ document }) => normalizeArticle(document as RawArticleLike)) ?? []
+    articles.push(...currentPageArticles)
+
+    hasMore = searchResult.found > searchResult.page * PER_PAGE
+    page = searchResult.page + 1
+  }
+  return articles
 }
 
 export async function getArticle(id: string) {

packages/karbon/src/runtime/api/feed.ts

@@ -1,35 +1,5 @@
 import { gql } from '@apollo/client/core/index.js'
 import { useStoripressClient } from '../composables/storipress-client'
-import { getAllWithPagination } from './helper'
-import type { RawArticleLike } from './normalize-article'
-import { normalizeArticle } from './normalize-article'
-
-const ListArticles = gql`
-  query ListArticles($page: Int!, $desk: ID, $desk_ids: [ID!]) {
-    articles(
-      page: $page
-      desk: $desk
-      desk_ids: $desk_ids
-      sortBy: [{ column: UPDATED_AT, order: DESC }]
-      published: true
-    ) {
-      paginatorInfo {
-        count
-        lastPage
-        hasMorePages
-      }
-      data {
-        id
-        title
-        slug
-        sid
-        published_at
-        html
-        plaintext
-      }
-    }
-  }
-`
 
 const GetDesk = gql`
   query GetDesk($slug: String) {
@@ -51,14 +21,9 @@ const GetDesk = gql`
   }
 `
 
-export function listFeedArticles(filter?: { desk: string; tag: string; author: string; desk_ids: string }) {
-  return getAllWithPagination(ListArticles, filter, ({ articles: { paginatorInfo, data } }) => {
-    const res = data.map((data: RawArticleLike) => normalizeArticle(data))
-    return {
-      paginatorInfo,
-      data: res,
-    }
-  })
+export async function listFeedArticles() {
+  const allArticles = (await $fetch('/_storipress/posts/__all.json')) ?? []
+  return allArticles
 }
 
 export async function getDeskWithSlug(slug: string) {

packages/karbon/src/runtime/api/normalize-article.ts

@@ -1,7 +1,7 @@
 import { destr } from 'destr'
 import truncate from 'lodash.truncate'
 import type { Segment } from '../lib/split-article'
-import type { ArticlePlan } from '../types'
+import type { ArticleDesk, ArticlePlan, ArticleTag } from '../types'
 import { useArticleFilter } from '#imports'
 
 export interface RawUserLike {
@@ -24,6 +24,9 @@ export interface RawArticleLike {
   plaintext: string
   plan: ArticlePlan
   authors: RawUserLike[]
+  tags: ArticleTag[]
+  desk: ArticleDesk
+  published_at: string
 }
 
 export interface PaidContent {
@@ -53,9 +56,12 @@ export function normalizeArticle({
   html,
   id,
   authors,
+  desk,
+  tags,
   ...rest
 }: RawArticleLike) {
   const articleFilter = useArticleFilter()
+  const rootDesk = desk.desk ? { desk: { ...desk.desk, id: String(desk.desk.id) } } : {}
 
   return {
     ...rest,
@@ -74,11 +80,21 @@ export function normalizeArticle({
       separator: /,? +/,
     }),
     cover: destr(cover),
-    authors: authors?.map(({ socials, ...rest }) => ({
+    authors: authors?.map(({ socials, id, ...rest }) => ({
       ...rest,
+      id: String(id),
       socials: destr(socials),
       name: rest.full_name,
     })),
+    desk: {
+      ...desk,
+      ...rootDesk,
+      id: String(desk.id),
+    },
+    tags: tags?.map(({ id, ...rest }) => ({
+      ...rest,
+      id: String(id),
+    })),
   }
 }
 

packages/karbon/src/runtime/api/sitemap.ts

@@ -3,7 +3,6 @@ import { identity } from 'remeda'
 import { createStoripressClient } from '../composables/storipress-client'
 import { storipressConfigCtx } from '../composables/storipress-base-client'
 import type { ModuleRuntimeConfig } from '../types'
-import { getAllWithPaginationViaGetPage } from './helper'
 
 const ListArticles = gql`
   query ListArticles($page: Int!) {
@@ -136,11 +135,7 @@ export async function getResources(runtimeConfig?: ModuleRuntimeConfig['storipre
       let resources: any = []
       switch (payloadScope) {
         case 'posts': {
-          const getPage = async (page: number) => {
-            const { data } = await client.query({ query, variables: { page } })
-            return data.articles
-          }
-          resources = await getAllWithPaginationViaGetPage(getPage)
+          resources = (await $fetch('/_storipress/posts/__all.json')) ?? []
           break
         }
         default: {

packages/karbon/src/runtime/composables/typesense-client.ts

@@ -0,0 +1,51 @@
+import { SearchClient } from 'typesense'
+import { getStoripressConfig } from './storipress-base-client'
+
+let typesenseClient: SearchClient
+
+export function useTypesenseClient() {
+  if (typesenseClient) return typesenseClient
+
+  const storipress = getStoripressConfig()
+  typesenseClient = new SearchClient({
+    nodes: [
+      {
+        host: storipress.searchDomain ?? '',
+        port: 443,
+        protocol: 'https',
+      },
+    ],
+    apiKey: storipress.searchKey,
+    connectionTimeoutSeconds: 5,
+  })
+  return typesenseClient
+}
+
+export interface TypesenseFilter {
+  desk_ids?: string[]
+  author_ids?: string[]
+  author_names?: string[]
+  tag_ids?: string[]
+  tag_names?: string[]
+}
+
+export const PER_PAGE = 100
+
+export function getSearchQuery(page = 1, filter: TypesenseFilter = {}) {
+  const { desk_ids, author_ids, author_names, tag_ids, tag_names } = filter
+  let filterBy = 'published:=true'
+  if (desk_ids?.length) filterBy += ` && desk_id:=[${desk_ids.join()}]`
+  if (author_ids?.length) filterBy += ` && author_ids:=[${author_ids.join()}]`
+  if (author_names?.length) filterBy += ` && author_names:=[${author_names.join()}]`
+  if (tag_ids?.length) filterBy += ` && tag_ids:=[${tag_ids.join()}]`
+  if (tag_names?.length) filterBy += ` && tag_names:=[${tag_names.join()}]`
+
+  return {
+    q: '*',
+    sort_by: 'published_at:desc,order:asc',
+    filter_by: filterBy,
+    per_page: PER_PAGE,
+    page,
+    query_by: 'title',
+  }
+}