feat: [SPMVP -6521] use typesense to fetch article list in karbon to reduce api server load

packages/karbon/package.json

@@ -174,6 +174,7 @@
     "tiny-invariant": "^1.3.1",
     "ts-pattern": "^5.0.5",
     "type-fest": "^4.3.1",
+    "typesense": "^1.7.1",
     "typesense-instantsearch-adapter": "^2.7.1",
     "unbuild": "^2.0.0",
     "unenv": "^1.7.4",
@@ -229,4 +230,4 @@
     "access": "public"
   },
   "gitHead": "8df1f4d5837a7e2ddbff6cc79f5fec256c34a394"
-}
+}

packages/karbon/src/runtime/api/article.ts

@@ -5,80 +5,18 @@ import { encrypt } from 'micro-aes-gcm'
 // This file contains global crypto polyfill
 import { CompactEncrypt } from '@storipress/jose-browser'
 import { useStoripressClient } from '../composables/storipress-client'
+import type { TypesenseFilter } from '../composables/typesense-client'
+import { PER_PAGE, getSearchQuery, useTypesenseClient } from '../composables/typesense-client'
 import { splitPaidContent } from '../lib/split-paid-content'
 import type { NormalSegment } from '../lib/split-article'
 import { splitArticle } from '../lib/split-article'
 import { getStoripressConfig } from '../composables/storipress-base-client'
 import { verboseInvariant } from '../utils/verbose-invariant'
-import { getAllWithPagination } from './helper'
 import type { PaidContent, RawArticleLike, _NormalizeArticle } from './normalize-article'
 import { normalizeArticle } from './normalize-article'
 
 export type { NormalizeArticle, PaidContent } from './normalize-article'
 
-export const ListArticles = gql`
-  query ListArticles($page: Int!) {
-    articles(page: $page, sortBy: [{ column: PUBLISHED_AT, order: DESC }], published: true) {
-      paginatorInfo {
-        lastPage
-        hasMorePages
-        count
-      }
-      data {
-        id
-        title
-        blurb
-        slug
-        sid
-        published_at
-        updated_at
-        featured
-        plan
-        cover
-        seo
-        layout {
-          id
-          name
-        }
-        desk {
-          id
-          name
-          slug
-          layout {
-            id
-            name
-          }
-          desk {
-            id
-            name
-            slug
-            layout {
-              id
-              name
-            }
-          }
-        }
-        tags {
-          id
-          slug
-          name
-        }
-        authors {
-          id
-          slug
-          bio
-          socials
-          avatar
-          email
-          location
-          first_name
-          last_name
-          full_name
-        }
-      }
-    }
-  }
-`
 const GetArticle = gql`
   query GetArticle($id: ID!) {
     article(id: $id) {
@@ -315,14 +253,23 @@ const GetArticle = gql`
   }
 `
 
-export async function listArticles(filter?: { desk: string; tag: string; author: string }) {
-  return getAllWithPagination(ListArticles, filter, ({ articles: { paginatorInfo, data } }) => {
-    const res = data.map((data: RawArticleLike) => normalizeArticle(data))
-    return {
-      paginatorInfo,
-      data: res,
-    }
-  })
+export async function listArticles(filter?: TypesenseFilter) {
+  const typesenseClient = useTypesenseClient()
+  const documents = typesenseClient?.collections('articles').documents()
+
+  const articles = []
+  let hasMore = true
+  let page = 1
+  while (hasMore) {
+    const searchResult = await documents?.search(getSearchQuery(page, filter), {})
+    const currentPageArticles =
+      searchResult?.hits?.map(({ document }) => normalizeArticle(document as RawArticleLike)) ?? []
+    articles.push(...currentPageArticles)
+
+    hasMore = searchResult.found > searchResult.page * PER_PAGE
+    page = searchResult.page + 1
+  }
+  return articles
 }
 
 export async function getArticle(id: string) {

packages/karbon/src/runtime/api/feed.ts

@@ -1,35 +1,5 @@
 import { gql } from '@apollo/client/core/index.js'
 import { useStoripressClient } from '../composables/storipress-client'
-import { getAllWithPagination } from './helper'
-import type { RawArticleLike } from './normalize-article'
-import { normalizeArticle } from './normalize-article'
-
-const ListArticles = gql`
-  query ListArticles($page: Int!, $desk: ID, $desk_ids: [ID!]) {
-    articles(
-      page: $page
-      desk: $desk
-      desk_ids: $desk_ids
-      sortBy: [{ column: UPDATED_AT, order: DESC }]
-      published: true
-    ) {
-      paginatorInfo {
-        count
-        lastPage
-        hasMorePages
-      }
-      data {
-        id
-        title
-        slug
-        sid
-        published_at
-        html
-        plaintext
-      }
-    }
-  }
-`
 
 const GetDesk = gql`
   query GetDesk($slug: String) {
@@ -51,14 +21,9 @@ const GetDesk = gql`
   }
 `
 
-export function listFeedArticles(filter?: { desk: string; tag: string; author: string; desk_ids: string }) {
-  return getAllWithPagination(ListArticles, filter, ({ articles: { paginatorInfo, data } }) => {
-    const res = data.map((data: RawArticleLike) => normalizeArticle(data))
-    return {
-      paginatorInfo,
-      data: res,
-    }
-  })
+export async function listFeedArticles() {
+  const allArticles = (await $fetch('/_storipress/posts/__all.json')) ?? []
+  return allArticles
 }
 
 export async function getDeskWithSlug(slug: string) {

packages/karbon/src/runtime/api/normalize-article.ts

@@ -24,6 +24,7 @@ export interface RawArticleLike {
   plaintext: string
   plan: ArticlePlan
   authors: RawUserLike[]
+  published_at: string
 }
 
 export interface PaidContent {

packages/karbon/src/runtime/api/sitemap.ts

@@ -3,7 +3,6 @@ import { identity } from 'remeda'
 import { createStoripressClient } from '../composables/storipress-client'
 import { storipressConfigCtx } from '../composables/storipress-base-client'
 import type { ModuleRuntimeConfig } from '../types'
-import { getAllWithPaginationViaGetPage } from './helper'
 
 const ListArticles = gql`
   query ListArticles($page: Int!) {
@@ -136,11 +135,7 @@ export async function getResources(runtimeConfig?: ModuleRuntimeConfig['storipre
       let resources: any = []
       switch (payloadScope) {
         case 'posts': {
-          const getPage = async (page: number) => {
-            const { data } = await client.query({ query, variables: { page } })
-            return data.articles
-          }
-          resources = await getAllWithPaginationViaGetPage(getPage)
+          resources = (await $fetch('/_storipress/posts/__all.json')) ?? []
           break
         }
         default: {

packages/karbon/src/runtime/composables/typesense-client.ts

@@ -0,0 +1,51 @@
+import { SearchClient } from 'typesense'
+import { getStoripressConfig } from './storipress-base-client'
+
+let typesenseClient: SearchClient
+
+export function useTypesenseClient() {
+  if (typesenseClient) return typesenseClient
+
+  const storipress = getStoripressConfig()
+  typesenseClient = new SearchClient({
+    nodes: [
+      {
+        host: storipress.searchDomain ?? '',
+        port: 443,
+        protocol: 'https',
+      },
+    ],
+    apiKey: storipress.searchKey,
+    connectionTimeoutSeconds: 5,
+  })
+  return typesenseClient
+}
+
+export interface TypesenseFilter {
+  desk_ids?: string[]
+  author_ids?: string[]
+  author_names?: string[]
+  tag_ids?: string[]
+  tag_names?: string[]
+}
+
+export const PER_PAGE = 100
+
+export function getSearchQuery(page = 1, filter: TypesenseFilter = {}) {
+  const { desk_ids, author_ids, author_names, tag_ids, tag_names } = filter
+  let filterBy = 'published:=true'
+  if (desk_ids?.length) filterBy += ` && desk_id:=[${desk_ids.join()}]`
+  if (author_ids?.length) filterBy += ` && author_ids:=[${author_ids.join()}]`
+  if (author_names?.length) filterBy += ` && author_names:=[${author_names.join()}]`
+  if (tag_ids?.length) filterBy += ` && tag_ids:=[${tag_ids.join()}]`
+  if (tag_names?.length) filterBy += ` && tag_names:=[${tag_names.join()}]`
+
+  return {
+    q: '*',
+    sort_by: 'published_at:desc,order:asc',
+    filter_by: filterBy,
+    per_page: PER_PAGE,
+    page,
+    query_by: 'title',
+  }
+}

packages/karbon/src/runtime/routes/atom-desk.xml.ts

@@ -2,82 +2,59 @@ import { defineEventHandler, sendNoContent, setHeader } from 'h3'
 import { Feed } from 'feed'
 import { encodePath, joinURL, withTrailingSlash } from 'ufo'
 import path from 'pathe'
-import { getDeskWithSlug, listFeedArticles } from '@storipress/karbon/internal'
-import type { Author } from '../composables/page-meta'
+import { getDeskWithSlug } from '@storipress/karbon/internal'
+import { listArticles } from '../api/article'
 import { useRuntimeConfig } from '#imports'
 import urls from '#sp-internal/storipress-urls.mjs'
 
-interface TArticle {
-  title: string
-  id: string
-  link: string
-  description: string
-  content: string
-  date: Date
-  author: Author[]
-  plaintext: string
-  html: string
-  published_at: string
-}
-
 export default defineEventHandler(async (e) => {
   setHeader(e, 'Content-Type', 'text/xml; charset=UTF-8')
   if (!process.dev) setHeader(e, 'Cache-Control', 'max-age=600, must-revalidate')
 
-  const fileName = e.context.params?.slug || ''
+  const fileName = e.context.params?.slug ?? ''
   if (!fileName.endsWith('.xml')) {
     return sendNoContent(e, 404)
   }
 
-  const slug = path.parse(fileName || '').name
+  const slug = path.parse(fileName).name
   const desk = await getDeskWithSlug(slug)
   if (!desk?.id) {
     return sendNoContent(e, 404)
   }
 
   const deskIds: string[] = desk.desks?.map(({ id }: { id: string }) => id) ?? []
 
-  type Filter = Record<'desk' | 'desk_ids', string | string[]>
-  const filter = {} as Filter
-  if (deskIds.length !== 0) {
-    filter.desk_ids = deskIds
-  } else {
-    filter.desk = desk.id
-  }
-
   const runtimeConfig = useRuntimeConfig()
-  const articles = await listFeedArticles(filter)
+  const articles = await listArticles({ desk_ids: deskIds })
 
-  const siteUrl = runtimeConfig.public.siteUrl
+  const siteUrl = runtimeConfig.public.siteUrl as string
   const feed = new Feed({
-    id: withTrailingSlash(runtimeConfig.public.siteUrl),
-    link: withTrailingSlash(runtimeConfig.public.siteUrl),
-    title: runtimeConfig.public.siteName,
-    description: runtimeConfig.public.siteDescription,
+    id: withTrailingSlash(runtimeConfig.public.siteUrl as string),
+    link: withTrailingSlash(runtimeConfig.public.siteUrl as string),
+    title: runtimeConfig.public.siteName as string,
+    description: runtimeConfig.public.siteDescription as string,
     updated: new Date(),
     feedLinks: {
       atom: joinURL(siteUrl, `/atom/${fileName}`),
     },
     copyright: `© ${runtimeConfig.public.siteName} ${new Date().getFullYear()} All Rights Reserved`,
   })
 
-  articles
-    .filter((article: TArticle) => article.published_at)
-    .forEach((article: TArticle) => {
-      const id = encodePath(urls.article.toURL(article, urls.article._context))
-      feed.addItem({
-        title: article.title,
-        id: joinURL(siteUrl, id),
-        link: joinURL(siteUrl, id),
-        description: article.plaintext.slice(0, 120),
-        date: new Date(article.published_at),
-        author:
-          article.author?.map((author) => ({
-            name: author.name,
-          })) || [],
-        content: article.html,
-      })
+  articles.forEach((article) => {
+    const id = encodePath(urls.article.toURL(article, urls.article._context))
+    feed.addItem({
+      title: article.title,
+      id: joinURL(siteUrl, id),
+      link: joinURL(siteUrl, id),
+      description: article.plaintext.slice(0, 120),
+      date: new Date(article.published_at),
+      author:
+        article.authors?.map((author) => ({
+          name: author.name,
+        })) || [],
+      content: article.html,
     })
+  })
 
   return feed.atom1()
 })

yarn.lock

@@ -3448,6 +3448,7 @@ __metadata:
     tsx: 3.12.10
     type-fest: ^4.3.1
     typescript: 5.2.2
+    typesense: ^1.7.1
     typesense-instantsearch-adapter: ^2.7.1
     unbuild: ^2.0.0
     unenv: ^1.7.4
@@ -17042,7 +17043,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"typesense@npm:^1.7.0":
+"typesense@npm:^1.7.0, typesense@npm:^1.7.1":
   version: 1.7.1
   resolution: "typesense@npm:1.7.1"
   dependencies: