✨ feat: support auto GC

README.md

@@ -8,7 +8,8 @@ Monitor your websites, showcase status including daily history, and get Slack no
 
 - 🦄 Written in TypeScript
 - ✨ Support remote csv monitors
-- 🚀 No max monitors limit, even with workers KV free tier
+- 🚀 No limit for max monitors of cron task, even with workers KV free tier
+- 🪁 [Auto GC](./src/worker/_helpers/store.ts#L77) for KV value size
 - 💎 More DX/UX detail you want
 
 ## Pre-requisites

src/helpers/memory.ts

@@ -0,0 +1,54 @@
+const SCALE = 1024
+export const aKiB = SCALE
+export const aMiB = SCALE ** 2
+export const aGiB = SCALE ** 3
+
+// ref: https://gist.github.com/rajinwonderland/36887887b8a8f12063f1d672e318e12e
+export function memorySizeOf(obj: unknown) {
+  let bytes = 0
+
+  function sizeOf(obj: unknown) {
+    if (obj !== null && obj !== undefined) {
+      switch (typeof obj) {
+        case 'number':
+          bytes += 8
+          break
+        case 'string':
+          bytes += obj.length * 2
+          break
+        case 'boolean':
+          bytes += 4
+          break
+        case 'object': {
+          const objClass = Object.prototype.toString.call(obj).slice(8, -1)
+          if (objClass === 'Object' || objClass === 'Array') {
+            for (const key in obj) {
+              if (!Object.prototype.hasOwnProperty.call(obj, key)) {
+                continue
+              }
+              sizeOf(obj[key as keyof typeof obj])
+            }
+          }
+          else { bytes += obj.toString().length * 2 }
+          break
+        }
+      }
+    }
+    return bytes
+  }
+
+  function formatByteSize(bytes: number) {
+    if (bytes < aKiB) {
+      return `${bytes} bytes`
+    }
+    else if (bytes < aMiB) {
+      return `${(bytes / aMiB).toFixed(3)} KiB`
+    }
+    else if (bytes < aGiB) {
+      return `${(bytes / aMiB).toFixed(3)} MiB`
+    }
+    else { return `${(bytes / aGiB).toFixed(3)} GiB` }
+  }
+
+  return { bytes, humanize: formatByteSize(sizeOf(obj)) }
+}

src/pages/index/components/MonitorPanel/index.tsx

@@ -4,8 +4,7 @@ import React from 'react'
 import type { DataV1 } from '#src/worker/_helpers/store'
 import type { Monitor } from '#src/types'
 
-import { config } from '#src/config'
-import { getHistoryDates } from '#src/worker/_helpers/datetime'
+import { getDisplayDays, getHistoryDates } from '#src/worker/_helpers/datetime'
 import { parseLocation } from '#src/helpers/locations'
 import { Tooltip, TooltipContent, TooltipTrigger } from '#src/components/Tooltip'
 import { getChecksItemRenderStatus, getTargetDateChecksItem } from '#src/helpers/checks'
@@ -31,7 +30,6 @@ const MonitorPanel: React.FC<IMonitorPanelProps> = (props) => {
   }
 
   const monitorIds = (Object.keys(data.monitorHistoryData) || [])
-  const displayDays = config.settings.displayDays || 90
   const allOperational = data.lastUpdate?.checks.allOperational
 
   const titleCls = allOperational ? cls`border-green-500 bg-green-300 text-green-800` : cls`border-red-500 bg-red-300 text-red-800`
@@ -168,7 +166,7 @@ const MonitorPanel: React.FC<IMonitorPanelProps> = (props) => {
                 )}
               </div>
               <ul className='flex gap-1'>
-                {getHistoryDates(displayDays).map((dateItem) => {
+                {getHistoryDates().map((dateItem) => {
                   const targetDateChecksItem = getTargetDateChecksItem(monitorData, dateItem)
                   const renderStatus = getChecksItemRenderStatus(monitorData, dateItem)
 
@@ -201,7 +199,7 @@ const MonitorPanel: React.FC<IMonitorPanelProps> = (props) => {
                       break
                   }
 
-                  const itemWidth = `calc(100% / ${displayDays})`
+                  const itemWidth = `calc(100% / ${getDisplayDays()})`
 
                   return (
                     <Tooltip key={dateItem}>

src/worker/_helpers/datetime.ts

@@ -1,8 +1,12 @@
+import { config } from '#src/config'
+
+export const getDisplayDays = () => config.settings.displayDays || 90
+
 export function getDate(date?: Date | null) {
   return (date ?? new Date()).toISOString().split('T')[0]
 }
 
-export function getHistoryDates(days: number) {
+export function getHistoryDates(days = getDisplayDays()) {
   const date = new Date()
   date.setDate(date.getDate() - days)
 

src/worker/_helpers/store.ts

@@ -1,4 +1,9 @@
+import { getHistoryDates } from './datetime'
+
+import type { Monitor } from '#src/types'
+
 import { config } from '#src/config'
+import { aMiB, memorySizeOf } from '#src/helpers/memory'
 import getRemoteMonitors from '#src/helpers/monitors'
 import { ensureWorkerEnv } from '#src/worker/_helpers'
 
@@ -61,12 +66,44 @@ export interface DataV1 {
   lastUpdate?: DataV1LastCheck
 }
 
-export async function upsertKvStore(value: DataV1 | null) {
+export async function upsertKvStore(value: DataV1 | null, allMonitors: Monitor[]) {
   ensureWorkerEnv()
-  const result = await (value === null ? KV_STORE.delete(DATA_KEY) : KV_STORE.put(DATA_KEY, JSON.stringify(value)))
+  const result = await (value === null
+    ? KV_STORE.delete(DATA_KEY)
+    : KV_STORE.put(DATA_KEY, JSON.stringify(cleanDataV1(value, allMonitors))))
   return result
 }
 
+export async function cleanDataV1(value: DataV1, allMonitors: Monitor[]) {
+  const { bytes } = memorySizeOf(JSON.stringify(value))
+
+  // https://developers.cloudflare.com/kv/platform/limits/
+  // Value max size 25 MiB, in case of exceptions, we clean data when bytes bigger than 24 MiB.
+  if (bytes < 24 * aMiB) {
+    return value
+  }
+
+  const { monitorHistoryData = {}, ...rest } = value
+  const historyDates = getHistoryDates()
+
+  return {
+    ...rest,
+    ...(Object.keys(monitorHistoryData).filter((item) => {
+      // Remove monitor data from state if missing in monitors config
+      return allMonitors.some((monitorItem) => monitorItem.id === item)
+    }).reduce<Record<string, MonitorAllData>>((previous, current) => {
+      const { checks, ...restHistoryData } = monitorHistoryData[current]
+      return { ...previous, current: {
+        ...restHistoryData,
+        // Remove dates older than config.settings.displayDays
+        checks: checks.filter((item) => {
+          return historyDates.includes(item.date)
+        }),
+      } }
+    }, {})),
+  }
+}
+
 export async function getStore() {
   ensureWorkerEnv()
   // https://developers.cloudflare.com/kv/api/read-key-value-pairs/
@@ -107,11 +144,13 @@ export async function prepareMonitors() {
     return {
       uncheckMonitors: allMonitors,
       lastCheckedMonitorIds: [],
+      allMonitors,
     }
   }
 
   return {
     uncheckMonitors,
     lastCheckedMonitorIds,
+    allMonitors,
   }
 }

src/worker/cron/index.ts

@@ -23,7 +23,7 @@ export async function handleCronTrigger(event: FetchEvent) {
   const { kvData } = await getStore()
   subrequests.required()
 
-  const { uncheckMonitors, lastCheckedMonitorIds } = await prepareMonitors()
+  const { uncheckMonitors, lastCheckedMonitorIds, allMonitors } = await prepareMonitors()
   console.debug('uncheckMonitors:', uncheckMonitors)
 
   for (const monitor of uncheckMonitors) {
@@ -139,6 +139,6 @@ export async function handleCronTrigger(event: FetchEvent) {
     },
   }
 
-  await upsertKvStore(kvData)
+  await upsertKvStore(kvData, allMonitors)
   return new Response('OK')
 }