diff --git a/convex/crons.ts b/convex/crons.ts index e36dd97..c35b2ce 100644 --- a/convex/crons.ts +++ b/convex/crons.ts @@ -26,9 +26,9 @@ crons.interval( crons.interval( 'skill-stat-events', - { minutes: 5 }, - internal.skillStatEvents.processSkillStatEventsInternal, - { batchSize: 100 }, + { minutes: 15 }, + internal.skillStatEvents.processSkillStatEventsAction, + {}, ) export default crons diff --git a/convex/schema.ts b/convex/schema.ts index 4ff8eb5..ec37898 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -289,6 +289,12 @@ const skillStatEvents = defineTable({ .index('by_unprocessed', ['processedAt']) .index('by_skill', ['skillId']) +const skillStatUpdateCursors = defineTable({ + key: v.string(), + cursorCreationTime: v.optional(v.number()), + updatedAt: v.number(), +}).index('by_key', ['key']) + const soulEmbeddings = defineTable({ soulId: v.id('souls'), versionId: v.id('soulVersions'), @@ -450,6 +456,7 @@ export default defineSchema({ skillLeaderboards, skillStatBackfillState, skillStatEvents, + skillStatUpdateCursors, comments, skillReports, soulComments, diff --git a/convex/search.ts b/convex/search.ts index 37fbf7e..6cd09e4 100644 --- a/convex/search.ts +++ b/convex/search.ts @@ -105,6 +105,14 @@ export const searchSkills: ReturnType = action({ }, }) +export const getBadgeMapsForSkills = internalQuery({ + args: { skillIds: v.array(v.id('skills')) }, + handler: async (ctx, args): Promise, SkillBadgeMap]>> => { + const badgeMap = await getSkillBadgeMaps(ctx, args.skillIds) + return Array.from(badgeMap.entries()) + }, +}) + export const hydrateResults = internalQuery({ args: { embeddingIds: v.array(v.id('skillEmbeddings')) }, handler: async (ctx, args): Promise => { diff --git a/convex/skillStatEvents.ts b/convex/skillStatEvents.ts index 3846c84..03d6722 100644 --- a/convex/skillStatEvents.ts +++ b/convex/skillStatEvents.ts @@ -17,7 +17,7 @@ import { v } from 'convex/values' import { internal } from './_generated/api' import type { Doc, Id } from './_generated/dataModel' import type { MutationCtx } from './_generated/server' -import { internalMutation } from './_generated/server' +import { internalAction, internalMutation, internalQuery } from './_generated/server' import { applySkillStatDeltas, bumpDailySkillStats } from './lib/skillStats' /** @@ -278,3 +278,291 @@ export const processSkillStatEventsInternal = internalMutation({ return { processed: events.length } }, }) + +// ============================================================================ +// Action-based processing (cursor-based, runs outside transaction window) +// ============================================================================ + +const CURSOR_KEY = 'skill_stat_events' +const EVENT_BATCH_SIZE = 500 +const MAX_SKILLS_PER_RUN = 500 + +/** + * Fetch a batch of events after the given cursor (by _creationTime). + * Returns events sorted by _creationTime ascending. + */ +export const getUnprocessedEventBatch = internalQuery({ + args: { + cursorCreationTime: v.optional(v.number()), + limit: v.optional(v.number()), + }, + handler: async (ctx, args) => { + const limit = args.limit ?? EVENT_BATCH_SIZE + const cursor = args.cursorCreationTime + + // Query events after the cursor using the built-in creation time index + const events = await ctx.db + .query('skillStatEvents') + .withIndex('by_creation_time', (q) => + cursor !== undefined ? q.gt('_creationTime', cursor) : q, + ) + .take(limit) + return events + }, +}) + +/** + * Get the current cursor position from the cursors table. + */ +export const getStatEventCursor = internalQuery({ + args: {}, + handler: async (ctx) => { + const cursor = await ctx.db + .query('skillStatUpdateCursors') + .withIndex('by_key', (q) => q.eq('key', CURSOR_KEY)) + .unique() + return cursor?.cursorCreationTime + }, +}) + +/** + * Validator for skill deltas passed to the mutation. + */ +const skillDeltaValidator = v.object({ + skillId: v.id('skills'), + downloads: v.number(), + stars: v.number(), + installsAllTime: v.number(), + installsCurrent: v.number(), + downloadEvents: v.array(v.number()), + installNewEvents: v.array(v.number()), +}) + +/** + * Apply aggregated stats to skills and update the cursor. + * This is a single atomic mutation that: + * 1. Updates all affected skills with their aggregated deltas + * 2. Updates daily stats for trending + * 3. Advances the cursor to the new position + */ +export const applyAggregatedStatsAndUpdateCursor = internalMutation({ + args: { + skillDeltas: v.array(skillDeltaValidator), + newCursor: v.number(), + }, + handler: async (ctx, args) => { + const now = Date.now() + + // Process each skill's aggregated deltas + for (const delta of args.skillDeltas) { + const skill = await ctx.db.get(delta.skillId) + + // Skill was deleted - skip + if (!skill) { + continue + } + + // Apply aggregated deltas to skill stats + if ( + delta.downloads !== 0 || + delta.stars !== 0 || + delta.installsAllTime !== 0 || + delta.installsCurrent !== 0 + ) { + const patch = applySkillStatDeltas(skill, { + downloads: delta.downloads, + stars: delta.stars, + installsAllTime: delta.installsAllTime, + installsCurrent: delta.installsCurrent, + }) + await ctx.db.patch(skill._id, { + ...patch, + updatedAt: now, + }) + } + + // Update daily stats for trending/leaderboards + for (const occurredAt of delta.downloadEvents) { + await bumpDailySkillStats(ctx, { skillId: delta.skillId, now: occurredAt, downloads: 1 }) + } + for (const occurredAt of delta.installNewEvents) { + await bumpDailySkillStats(ctx, { skillId: delta.skillId, now: occurredAt, installs: 1 }) + } + } + + // Update cursor position (upsert) + const existingCursor = await ctx.db + .query('skillStatUpdateCursors') + .withIndex('by_key', (q) => q.eq('key', CURSOR_KEY)) + .unique() + + if (existingCursor) { + await ctx.db.patch(existingCursor._id, { + cursorCreationTime: args.newCursor, + updatedAt: now, + }) + } else { + await ctx.db.insert('skillStatUpdateCursors', { + key: CURSOR_KEY, + cursorCreationTime: args.newCursor, + updatedAt: now, + }) + } + + return { skillsUpdated: args.skillDeltas.length } + }, +}) + +/** + * Action that processes skill stat events in batches outside the transaction window. + * + * Algorithm: + * 1. Get current cursor position + * 2. Fetch events in batches of 500, aggregating as we go + * 3. Stop when we have >= 500 unique skills OR run out of events + * 4. Call mutation to apply all deltas and update cursor atomically + * 5. Self-schedule if we stopped due to skill limit (not exhaustion) + */ +export const processSkillStatEventsAction = internalAction({ + args: {}, + handler: async (ctx) => { + // Get current cursor position (convert null to undefined for consistency) + const cursorResult = await ctx.runQuery(internal.skillStatEvents.getStatEventCursor) + let cursor: number | undefined = cursorResult ?? undefined + + console.log(`[STAT-AGG] Starting aggregation, cursor=${cursor ?? 'none'}`) + + // Aggregated deltas per skill + const aggregatedBySkill = new Map< + Id<'skills'>, + { + downloads: number + stars: number + installsAllTime: number + installsCurrent: number + downloadEvents: number[] + installNewEvents: number[] + } + >() + + let maxCreationTime: number | undefined = cursor + let exhausted = false + let totalEventsFetched = 0 + + // Fetch and aggregate until we have enough skills or run out of events + while (aggregatedBySkill.size < MAX_SKILLS_PER_RUN) { + const events = await ctx.runQuery(internal.skillStatEvents.getUnprocessedEventBatch, { + cursorCreationTime: cursor, + limit: EVENT_BATCH_SIZE, + }) + + if (events.length === 0) { + exhausted = true + break + } + + totalEventsFetched += events.length + const skillsBefore = aggregatedBySkill.size + + // Aggregate events into per-skill deltas + for (const event of events) { + let skillDelta = aggregatedBySkill.get(event.skillId) + if (!skillDelta) { + skillDelta = { + downloads: 0, + stars: 0, + installsAllTime: 0, + installsCurrent: 0, + downloadEvents: [], + installNewEvents: [], + } + aggregatedBySkill.set(event.skillId, skillDelta) + } + + // Apply event to aggregated deltas + switch (event.kind) { + case 'download': + skillDelta.downloads += 1 + skillDelta.downloadEvents.push(event.occurredAt) + break + case 'star': + skillDelta.stars += 1 + break + case 'unstar': + skillDelta.stars -= 1 + break + case 'install_new': + skillDelta.installsAllTime += 1 + skillDelta.installsCurrent += 1 + skillDelta.installNewEvents.push(event.occurredAt) + break + case 'install_reactivate': + skillDelta.installsCurrent += 1 + break + case 'install_deactivate': + skillDelta.installsCurrent -= 1 + break + case 'install_clear': + if (event.delta) { + skillDelta.installsAllTime += event.delta.allTime + skillDelta.installsCurrent += event.delta.current + } + break + } + + // Track highest _creationTime seen + if (maxCreationTime === undefined || event._creationTime > maxCreationTime) { + maxCreationTime = event._creationTime + } + } + + // Update cursor for next batch fetch + cursor = events[events.length - 1]._creationTime + + console.log( + `[STAT-AGG] Fetched ${events.length} events, ${aggregatedBySkill.size - skillsBefore} new skills (${aggregatedBySkill.size} total)`, + ) + + // If we got fewer than requested, we've exhausted the events + if (events.length < EVENT_BATCH_SIZE) { + exhausted = true + break + } + } + + // If we have nothing to process, we're done + if (aggregatedBySkill.size === 0 || maxCreationTime === undefined) { + console.log('[STAT-AGG] No events to process, done') + return { processed: 0, skillsUpdated: 0, exhausted: true } + } + + // Convert map to array for mutation + const skillDeltas = Array.from(aggregatedBySkill.entries()).map(([skillId, delta]) => ({ + skillId, + ...delta, + })) + + console.log( + `[STAT-AGG] Running mutation for ${skillDeltas.length} skills (${totalEventsFetched} total events)`, + ) + + // Apply all deltas and update cursor atomically + await ctx.runMutation(internal.skillStatEvents.applyAggregatedStatsAndUpdateCursor, { + skillDeltas, + newCursor: maxCreationTime, + }) + + // Self-schedule if we stopped because of skill limit, not exhaustion + if (!exhausted) { + console.log('[STAT-AGG] More events remaining, self-scheduling') + await ctx.scheduler.runAfter(0, internal.skillStatEvents.processSkillStatEventsAction, {}) + } else { + console.log('[STAT-AGG] All events processed, done') + } + + return { + skillsUpdated: skillDeltas.length, + exhausted, + } + }, +}) diff --git a/src/components/Header.tsx b/src/components/Header.tsx index 76cab5c..37c0b73 100644 --- a/src/components/Header.tsx +++ b/src/components/Header.tsx @@ -63,7 +63,13 @@ export default function Header() { {isSoulMode ? ( Souls @@ -76,6 +82,7 @@ export default function Header() { dir: undefined, highlighted: undefined, view: undefined, + focus: undefined, }} > Skills @@ -89,13 +96,20 @@ export default function Header() { to={isSoulMode ? '/souls' : '/skills'} search={ isSoulMode - ? { q: undefined, sort: undefined, dir: undefined, view: undefined } + ? { + q: undefined, + sort: undefined, + dir: undefined, + view: undefined, + focus: 'search', + } : { q: undefined, sort: undefined, dir: undefined, highlighted: undefined, view: undefined, + focus: 'search', } } > @@ -126,7 +140,13 @@ export default function Header() { {isSoulMode ? ( Souls @@ -139,6 +159,7 @@ export default function Header() { dir: undefined, highlighted: undefined, view: undefined, + focus: undefined, }} > Skills @@ -160,13 +181,20 @@ export default function Header() { to={isSoulMode ? '/souls' : '/skills'} search={ isSoulMode - ? { q: undefined, sort: undefined, dir: undefined, view: undefined } + ? { + q: undefined, + sort: undefined, + dir: undefined, + view: undefined, + focus: 'search', + } : { q: undefined, sort: undefined, dir: undefined, highlighted: undefined, view: undefined, + focus: 'search', } } > diff --git a/src/routeTree.gen.ts b/src/routeTree.gen.ts index 84702e8..bc0e5b6 100644 --- a/src/routeTree.gen.ts +++ b/src/routeTree.gen.ts @@ -12,10 +12,10 @@ import { Route as rootRouteImport } from './routes/__root' import { Route as UploadRouteImport } from './routes/upload' import { Route as StarsRouteImport } from './routes/stars' import { Route as SettingsRouteImport } from './routes/settings' +import { Route as ManagementRouteImport } from './routes/management' import { Route as ImportRouteImport } from './routes/import' import { Route as DashboardRouteImport } from './routes/dashboard' import { Route as AdminRouteImport } from './routes/admin' -import { Route as ManagementRouteImport } from './routes/management' import { Route as IndexRouteImport } from './routes/index' import { Route as SoulsIndexRouteImport } from './routes/souls/index' import { Route as SkillsIndexRouteImport } from './routes/skills/index' @@ -39,6 +39,11 @@ const SettingsRoute = SettingsRouteImport.update({ path: '/settings', getParentRoute: () => rootRouteImport, } as any) +const ManagementRoute = ManagementRouteImport.update({ + id: '/management', + path: '/management', + getParentRoute: () => rootRouteImport, +} as any) const ImportRoute = ImportRouteImport.update({ id: '/import', path: '/import', @@ -54,11 +59,6 @@ const AdminRoute = AdminRouteImport.update({ path: '/admin', getParentRoute: () => rootRouteImport, } as any) -const ManagementRoute = ManagementRouteImport.update({ - id: '/management', - path: '/management', - getParentRoute: () => rootRouteImport, -} as any) const IndexRoute = IndexRouteImport.update({ id: '/', path: '/', @@ -198,9 +198,9 @@ export interface FileRouteTypes { export interface RootRouteChildren { IndexRoute: typeof IndexRoute AdminRoute: typeof AdminRoute - ManagementRoute: typeof ManagementRoute DashboardRoute: typeof DashboardRoute ImportRoute: typeof ImportRoute + ManagementRoute: typeof ManagementRoute SettingsRoute: typeof SettingsRoute StarsRoute: typeof StarsRoute UploadRoute: typeof UploadRoute @@ -235,6 +235,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof SettingsRouteImport parentRoute: typeof rootRouteImport } + '/management': { + id: '/management' + path: '/management' + fullPath: '/management' + preLoaderRoute: typeof ManagementRouteImport + parentRoute: typeof rootRouteImport + } '/import': { id: '/import' path: '/import' @@ -256,13 +263,6 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AdminRouteImport parentRoute: typeof rootRouteImport } - '/management': { - id: '/management' - path: '/management' - fullPath: '/management' - preLoaderRoute: typeof ManagementRouteImport - parentRoute: typeof rootRouteImport - } '/': { id: '/' path: '/' @@ -318,9 +318,9 @@ declare module '@tanstack/react-router' { const rootRouteChildren: RootRouteChildren = { IndexRoute: IndexRoute, AdminRoute: AdminRoute, - ManagementRoute: ManagementRoute, DashboardRoute: DashboardRoute, ImportRoute: ImportRoute, + ManagementRoute: ManagementRoute, SettingsRoute: SettingsRoute, StarsRoute: StarsRoute, UploadRoute: UploadRoute, diff --git a/src/routes/index.tsx b/src/routes/index.tsx index 4a3dde4..b3087ba 100644 --- a/src/routes/index.tsx +++ b/src/routes/index.tsx @@ -20,7 +20,10 @@ function Home() { function SkillsHome() { const highlighted = - (useQuery(api.skills.list, { batch: 'highlighted', limit: 6 }) as PublicSkill[]) ?? [] + (useQuery(api.skills.list, { + batch: 'highlighted', + limit: 6, + }) as PublicSkill[]) ?? [] const latest = (useQuery(api.skills.list, { limit: 12 }) as PublicSkill[]) ?? [] return ( @@ -46,6 +49,7 @@ function SkillsHome() { dir: undefined, highlighted: undefined, view: undefined, + focus: undefined, }} className="btn" > @@ -118,6 +122,7 @@ function SkillsHome() { dir: undefined, highlighted: undefined, view: undefined, + focus: undefined, }} className="btn" > @@ -160,7 +165,13 @@ function OnlyCrabsHome() { Browse souls @@ -179,6 +190,7 @@ function OnlyCrabsHome() { sort: undefined, dir: undefined, view: undefined, + focus: undefined, }, }) }} @@ -222,7 +234,13 @@ function OnlyCrabsHome() {
See all souls diff --git a/src/routes/skills/index.tsx b/src/routes/skills/index.tsx index a153653..a000538 100644 --- a/src/routes/skills/index.tsx +++ b/src/routes/skills/index.tsx @@ -53,6 +53,7 @@ export const Route = createFileRoute('/skills/')({ ? true : undefined, view: search.view === 'cards' || search.view === 'list' ? search.view : undefined, + focus: search.focus === 'search' ? 'search' : undefined, } }, component: SkillsIndex, @@ -73,6 +74,7 @@ export function SkillsIndex() { const searchRequest = useRef(0) const loadMoreRef = useRef(null) + const searchInputRef = useRef(null) const trimmedQuery = useMemo(() => query.trim(), [query]) const hasQuery = trimmedQuery.length > 0 const searchKey = trimmedQuery ? `${trimmedQuery}::${highlightedOnly ? '1' : '0'}` : '' @@ -96,6 +98,15 @@ export function SkillsIndex() { setQuery(search.q ?? '') }, [search.q]) + // Auto-focus search input when focus=search param is present + useEffect(() => { + if (search.focus === 'search' && searchInputRef.current) { + searchInputRef.current.focus() + // Clear the focus param from URL to avoid re-focusing on navigation + void navigate({ search: (prev) => ({ ...prev, focus: undefined }), replace: true }) + } + }, [search.focus, navigate]) + useEffect(() => { if (!searchKey) { setSearchResults([]) @@ -225,6 +236,7 @@ export function SkillsIndex() {
{ diff --git a/src/routes/souls/index.tsx b/src/routes/souls/index.tsx index 31717e1..4491060 100644 --- a/src/routes/souls/index.tsx +++ b/src/routes/souls/index.tsx @@ -27,6 +27,7 @@ export const Route = createFileRoute('/souls/')({ sort: typeof search.sort === 'string' ? parseSort(search.sort) : undefined, dir: search.dir === 'asc' || search.dir === 'desc' ? search.dir : undefined, view: search.view === 'cards' || search.view === 'list' ? search.view : undefined, + focus: search.focus === 'search' ? 'search' : undefined, } }, component: SoulsIndex, @@ -43,12 +44,22 @@ function SoulsIndex() { const souls = useQuery(api.souls.list, { limit: 500 }) as PublicSoul[] | undefined const ensureSoulSeeds = useAction(api.seed.ensureSoulSeeds) const seedEnsuredRef = useRef(false) + const searchInputRef = useRef(null) const isLoadingSouls = souls === undefined useEffect(() => { setQuery(search.q ?? '') }, [search.q]) + // Auto-focus search input when focus=search param is present + useEffect(() => { + if (search.focus === 'search' && searchInputRef.current) { + searchInputRef.current.focus() + // Clear the focus param from URL to avoid re-focusing on navigation + void navigate({ search: (prev) => ({ ...prev, focus: undefined }), replace: true }) + } + }, [search.focus, navigate]) + useEffect(() => { if (seedEnsuredRef.current) return seedEnsuredRef.current = true @@ -108,6 +119,7 @@ function SoulsIndex() {
{