diff --git a/plugin-server/src/config/config.ts b/plugin-server/src/config/config.ts index 5dc2e9bad119a..a263c2cbf147a 100644 --- a/plugin-server/src/config/config.ts +++ b/plugin-server/src/config/config.ts @@ -142,6 +142,10 @@ export function getDefaultConfig(): PluginsServerConfig { HOG_HOOK_URL: '', CAPTURE_CONFIG_REDIS_HOST: null, + // posthog + POSTHOG_API_KEY: '', + POSTHOG_HOST_URL: 'http://localhost:8010', + STARTUP_PROFILE_DURATION_SECONDS: 300, // 5 minutes STARTUP_PROFILE_CPU: false, STARTUP_PROFILE_HEAP: false, diff --git a/plugin-server/src/main/pluginsServer.ts b/plugin-server/src/main/pluginsServer.ts index 119715add0266..122bd9fa1fb92 100644 --- a/plugin-server/src/main/pluginsServer.ts +++ b/plugin-server/src/main/pluginsServer.ts @@ -27,7 +27,8 @@ import { closeHub, createHub, createKafkaClient } from '../utils/db/hub' import { PostgresRouter } from '../utils/db/postgres' import { createRedisClient } from '../utils/db/redis' import { cancelAllScheduledJobs } from '../utils/node-schedule' -import { captureException, posthog } from '../utils/posthog' +import { captureException } from '../utils/posthog' +import { flush as posthogFlush, shutdown as posthogShutdown } from '../utils/posthog' import { PubSub } from '../utils/pubsub' import { status } from '../utils/status' import { delay } from '../utils/utils' @@ -132,7 +133,7 @@ export async function startPluginsServer( pubSub?.stop(), graphileWorker?.stop(), ...services.map((service) => service.onShutdown()), - posthog.shutdown(), + posthogShutdown(), ]) if (serverInstance.hub) { @@ -390,7 +391,7 @@ export async function startPluginsServer( // we need to create them. We only initialize the ones we need. const postgres = hub?.postgres ?? new PostgresRouter(serverConfig) const kafka = hub?.kafka ?? createKafkaClient(serverConfig) - const teamManager = hub?.teamManager ?? new TeamManager(postgres, serverConfig) + const teamManager = hub?.teamManager ?? new TeamManager(postgres) const organizationManager = hub?.organizationManager ?? new OrganizationManager(postgres, teamManager) const kafkaProducerWrapper = hub?.kafkaProducer ?? (await KafkaProducerWrapper.create(serverConfig)) const rustyHook = hub?.rustyHook ?? new RustyHook(serverConfig) @@ -403,7 +404,7 @@ export async function startPluginsServer( ) const actionManager = hub?.actionManager ?? new ActionManager(postgres, serverConfig) - const actionMatcher = hub?.actionMatcher ?? new ActionMatcher(postgres, actionManager, teamManager) + const actionMatcher = hub?.actionMatcher ?? new ActionMatcher(postgres, actionManager) const groupTypeManager = new GroupTypeManager(postgres, teamManager, serverConfig.SITE_URL) services.push( @@ -610,7 +611,7 @@ export async function startPluginsServer( captureException(error) status.error('💥', 'Launchpad failure!', { error: error.stack ?? error }) void Sentry.flush().catch(() => null) // Flush Sentry in the background - void posthog.flush().catch(() => null) + posthogFlush() status.error('💥', 'Exception while starting server, shutting down!', { error }) await closeJobs() process.exit(1) diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts index c2f90a85ec666..0bb9cb5de2540 100644 --- a/plugin-server/src/types.ts +++ b/plugin-server/src/types.ts @@ -323,6 +323,10 @@ export interface PluginsServerConfig extends CdpConfig, IngestionConsumerConfig CYCLOTRON_DATABASE_URL: string CYCLOTRON_SHARD_DEPTH_LIMIT: number + // posthog + POSTHOG_API_KEY: string + POSTHOG_HOST_URL: string + // cookieless COOKIELESS_DISABLED: boolean COOKIELESS_FORCE_STATELESS_MODE: boolean diff --git a/plugin-server/src/utils/db/hub.ts b/plugin-server/src/utils/db/hub.ts index 6d64baa2f4c70..373333ce7c7ba 100644 --- a/plugin-server/src/utils/db/hub.ts +++ b/plugin-server/src/utils/db/hub.ts @@ -133,14 +133,14 @@ export async function createHub( serverConfig.PLUGINS_DEFAULT_LOG_LEVEL, serverConfig.PERSON_INFO_CACHE_TTL ) - const teamManager = new TeamManager(postgres, serverConfig) + const teamManager = new TeamManager(postgres) const organizationManager = new OrganizationManager(postgres, teamManager) const pluginsApiKeyManager = new PluginsApiKeyManager(db) const rootAccessManager = new RootAccessManager(db) const rustyHook = new RustyHook(serverConfig) const actionManager = new ActionManager(postgres, serverConfig) - const actionMatcher = new ActionMatcher(postgres, actionManager, teamManager) + const actionMatcher = new ActionMatcher(postgres, actionManager) const groupTypeManager = new GroupTypeManager(postgres, teamManager) const cookielessManager = new CookielessManager(serverConfig, redisPool, teamManager) diff --git a/plugin-server/src/utils/posthog.ts b/plugin-server/src/utils/posthog.ts index fc32dd1248fc6..4d47bf0e9d242 100644 --- a/plugin-server/src/utils/posthog.ts +++ b/plugin-server/src/utils/posthog.ts @@ -2,37 +2,57 @@ import { captureException as captureSentryException, captureMessage as captureSe import { PostHog } from 'posthog-node' import { SeverityLevel } from 'posthog-node/src/extensions/error-tracking/types' +import { defaultConfig } from '../config/config' import { Team } from '../types' -export const posthog = new PostHog('sTMFPsFhdP1Ssg', { - host: 'https://us.i.posthog.com', - enableExceptionAutocapture: false, // TODO - disabled while data volume is a problem, PS seems /extremely/ chatty exceptions wise -}) +const posthog = defaultConfig.POSTHOG_API_KEY + ? new PostHog(defaultConfig.POSTHOG_API_KEY, { + host: defaultConfig.POSTHOG_HOST_URL, + enableExceptionAutocapture: false, // TODO - disabled while data volume is a problem, PS seems /extremely/ chatty exceptions wise + }) + : null -if (process.env.NODE_ENV === 'test') { +if (process.env.NODE_ENV === 'test' && posthog) { void posthog.disable() } -export const captureTeamEvent = (team: Team, event: string, properties: Record = {}): void => { - posthog.capture({ - distinctId: team.uuid, - event, - properties: { - team: team.uuid, - ...properties, - }, - groups: { - project: team.uuid, - organization: team.organization_id, - instance: process.env.SITE_URL ?? 'unknown', - }, - }) +export function captureTeamEvent( + team: Team, + event: string, + properties: Record = {}, + distinctId: string | null = null +): void { + if (posthog) { + posthog.capture({ + distinctId: distinctId ?? team.uuid, + event, + properties: { + team: team.uuid, + ...properties, + }, + groups: { + project: team.uuid, + organization: team.organization_id, + instance: process.env.SITE_URL ?? 'unknown', + }, + }) + } +} + +export function shutdown(): Promise | null { + return posthog ? posthog.shutdown() : null +} + +export function flush(): void { + if (posthog) { + void posthog.flush().catch(() => null) + } } // We use sentry-style hints rather than our flat property list all over the place, // so define a type for them that we can flatten internally -export type Primitive = number | string | boolean | bigint | symbol | null | undefined -export interface ExceptionHint { +type Primitive = number | string | boolean | bigint | symbol | null | undefined +interface ExceptionHint { level: SeverityLevel tags: Record extra: Record @@ -47,8 +67,7 @@ export function captureException(exception: any, hint?: Partial): sentryId = captureSentryException(exception, hint) } - // TODO - this sampling is a hack while we work on our data consumption in error tracking - if (process.env.NODE_ENV === 'production' && Math.random() < 0.1) { + if (posthog) { let additionalProperties = {} if (hint) { additionalProperties = { diff --git a/plugin-server/src/worker/ingestion/action-matcher.ts b/plugin-server/src/worker/ingestion/action-matcher.ts index 752eb52abb84a..fc49559b75592 100644 --- a/plugin-server/src/worker/ingestion/action-matcher.ts +++ b/plugin-server/src/worker/ingestion/action-matcher.ts @@ -24,7 +24,6 @@ import { mutatePostIngestionEventWithElementsList } from '../../utils/event' import { captureException } from '../../utils/posthog' import { stringify } from '../../utils/utils' import { ActionManager } from './action-manager' -import { TeamManager } from './team-manager' /** These operators can only be matched if the provided filter's value has the right type. */ const propertyOperatorToRequiredValueType: Partial> = { @@ -133,11 +132,7 @@ export function matchString(actual: string, expected: string, matching: StringMa } export class ActionMatcher { - constructor( - private postgres: PostgresRouter, - private actionManager: ActionManager, - private teamManager: TeamManager - ) {} + constructor(private postgres: PostgresRouter, private actionManager: ActionManager) {} public hasWebhooks(teamId: number): boolean { return Object.keys(this.actionManager.getTeamActions(teamId)).length > 0 diff --git a/plugin-server/src/worker/ingestion/team-manager.ts b/plugin-server/src/worker/ingestion/team-manager.ts index d787c50c6c948..b51e380dd506d 100644 --- a/plugin-server/src/worker/ingestion/team-manager.ts +++ b/plugin-server/src/worker/ingestion/team-manager.ts @@ -3,18 +3,17 @@ import LRU from 'lru-cache' import { ONE_MINUTE } from '../../config/constants' import { TeamIDWithConfig } from '../../main/ingestion-queues/session-recording/session-recordings-consumer' -import { PipelineEvent, PluginsServerConfig, ProjectId, Team, TeamId } from '../../types' +import { PipelineEvent, ProjectId, Team, TeamId } from '../../types' import { PostgresRouter, PostgresUse } from '../../utils/db/postgres' import { timeoutGuard } from '../../utils/db/utils' -import { posthog } from '../../utils/posthog' +import { captureTeamEvent } from '../../utils/posthog' export class TeamManager { postgres: PostgresRouter teamCache: LRU tokenToTeamIdCache: LRU - instanceSiteUrl: string - constructor(postgres: PostgresRouter, serverConfig: PluginsServerConfig) { + constructor(postgres: PostgresRouter) { this.postgres = postgres this.teamCache = new LRU({ @@ -27,7 +26,6 @@ export class TeamManager { maxAge: 5 * ONE_MINUTE, // Expiration for negative lookups, positive lookups will expire via teamCache first updateAgeOnGet: false, // Make default behaviour explicit }) - this.instanceSiteUrl = serverConfig.SITE_URL || 'unknown' } public async getTeamForEvent(event: PipelineEvent): Promise { @@ -132,21 +130,16 @@ export class TeamManager { ) const distinctIds: { distinct_id: string }[] = organizationMembers.rows for (const { distinct_id } of distinctIds) { - posthog.capture({ - distinctId: distinct_id, - event: 'first team event ingested', - properties: { - team: team.uuid, + captureTeamEvent( + team, + 'first team event ingested', + { sdk: properties.$lib, realm: properties.realm, host: properties.$host, }, - groups: { - project: team.uuid, - organization: team.organization_id, - instance: this.instanceSiteUrl, - }, - }) + distinct_id + ) } } } diff --git a/plugin-server/tests/main/ingestion-queues/each-batch-webhooks.test.ts b/plugin-server/tests/main/ingestion-queues/each-batch-webhooks.test.ts index cd4317e4efccd..7b47297cf0bff 100644 --- a/plugin-server/tests/main/ingestion-queues/each-batch-webhooks.test.ts +++ b/plugin-server/tests/main/ingestion-queues/each-batch-webhooks.test.ts @@ -84,7 +84,7 @@ describe('eachMessageWebhooksHandlers', () => { it('calls runWebhooksHandlersEventPipeline', async () => { const actionManager = new ActionManager(hub.postgres, hub) - const actionMatcher = new ActionMatcher(hub.postgres, actionManager, hub.teamManager) + const actionMatcher = new ActionMatcher(hub.postgres, actionManager) const hookCannon = new HookCommander( hub.postgres, hub.teamManager, diff --git a/plugin-server/tests/main/ingestion-queues/each-batch.test.ts b/plugin-server/tests/main/ingestion-queues/each-batch.test.ts index 0f2d7cc3b1807..4b0546dbc324b 100644 --- a/plugin-server/tests/main/ingestion-queues/each-batch.test.ts +++ b/plugin-server/tests/main/ingestion-queues/each-batch.test.ts @@ -172,11 +172,7 @@ describe('eachBatchX', () => { describe('eachBatchWebhooksHandlers', () => { it('calls runWebhooksHandlersEventPipeline', async () => { const actionManager = new ActionManager(queue.pluginsServer.postgres, queue.pluginsServer) - const actionMatcher = new ActionMatcher( - queue.pluginsServer.postgres, - actionManager, - queue.pluginsServer.teamManager - ) + const actionMatcher = new ActionMatcher(queue.pluginsServer.postgres, actionManager) const hookCannon = new HookCommander( queue.pluginsServer.postgres, queue.pluginsServer.teamManager, diff --git a/plugin-server/tests/main/process-event.test.ts b/plugin-server/tests/main/process-event.test.ts index 481262fb9528b..fc40789043fe9 100644 --- a/plugin-server/tests/main/process-event.test.ts +++ b/plugin-server/tests/main/process-event.test.ts @@ -10,6 +10,8 @@ import { PluginEvent } from '@posthog/plugin-scaffold/src/types' import * as IORedis from 'ioredis' import { DateTime } from 'luxon' +import { captureTeamEvent } from '~/src/utils/posthog' + import { KAFKA_EVENTS_PLUGIN_INGESTION } from '../../src/config/kafka-topics' import { ClickHouseEvent, @@ -24,7 +26,6 @@ import { import { closeHub, createHub } from '../../src/utils/db/hub' import { PostgresUse } from '../../src/utils/db/postgres' import { personInitialAndUTMProperties } from '../../src/utils/db/utils' -import { posthog } from '../../src/utils/posthog' import { UUIDT } from '../../src/utils/utils' import { EventPipelineRunner } from '../../src/worker/ingestion/event-pipeline/runner' import { EventsProcessor } from '../../src/worker/ingestion/process-event' @@ -34,6 +35,10 @@ import { createUserTeamAndOrganization, getFirstTeam, getTeams, resetTestDatabas jest.mock('../../src/utils/status') jest.setTimeout(600000) // 600 sec timeout. +jest.mock('../../src/utils/posthog', () => ({ + ...jest.requireActual('../../src/utils/posthog'), + captureTeamEvent: jest.fn(), +})) export async function createPerson( server: Hub, @@ -880,9 +885,6 @@ test('capture first team event', async () => { 'testTag' ) - posthog.capture = jest.fn() as any - posthog.identify = jest.fn() as any - await processEvent( '2', '', @@ -900,18 +902,12 @@ test('capture first team event', async () => { new UUIDT().toString() ) - expect(posthog.capture).toHaveBeenCalledWith({ - distinctId: 'plugin_test_user_distinct_id_1001', - event: 'first team event ingested', - properties: { - team: team.uuid, - }, - groups: { - project: team.uuid, - organization: team.organization_id, - instance: 'unknown', - }, - }) + expect(captureTeamEvent).toHaveBeenCalledWith( + expect.objectContaining({ uuid: team.uuid, organization_id: team.organization_id }), + 'first team event ingested', + { host: undefined, realm: undefined, sdk: undefined }, + 'plugin_test_user_distinct_id_1001' + ) team = await getFirstTeam(hub) expect(team.ingested_event).toEqual(true) diff --git a/plugin-server/tests/worker/ingestion/action-matcher.test.ts b/plugin-server/tests/worker/ingestion/action-matcher.test.ts index 7bf6fa15ea660..ede5e7d0fea83 100644 --- a/plugin-server/tests/worker/ingestion/action-matcher.test.ts +++ b/plugin-server/tests/worker/ingestion/action-matcher.test.ts @@ -54,7 +54,7 @@ describe('ActionMatcher', () => { hub = await createHub() actionManager = new ActionManager(hub.db.postgres, hub) await actionManager.start() - actionMatcher = new ActionMatcher(hub.db.postgres, actionManager, hub.teamManager) + actionMatcher = new ActionMatcher(hub.db.postgres, actionManager) actionCounter = 0 }) diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/event-pipeline-integration.test.ts b/plugin-server/tests/worker/ingestion/event-pipeline/event-pipeline-integration.test.ts index b3838ef300a44..808594f819efb 100644 --- a/plugin-server/tests/worker/ingestion/event-pipeline/event-pipeline-integration.test.ts +++ b/plugin-server/tests/worker/ingestion/event-pipeline/event-pipeline-integration.test.ts @@ -46,7 +46,7 @@ describe('Event Pipeline integration test', () => { actionManager = new ActionManager(hub.db.postgres, hub) await actionManager.start() - actionMatcher = new ActionMatcher(hub.db.postgres, actionManager, hub.teamManager) + actionMatcher = new ActionMatcher(hub.db.postgres, actionManager) hookCannon = new HookCommander( hub.db.postgres, hub.teamManager, diff --git a/plugin-server/tests/worker/ingestion/team-manager.test.ts b/plugin-server/tests/worker/ingestion/team-manager.test.ts index 94d31e7e0b60f..a201dfdf63b15 100644 --- a/plugin-server/tests/worker/ingestion/team-manager.test.ts +++ b/plugin-server/tests/worker/ingestion/team-manager.test.ts @@ -6,12 +6,6 @@ import { TeamManager } from '../../../src/worker/ingestion/team-manager' import { resetTestDatabase } from '../../helpers/sql' jest.mock('../../../src/utils/status') -jest.mock('../../../src/utils/posthog', () => ({ - posthog: { - identify: jest.fn(), - capture: jest.fn(), - }, -})) describe('TeamManager()', () => { let teamManager: TeamManager @@ -20,7 +14,7 @@ describe('TeamManager()', () => { beforeEach(async () => { await resetTestDatabase() postgres = new PostgresRouter(defaultConfig) - teamManager = new TeamManager(postgres, defaultConfig) + teamManager = new TeamManager(postgres) // @ts-expect-error TODO: Fix underlying settings, is this really working? Settings.defaultZoneName = 'utc' diff --git a/plugin-server/tests/worker/plugins/run.test.ts b/plugin-server/tests/worker/plugins/run.test.ts index 291835b406871..88a70cc50a842 100644 --- a/plugin-server/tests/worker/plugins/run.test.ts +++ b/plugin-server/tests/worker/plugins/run.test.ts @@ -279,7 +279,7 @@ describe('runComposeWebhook', () => { queueMetric: jest.fn(), queueError: jest.fn(), } as any, - actionMatcher: new ActionMatcher(mockPostgres, mockActionManager, {} as any), + actionMatcher: new ActionMatcher(mockPostgres, mockActionManager), } })