chore: disable plugin server posthog capture if not set

plugin-server/src/config/config.ts

@@ -142,6 +142,10 @@ export function getDefaultConfig(): PluginsServerConfig {
         HOG_HOOK_URL: '',
         CAPTURE_CONFIG_REDIS_HOST: null,
 
+        // posthog
+        POSTHOG_API_KEY: '',
+        POSTHOG_HOST_URL: 'http://localhost:8010',
+
         STARTUP_PROFILE_DURATION_SECONDS: 300, // 5 minutes
         STARTUP_PROFILE_CPU: false,
         STARTUP_PROFILE_HEAP: false,

plugin-server/src/main/pluginsServer.ts

@@ -27,7 +27,8 @@ import { closeHub, createHub, createKafkaClient } from '../utils/db/hub'
 import { PostgresRouter } from '../utils/db/postgres'
 import { createRedisClient } from '../utils/db/redis'
 import { cancelAllScheduledJobs } from '../utils/node-schedule'
-import { captureException, posthog } from '../utils/posthog'
+import { captureException } from '../utils/posthog'
+import { flush as posthogFlush, shutdown as posthogShutdown } from '../utils/posthog'
 import { PubSub } from '../utils/pubsub'
 import { status } from '../utils/status'
 import { delay } from '../utils/utils'
@@ -132,7 +133,7 @@ export async function startPluginsServer(
             pubSub?.stop(),
             graphileWorker?.stop(),
             ...services.map((service) => service.onShutdown()),
-            posthog.shutdown(),
+            posthogShutdown(),
         ])
 
         if (serverInstance.hub) {
@@ -390,7 +391,7 @@ export async function startPluginsServer(
             // we need to create them. We only initialize the ones we need.
             const postgres = hub?.postgres ?? new PostgresRouter(serverConfig)
             const kafka = hub?.kafka ?? createKafkaClient(serverConfig)
-            const teamManager = hub?.teamManager ?? new TeamManager(postgres, serverConfig)
+            const teamManager = hub?.teamManager ?? new TeamManager(postgres)
             const organizationManager = hub?.organizationManager ?? new OrganizationManager(postgres, teamManager)
             const kafkaProducerWrapper = hub?.kafkaProducer ?? (await KafkaProducerWrapper.create(serverConfig))
             const rustyHook = hub?.rustyHook ?? new RustyHook(serverConfig)
@@ -403,7 +404,7 @@ export async function startPluginsServer(
                 )
 
             const actionManager = hub?.actionManager ?? new ActionManager(postgres, serverConfig)
-            const actionMatcher = hub?.actionMatcher ?? new ActionMatcher(postgres, actionManager, teamManager)
+            const actionMatcher = hub?.actionMatcher ?? new ActionMatcher(postgres, actionManager)
             const groupTypeManager = new GroupTypeManager(postgres, teamManager, serverConfig.SITE_URL)
 
             services.push(
@@ -610,7 +611,7 @@ export async function startPluginsServer(
         captureException(error)
         status.error('💥', 'Launchpad failure!', { error: error.stack ?? error })
         void Sentry.flush().catch(() => null) // Flush Sentry in the background
-        void posthog.flush().catch(() => null)
+        posthogFlush()
         status.error('💥', 'Exception while starting server, shutting down!', { error })
         await closeJobs()
         process.exit(1)

plugin-server/src/types.ts

@@ -323,6 +323,10 @@ export interface PluginsServerConfig extends CdpConfig, IngestionConsumerConfig
     CYCLOTRON_DATABASE_URL: string
     CYCLOTRON_SHARD_DEPTH_LIMIT: number
 
+    // posthog
+    POSTHOG_API_KEY: string
+    POSTHOG_HOST_URL: string
+
     // cookieless
     COOKIELESS_DISABLED: boolean
     COOKIELESS_FORCE_STATELESS_MODE: boolean

plugin-server/src/utils/db/hub.ts

@@ -133,14 +133,14 @@ export async function createHub(
         serverConfig.PLUGINS_DEFAULT_LOG_LEVEL,
         serverConfig.PERSON_INFO_CACHE_TTL
     )
-    const teamManager = new TeamManager(postgres, serverConfig)
+    const teamManager = new TeamManager(postgres)
     const organizationManager = new OrganizationManager(postgres, teamManager)
     const pluginsApiKeyManager = new PluginsApiKeyManager(db)
     const rootAccessManager = new RootAccessManager(db)
     const rustyHook = new RustyHook(serverConfig)
 
     const actionManager = new ActionManager(postgres, serverConfig)
-    const actionMatcher = new ActionMatcher(postgres, actionManager, teamManager)
+    const actionMatcher = new ActionMatcher(postgres, actionManager)
     const groupTypeManager = new GroupTypeManager(postgres, teamManager)
 
     const cookielessManager = new CookielessManager(serverConfig, redisPool, teamManager)

plugin-server/src/utils/posthog.ts

@@ -2,37 +2,57 @@ import { captureException as captureSentryException, captureMessage as captureSe
 import { PostHog } from 'posthog-node'
 import { SeverityLevel } from 'posthog-node/src/extensions/error-tracking/types'
 
+import { defaultConfig } from '../config/config'
 import { Team } from '../types'
 
-export const posthog = new PostHog('sTMFPsFhdP1Ssg', {
-    host: 'https://us.i.posthog.com',
-    enableExceptionAutocapture: false, // TODO - disabled while data volume is a problem, PS seems /extremely/ chatty exceptions wise
-})
+const posthog = defaultConfig.POSTHOG_API_KEY
+    ? new PostHog(defaultConfig.POSTHOG_API_KEY, {
+          host: defaultConfig.POSTHOG_HOST_URL,
+          enableExceptionAutocapture: false, // TODO - disabled while data volume is a problem, PS seems /extremely/ chatty exceptions wise
+      })
+    : null
 
-if (process.env.NODE_ENV === 'test') {
+if (process.env.NODE_ENV === 'test' && posthog) {
     void posthog.disable()
 }
 
-export const captureTeamEvent = (team: Team, event: string, properties: Record<string, any> = {}): void => {
-    posthog.capture({
-        distinctId: team.uuid,
-        event,
-        properties: {
-            team: team.uuid,
-            ...properties,
-        },
-        groups: {
-            project: team.uuid,
-            organization: team.organization_id,
-            instance: process.env.SITE_URL ?? 'unknown',
-        },
-    })
+export function captureTeamEvent(
+    team: Team,
+    event: string,
+    properties: Record<string, any> = {},
+    distinctId: string | null = null
+): void {
+    if (posthog) {
+        posthog.capture({
+            distinctId: distinctId ?? team.uuid,
+            event,
+            properties: {
+                team: team.uuid,
+                ...properties,
+            },
+            groups: {
+                project: team.uuid,
+                organization: team.organization_id,
+                instance: process.env.SITE_URL ?? 'unknown',
+            },
+        })
+    }
+}
+
+export function shutdown(): Promise<void> | null {
+    return posthog ? posthog.shutdown() : null
+}
+
+export function flush(): void {
+    if (posthog) {
+        void posthog.flush().catch(() => null)
+    }
 }
 
 // We use sentry-style hints rather than our flat property list all over the place,
 // so define a type for them that we can flatten internally
-export type Primitive = number | string | boolean | bigint | symbol | null | undefined
-export interface ExceptionHint {
+type Primitive = number | string | boolean | bigint | symbol | null | undefined
+interface ExceptionHint {
     level: SeverityLevel
     tags: Record<string, Primitive>
     extra: Record<string, any>
@@ -47,8 +67,7 @@ export function captureException(exception: any, hint?: Partial<ExceptionHint>):
         sentryId = captureSentryException(exception, hint)
     }
 
-    // TODO - this sampling is a hack while we work on our data consumption in error tracking
-    if (process.env.NODE_ENV === 'production' && Math.random() < 0.1) {
+    if (posthog) {
         let additionalProperties = {}
         if (hint) {
             additionalProperties = {

plugin-server/src/worker/ingestion/action-matcher.ts

@@ -24,7 +24,6 @@ import { mutatePostIngestionEventWithElementsList } from '../../utils/event'
 import { captureException } from '../../utils/posthog'
 import { stringify } from '../../utils/utils'
 import { ActionManager } from './action-manager'
-import { TeamManager } from './team-manager'
 
 /** These operators can only be matched if the provided filter's value has the right type. */
 const propertyOperatorToRequiredValueType: Partial<Record<PropertyOperator, string[]>> = {
@@ -133,11 +132,7 @@ export function matchString(actual: string, expected: string, matching: StringMa
 }
 
 export class ActionMatcher {
-    constructor(
-        private postgres: PostgresRouter,
-        private actionManager: ActionManager,
-        private teamManager: TeamManager
-    ) {}
+    constructor(private postgres: PostgresRouter, private actionManager: ActionManager) {}
 
     public hasWebhooks(teamId: number): boolean {
         return Object.keys(this.actionManager.getTeamActions(teamId)).length > 0

plugin-server/src/worker/ingestion/team-manager.ts

@@ -3,18 +3,17 @@ import LRU from 'lru-cache'
 
 import { ONE_MINUTE } from '../../config/constants'
 import { TeamIDWithConfig } from '../../main/ingestion-queues/session-recording/session-recordings-consumer'
-import { PipelineEvent, PluginsServerConfig, ProjectId, Team, TeamId } from '../../types'
+import { PipelineEvent, ProjectId, Team, TeamId } from '../../types'
 import { PostgresRouter, PostgresUse } from '../../utils/db/postgres'
 import { timeoutGuard } from '../../utils/db/utils'
-import { posthog } from '../../utils/posthog'
+import { captureTeamEvent } from '../../utils/posthog'
 
 export class TeamManager {
     postgres: PostgresRouter
     teamCache: LRU<TeamId, Team | null>
     tokenToTeamIdCache: LRU<string, TeamId | null>
-    instanceSiteUrl: string
 
-    constructor(postgres: PostgresRouter, serverConfig: PluginsServerConfig) {
+    constructor(postgres: PostgresRouter) {
         this.postgres = postgres
 
         this.teamCache = new LRU({
@@ -27,7 +26,6 @@ export class TeamManager {
             maxAge: 5 * ONE_MINUTE, // Expiration for negative lookups, positive lookups will expire via teamCache first
             updateAgeOnGet: false, // Make default behaviour explicit
         })
-        this.instanceSiteUrl = serverConfig.SITE_URL || 'unknown'
     }
 
     public async getTeamForEvent(event: PipelineEvent): Promise<Team | null> {
@@ -132,21 +130,16 @@ export class TeamManager {
             )
             const distinctIds: { distinct_id: string }[] = organizationMembers.rows
             for (const { distinct_id } of distinctIds) {
-                posthog.capture({
-                    distinctId: distinct_id,
-                    event: 'first team event ingested',
-                    properties: {
-                        team: team.uuid,
+                captureTeamEvent(
+                    team,
+                    'first team event ingested',
+                    {
                         sdk: properties.$lib,
                         realm: properties.realm,
                         host: properties.$host,
                     },
-                    groups: {
-                        project: team.uuid,
-                        organization: team.organization_id,
-                        instance: this.instanceSiteUrl,
-                    },
-                })
+                    distinct_id
+                )
             }
         }
     }

plugin-server/tests/main/ingestion-queues/each-batch-webhooks.test.ts

@@ -84,7 +84,7 @@ describe('eachMessageWebhooksHandlers', () => {
 
     it('calls runWebhooksHandlersEventPipeline', async () => {
         const actionManager = new ActionManager(hub.postgres, hub)
-        const actionMatcher = new ActionMatcher(hub.postgres, actionManager, hub.teamManager)
+        const actionMatcher = new ActionMatcher(hub.postgres, actionManager)
         const hookCannon = new HookCommander(
             hub.postgres,
             hub.teamManager,

plugin-server/tests/main/ingestion-queues/each-batch.test.ts

@@ -172,11 +172,7 @@ describe('eachBatchX', () => {
     describe('eachBatchWebhooksHandlers', () => {
         it('calls runWebhooksHandlersEventPipeline', async () => {
             const actionManager = new ActionManager(queue.pluginsServer.postgres, queue.pluginsServer)
-            const actionMatcher = new ActionMatcher(
-                queue.pluginsServer.postgres,
-                actionManager,
-                queue.pluginsServer.teamManager
-            )
+            const actionMatcher = new ActionMatcher(queue.pluginsServer.postgres, actionManager)
             const hookCannon = new HookCommander(
                 queue.pluginsServer.postgres,
                 queue.pluginsServer.teamManager,

plugin-server/tests/main/process-event.test.ts

@@ -10,6 +10,8 @@ import { PluginEvent } from '@posthog/plugin-scaffold/src/types'
 import * as IORedis from 'ioredis'
 import { DateTime } from 'luxon'
 
+import { captureTeamEvent } from '~/src/utils/posthog'
+
 import { KAFKA_EVENTS_PLUGIN_INGESTION } from '../../src/config/kafka-topics'
 import {
     ClickHouseEvent,
@@ -24,7 +26,6 @@ import {
 import { closeHub, createHub } from '../../src/utils/db/hub'
 import { PostgresUse } from '../../src/utils/db/postgres'
 import { personInitialAndUTMProperties } from '../../src/utils/db/utils'
-import { posthog } from '../../src/utils/posthog'
 import { UUIDT } from '../../src/utils/utils'
 import { EventPipelineRunner } from '../../src/worker/ingestion/event-pipeline/runner'
 import { EventsProcessor } from '../../src/worker/ingestion/process-event'
@@ -34,6 +35,10 @@ import { createUserTeamAndOrganization, getFirstTeam, getTeams, resetTestDatabas
 
 jest.mock('../../src/utils/status')
 jest.setTimeout(600000) // 600 sec timeout.
+jest.mock('../../src/utils/posthog', () => ({
+    ...jest.requireActual('../../src/utils/posthog'),
+    captureTeamEvent: jest.fn(),
+}))
 
 export async function createPerson(
     server: Hub,
@@ -880,9 +885,6 @@ test('capture first team event', async () => {
         'testTag'
     )
 
-    posthog.capture = jest.fn() as any
-    posthog.identify = jest.fn() as any
-
     await processEvent(
         '2',
         '',
@@ -900,18 +902,12 @@ test('capture first team event', async () => {
         new UUIDT().toString()
     )
 
-    expect(posthog.capture).toHaveBeenCalledWith({
-        distinctId: 'plugin_test_user_distinct_id_1001',
-        event: 'first team event ingested',
-        properties: {
-            team: team.uuid,
-        },
-        groups: {
-            project: team.uuid,
-            organization: team.organization_id,
-            instance: 'unknown',
-        },
-    })
+    expect(captureTeamEvent).toHaveBeenCalledWith(
+        expect.objectContaining({ uuid: team.uuid, organization_id: team.organization_id }),
+        'first team event ingested',
+        { host: undefined, realm: undefined, sdk: undefined },
+        'plugin_test_user_distinct_id_1001'
+    )
 
     team = await getFirstTeam(hub)
     expect(team.ingested_event).toEqual(true)

plugin-server/tests/worker/ingestion/action-matcher.test.ts

@@ -54,7 +54,7 @@ describe('ActionMatcher', () => {
         hub = await createHub()
         actionManager = new ActionManager(hub.db.postgres, hub)
         await actionManager.start()
-        actionMatcher = new ActionMatcher(hub.db.postgres, actionManager, hub.teamManager)
+        actionMatcher = new ActionMatcher(hub.db.postgres, actionManager)
         actionCounter = 0
     })
 

plugin-server/tests/worker/ingestion/event-pipeline/event-pipeline-integration.test.ts

@@ -46,7 +46,7 @@ describe('Event Pipeline integration test', () => {
 
         actionManager = new ActionManager(hub.db.postgres, hub)
         await actionManager.start()
-        actionMatcher = new ActionMatcher(hub.db.postgres, actionManager, hub.teamManager)
+        actionMatcher = new ActionMatcher(hub.db.postgres, actionManager)
         hookCannon = new HookCommander(
             hub.db.postgres,
             hub.teamManager,

plugin-server/tests/worker/ingestion/team-manager.test.ts

@@ -6,12 +6,6 @@ import { TeamManager } from '../../../src/worker/ingestion/team-manager'
 import { resetTestDatabase } from '../../helpers/sql'
 
 jest.mock('../../../src/utils/status')
-jest.mock('../../../src/utils/posthog', () => ({
-    posthog: {
-        identify: jest.fn(),
-        capture: jest.fn(),
-    },
-}))
 
 describe('TeamManager()', () => {
     let teamManager: TeamManager
@@ -20,7 +14,7 @@ describe('TeamManager()', () => {
     beforeEach(async () => {
         await resetTestDatabase()
         postgres = new PostgresRouter(defaultConfig)
-        teamManager = new TeamManager(postgres, defaultConfig)
+        teamManager = new TeamManager(postgres)
 
         // @ts-expect-error TODO: Fix underlying settings, is this really working?
         Settings.defaultZoneName = 'utc'

plugin-server/tests/worker/plugins/run.test.ts

@@ -279,7 +279,7 @@ describe('runComposeWebhook', () => {
                 queueMetric: jest.fn(),
                 queueError: jest.fn(),
             } as any,
-            actionMatcher: new ActionMatcher(mockPostgres, mockActionManager, {} as any),
+            actionMatcher: new ActionMatcher(mockPostgres, mockActionManager),
         }
     })