Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(cdp): Mini purge of old ingesters #29341

Open
wants to merge 21 commits into
base: master
Choose a base branch
from
+508 −1,320
Open

feat(cdp): Mini purge of old ingesters #29341

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
777fe46
Removed capas
benjackwhite Feb 28, 2025
57478eb
Fix
benjackwhite Feb 28, 2025
924201b
More removals
benjackwhite Feb 28, 2025
3b705bd
Remove!
benjackwhite Feb 28, 2025
30c2b1a
Added tests for person upgrades
benjackwhite Feb 28, 2025
e9be669
Swapped tests over to consumer tests
benjackwhite Feb 28, 2025
088deaf
Fix
benjackwhite Feb 28, 2025
852cde4
fix
benjackwhite Feb 28, 2025
83a28cd
Fix
benjackwhite Feb 28, 2025
2bca67f
Merge branch 'master' into feat/remove-old-ingesters
benjackwhite Feb 28, 2025
1221d71
Fixes
benjackwhite Feb 28, 2025
7e8dedf
Fix
benjackwhite Feb 28, 2025
44ae8d0
fix
benjackwhite Feb 28, 2025
c2e23a3
Fkix
benjackwhite Feb 28, 2025
390b38f
fix
benjackwhite Feb 28, 2025
37f3142
Removed skipped recordings tests
benjackwhite Feb 28, 2025
262d3e2
Fix
benjackwhite Feb 28, 2025
c0d1f5b
Merge branch 'master' into feat/remove-old-ingesters
benjackwhite Feb 28, 2025
89f9366
Fix
benjackwhite Feb 28, 2025
22c4763
Fixes
benjackwhite Feb 28, 2025
182ca0e
Fixes
benjackwhite Feb 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 0 additions & 35 deletions plugin-server/src/capabilities.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,38 +39,6 @@ export function getPluginServerCapabilities(config: PluginsServerConfig): Plugin
ingestionV2: true,
...sharedCapabilities,
}
case PluginServerMode.ingestion:
// NOTE: this mode will be removed in the future and replaced with
// `analytics-ingestion` and `recordings-ingestion` modes.
return {
mmdb: true,
ingestion: true,
...sharedCapabilities,
}
case PluginServerMode.ingestion_overflow:
return {
mmdb: true,
ingestionOverflow: true,
...sharedCapabilities,
}
case PluginServerMode.ingestion_historical:
return {
mmdb: true,
ingestionHistorical: true,
...sharedCapabilities,
}
case PluginServerMode.events_ingestion:
return {
mmdb: true,
eventsIngestionPipelines: true,
...sharedCapabilities,
}
case PluginServerMode.analytics_ingestion:
return {
mmdb: true,
ingestion: true,
...sharedCapabilities,
}
case PluginServerMode.recordings_blob_ingestion:
return {
sessionRecordingBlobIngestion: true,
Expand Down Expand Up @@ -147,9 +115,6 @@ export function getPluginServerCapabilities(config: PluginsServerConfig): Plugin
case PluginServerMode.functional_tests:
return {
mmdb: true,
ingestion: true,
ingestionHistorical: true,
eventsIngestionPipelines: true,
pluginScheduledTasks: true,
processPluginJobs: true,
processAsyncOnEventHandlers: true,
Expand Down
216 changes: 216 additions & 0 deletions plugin-server/src/ingestion/__snapshots__/ingestion-consumer.test.ts.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,222 @@ exports[`IngestionConsumer typical event processing event with common distinct_i
]
`;

exports[`IngestionConsumer typical event processing forced person upgrade 1`] = `
[
{
"key": "<REPLACED-UUID-0>",
"topic": "clickhouse_events_json_test",
"value": {
"created_at": "2025-01-01 00:00:00.000",
"distinct_id": "user-1",
"elements_chain": "",
"event": "$pageview",
"person_created_at": "1970-01-01 00:00:05",
"person_id": "<REPLACED-UUID-1>",
"person_mode": "propertyless",
"person_properties": "{}",
"project_id": 2,
"properties": "{"$process_person_profile":false,"$ip":"127.0.0.1"}",
"team_id": 2,
"timestamp": "2025-01-01 00:00:00.000",
"uuid": "<REPLACED-UUID-1>",
},
},
{
"key": "<REPLACED-UUID-2>",
"topic": "clickhouse_events_json_test",
"value": {
"created_at": "2025-01-01 00:00:00.000",
"distinct_id": "user-1",
"elements_chain": "",
"event": "$identify",
"person_created_at": "2025-01-01 00:00:00",
"person_id": "<REPLACED-UUID-3>",
"person_mode": "full",
"person_properties": "{"email":"test@example.com","$creator_event_uuid":"<REPLACED-UUID-3>"}",
"project_id": 2,
"properties": "{"$set":{"email":"test@example.com"},"$ip":"127.0.0.1"}",
"team_id": 2,
"timestamp": "2025-01-01 00:00:00.000",
"uuid": "<REPLACED-UUID-3>",
},
},
{
"key": "<REPLACED-UUID-3>",
"topic": "clickhouse_events_json_test",
"value": {
"created_at": "2025-01-01 00:00:00.000",
"distinct_id": "user-1",
"elements_chain": "",
"event": "$pageview",
"person_created_at": "2025-01-01 00:00:00",
"person_id": "<REPLACED-UUID-4>",
"person_mode": "force_upgrade",
"person_properties": "{}",
"project_id": 2,
"properties": "{"$process_person_profile":false,"$ip":"127.0.0.1"}",
"team_id": 2,
"timestamp": "2025-01-01 00:02:00.000",
"uuid": "<REPLACED-UUID-4>",
},
},
{
"key": "<REPLACED-UUID-4>",
"topic": "clickhouse_events_json_test",
"value": {
"created_at": "2025-01-01 00:00:00.000",
"distinct_id": "user-1",
"elements_chain": "",
"event": "$pageview",
"person_created_at": "2025-01-01 00:00:00",
"person_id": "<REPLACED-UUID-5>",
"person_mode": "full",
"person_properties": "{"email":"test@example.com","update3":"3","$creator_event_uuid":"<REPLACED-UUID-5>"}",
"project_id": 2,
"properties": "{"$set":{"update3":"3"},"$ip":"127.0.0.1"}",
"team_id": 2,
"timestamp": "2025-01-01 00:03:00.000",
"uuid": "<REPLACED-UUID-5>",
},
},
{
"key": null,
"topic": "clickhouse_person_test",
"value": {
"created_at": "2025-01-01 00:00:00",
"id": "<REPLACED-UUID-5>",
"is_deleted": 0,
"is_identified": 0,
"properties": "{"email":"test@example.com","$creator_event_uuid":"<REPLACED-UUID-5>"}",
"team_id": 2,
"version": 0,
},
},
{
"key": null,
"topic": "clickhouse_person_distinct_id_test",
"value": {
"distinct_id": "user-1",
"is_deleted": 0,
"person_id": "<REPLACED-UUID-5>",
"team_id": 2,
"version": 0,
},
},
{
"key": null,
"topic": "clickhouse_person_test",
"value": {
"created_at": "2025-01-01 00:00:00",
"id": "<REPLACED-UUID-5>",
"is_deleted": 0,
"is_identified": 0,
"properties": "{"email":"test@example.com","update3":"3","$creator_event_uuid":"<REPLACED-UUID-5>"}",
"team_id": 2,
"version": 1,
},
},
]
`;

exports[`IngestionConsumer typical event processing groups 1`] = `
[
{
"key": "<REPLACED-UUID-0>",
"topic": "clickhouse_events_json_test",
"value": {
"created_at": "2025-01-01 00:00:00.000",
"distinct_id": "user-1",
"elements_chain": "",
"event": "$pageview",
"person_created_at": "2025-01-01 00:00:00",
"person_id": "<REPLACED-UUID-1>",
"person_mode": "full",
"person_properties": "{"$creator_event_uuid":"<REPLACED-UUID-2>"}",
"project_id": 2,
"properties": "{"$groups":{"a":"group-a","b":"group-b","c":"group-c","d":"group-d","e":"group-e","f":"group-f"},"$ip":"127.0.0.1","$group_0":"group-a","$group_1":"group-b","$group_2":"group-c","$group_3":"group-d","$group_4":"group-e"}",
"team_id": 2,
"timestamp": "2025-01-01 00:00:00.000",
"uuid": "<REPLACED-UUID-1>",
},
},
{
"key": "<REPLACED-UUID-2>",
"topic": "clickhouse_events_json_test",
"value": {
"created_at": "2025-01-01 00:00:00.000",
"distinct_id": "user-1",
"elements_chain": "",
"event": "$groupidentify",
"person_created_at": "2025-01-01 00:00:00",
"person_id": "<REPLACED-UUID-3>",
"person_mode": "full",
"person_properties": "{"$creator_event_uuid":"<REPLACED-UUID-3>"}",
"project_id": 2,
"properties": "{"$group_type":"a","$group_key":"group-a","$group_set":{"id":"group-a","foo":"bar"},"$ip":"127.0.0.1"}",
"team_id": 2,
"timestamp": "2025-01-01 00:00:00.000",
"uuid": "<REPLACED-UUID-3>",
},
},
{
"key": "<REPLACED-UUID-3>",
"topic": "clickhouse_events_json_test",
"value": {
"created_at": "2025-01-01 00:00:00.000",
"distinct_id": "user-1",
"elements_chain": "",
"event": "$groupidentify",
"person_created_at": "2025-01-01 00:00:00",
"person_id": "<REPLACED-UUID-4>",
"person_mode": "full",
"person_properties": "{"$creator_event_uuid":"<REPLACED-UUID-4>"}",
"project_id": 2,
"properties": "{"$group_type":"f","$group_key":"group-f","$group_set":{"id":"group-f","foo":"bar"},"$ip":"127.0.0.1"}",
"team_id": 2,
"timestamp": "2025-01-01 00:00:00.000",
"uuid": "<REPLACED-UUID-4>",
},
},
{
"key": null,
"topic": "clickhouse_person_test",
"value": {
"created_at": "2025-01-01 00:00:00",
"id": "<REPLACED-UUID-4>",
"is_deleted": 0,
"is_identified": 0,
"properties": "{"$creator_event_uuid":"<REPLACED-UUID-4>"}",
"team_id": 2,
"version": 0,
},
},
{
"key": null,
"topic": "clickhouse_person_distinct_id_test",
"value": {
"distinct_id": "user-1",
"is_deleted": 0,
"person_id": "<REPLACED-UUID-4>",
"team_id": 2,
"version": 0,
},
},
{
"key": null,
"topic": "clickhouse_groups_test",
"value": {
"created_at": "2025-01-01 00:00:00",
"group_key": "group-a",
"group_properties": "{"id":"group-a","foo":"bar"}",
"group_type_index": 0,
"team_id": 2,
"version": 1,
},
},
]
`;

exports[`IngestionConsumer typical event processing malformed event 1`] = `
[
{
Expand Down
68 changes: 68 additions & 0 deletions plugin-server/src/ingestion/ingestion-consumer.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,32 @@ describe('IngestionConsumer', () => {
'person processing off',
() => [createEvent({ event: '$pageview', properties: { $process_person_profile: false } })],
],
[
'forced person upgrade',
() => [
createEvent({
event: '$pageview',
properties: { $process_person_profile: false, $set: { update1: '1' } },
}),
createEvent({
event: '$identify',
properties: { $process_person_profile: true, $set: { email: 'test@example.com' } },
}),
// Add an event at least a minute in the future and it should get force upgraded
createEvent({
event: '$pageview',
properties: { $process_person_profile: false, $set: { update2: '2' } },
timestamp: DateTime.now().plus({ minutes: 2 }).toISO(),
}),
// Add a person-full event and ensure all properties are there that should be
createEvent({
event: '$pageview',
properties: { $process_person_profile: true, $set: { update3: '3' } },
timestamp: DateTime.now().plus({ minutes: 3 }).toISO(),
}),
// Snapshot should contain update2 and update3 but not update1
],
],
[
'client ingestion warning',
() => [
Expand All @@ -480,6 +506,48 @@ describe('IngestionConsumer', () => {
}),
],
],

[
'groups',
() => [
createEvent({
event: '$pageview',
properties: {
$groups: {
a: 'group-a',
b: 'group-b',
c: 'group-c',
d: 'group-d',
e: 'group-e',
f: 'group-f',
},
},
}),
createEvent({
event: '$groupidentify',
properties: {
$group_type: 'a',
$group_key: 'group-a',
$group_set: {
id: 'group-a',
foo: 'bar',
},
},
}),
// This triggers an event but not a groups clickhouse change as the max groups is already hit
createEvent({
event: '$groupidentify',
properties: {
$group_type: 'f',
$group_key: 'group-f',
$group_set: {
id: 'group-f',
foo: 'bar',
},
},
}),
],
],
]

it.each(eventTests)('%s', async (_, createEvents) => {
Expand Down
Loading
Loading