perf: Enable property group optimizations for all new teams by default

latest_migrations.manifest

@@ -5,7 +5,7 @@ contenttypes: 0002_remove_content_type_name
 ee: 0016_rolemembership_organization_member
 otp_static: 0002_throttling
 otp_totp: 0002_auto_20190420_0723
-posthog: 0472_experiment_metrics
+posthog: 0473_alter_team_modifiers
 sessions: 0001_initial
 social_django: 0010_uid_db_index
 two_factor: 0007_auto_20201201_1019

posthog/hogql/database/test/__snapshots__/test_database.ambr

@@ -219,7 +219,7 @@
                       "pdi"
                   ],
                   "hogql_value": "person",
-                  "id": null,
+                  "id": "person",
                   "name": "person",
                   "schema_valid": true,
                   "table": "persons",
@@ -472,7 +472,7 @@
                       "person"
                   ],
                   "hogql_value": "override",
-                  "id": null,
+                  "id": "override",
                   "name": "override",
                   "schema_valid": true,
                   "table": "person_distinct_id_overrides",

posthog/hogql/test/__snapshots__/test_query.ambr

@@ -33,15 +33,15 @@
 
   SELECT events.event AS event, events.distinct_id AS distinct_id 
   FROM events 
-  WHERE and(equals(events.team_id, 420), equals(events.distinct_id, %(hogql_val_0)s), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', ''), %(hogql_val_2)s), 0)) 
+  WHERE and(equals(events.team_id, 420), equals(events.distinct_id, %(hogql_val_1)s), equals(events.properties_group_custom[%(hogql_val_2)s], %(hogql_val_3)s)) 
   LIMIT 100 
   SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0
 
   -- HogQL
 
   SELECT event, distinct_id 
   FROM events 
-  WHERE and(equals(distinct_id, 'RANDOM_TEST_ID::UUID'), equals(properties.index, '4')) 
+  WHERE and(equals(distinct_id, 'RANDOM_TEST_ID::UUID'), equals(events.properties_group_custom[%(hogql_val_0)s], '4')) 
   LIMIT 100
   '''
 # ---
@@ -50,7 +50,7 @@
 
   SELECT event, distinct_id 
   FROM events 
-  WHERE and(equals(distinct_id, 'RANDOM_TEST_ID::UUID'), equals(properties.index, '4')) 
+  WHERE and(equals(distinct_id, 'RANDOM_TEST_ID::UUID'), equals(events.properties_group_custom[%(hogql_val_0)s], '4')) 
   LIMIT 100
   '''
 # ---
@@ -60,15 +60,15 @@
 
   SELECT events.event AS event, events.distinct_id AS distinct_id 
   FROM events 
-  WHERE and(equals(events.team_id, 420), equals(events.distinct_id, %(hogql_val_0)s), and(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', ''), %(hogql_val_2)s), 0), less(toTimeZone(events.timestamp, %(hogql_val_3)s), toDateTime64('2020-01-02 00:00:00.000000', 6, 'UTC')), greaterOrEquals(toTimeZone(events.timestamp, %(hogql_val_4)s), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')))) 
+  WHERE and(equals(events.team_id, 420), equals(events.distinct_id, %(hogql_val_1)s), and(equals(events.properties_group_custom[%(hogql_val_2)s], %(hogql_val_3)s), less(toTimeZone(events.timestamp, %(hogql_val_4)s), toDateTime64('2020-01-02 00:00:00.000000', 6, 'UTC')), greaterOrEquals(toTimeZone(events.timestamp, %(hogql_val_5)s), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')))) 
   LIMIT 100 
   SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0
 
   -- HogQL
 
   SELECT event, distinct_id 
   FROM events 
-  WHERE and(equals(distinct_id, 'RANDOM_TEST_ID::UUID'), and(equals(properties.index, '4'), less(timestamp, toDateTime('2020-01-02 00:00:00.000000')), greaterOrEquals(timestamp, toDateTime('2020-01-01 00:00:00.000000')))) 
+  WHERE and(equals(distinct_id, 'RANDOM_TEST_ID::UUID'), and(equals(events.properties_group_custom[%(hogql_val_0)s], '4'), less(timestamp, toDateTime('2020-01-02 00:00:00.000000')), greaterOrEquals(timestamp, toDateTime('2020-01-01 00:00:00.000000')))) 
   LIMIT 100
   '''
 # ---
@@ -77,7 +77,7 @@
 
   SELECT event, distinct_id 
   FROM events 
-  WHERE and(equals(distinct_id, 'RANDOM_TEST_ID::UUID'), and(equals(properties.index, '4'), less(timestamp, toDateTime('2020-01-02 00:00:00.000000')), greaterOrEquals(timestamp, toDateTime('2020-01-01 00:00:00.000000')))) 
+  WHERE and(equals(distinct_id, 'RANDOM_TEST_ID::UUID'), and(equals(events.properties_group_custom[%(hogql_val_0)s], '4'), less(timestamp, toDateTime('2020-01-02 00:00:00.000000')), greaterOrEquals(timestamp, toDateTime('2020-01-01 00:00:00.000000')))) 
   LIMIT 100
   '''
 # ---
@@ -439,7 +439,7 @@
 
   SELECT count(), events.event AS event 
   FROM events 
-  WHERE and(equals(events.team_id, 420), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', ''), %(hogql_val_1)s), 0)) 
+  WHERE and(equals(events.team_id, 420), equals(events.properties_group_custom[%(hogql_val_1)s], %(hogql_val_2)s)) 
   GROUP BY events.event 
   LIMIT 100 
   SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0
@@ -448,7 +448,7 @@
 
   SELECT count(), event 
   FROM events 
-  WHERE equals(properties.random_uuid, 'RANDOM_TEST_ID::UUID') 
+  WHERE equals(events.properties_group_custom[%(hogql_val_0)s], 'RANDOM_TEST_ID::UUID') 
   GROUP BY event 
   LIMIT 100
   '''
@@ -900,7 +900,7 @@
   FROM (
   SELECT count() AS count, events.event AS event 
   FROM events 
-  WHERE and(equals(events.team_id, 420), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', ''), %(hogql_val_1)s), 0)) 
+  WHERE and(equals(events.team_id, 420), equals(events.properties_group_custom[%(hogql_val_1)s], %(hogql_val_2)s)) 
   GROUP BY events.event) 
   GROUP BY count, event 
   LIMIT 100 
@@ -912,7 +912,7 @@
   FROM (
   SELECT count() AS count, event 
   FROM events 
-  WHERE equals(properties.random_uuid, 'RANDOM_TEST_ID::UUID') 
+  WHERE equals(events.properties_group_custom[%(hogql_val_0)s], 'RANDOM_TEST_ID::UUID') 
   GROUP BY event) 
   GROUP BY count, event 
   LIMIT 100
@@ -926,7 +926,7 @@
   FROM (
   SELECT count(*) AS count, events.event AS event 
   FROM events 
-  WHERE and(equals(events.team_id, 420), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', ''), %(hogql_val_1)s), 0)) 
+  WHERE and(equals(events.team_id, 420), equals(events.properties_group_custom[%(hogql_val_1)s], %(hogql_val_2)s)) 
   GROUP BY events.event) AS c 
   GROUP BY c.count, c.event 
   LIMIT 100 
@@ -938,7 +938,7 @@
   FROM (
   SELECT count(*) AS count, event 
   FROM events 
-  WHERE equals(properties.random_uuid, 'RANDOM_TEST_ID::UUID') 
+  WHERE equals(events.properties_group_custom[%(hogql_val_0)s], 'RANDOM_TEST_ID::UUID') 
   GROUP BY event) AS c 
   GROUP BY count, event 
   LIMIT 100
@@ -952,10 +952,10 @@
   FROM (
   SELECT col_a AS col_a, groupArray(tuple(col_b, col_c)) AS g 
   FROM (
-  SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS col_a, events.event AS col_b, count() AS col_c 
+  SELECT has(events.properties_group_custom, %(hogql_val_0)s) ? events.properties_group_custom[%(hogql_val_0)s] : null AS col_a, events.event AS col_b, count() AS col_c 
   FROM events 
   WHERE equals(events.team_id, 420) 
-  GROUP BY replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', ''), events.event) 
+  GROUP BY has(events.properties_group_custom, %(hogql_val_1)s) ? events.properties_group_custom[%(hogql_val_1)s] : null, events.event) 
   GROUP BY col_a) 
   GROUP BY col_a ORDER BY col_a ASC 
   LIMIT 100 
@@ -985,10 +985,10 @@
   FROM (
   SELECT PIVOT_TABLE_COL_ABC.col_a AS col_a, groupArray(tuple(PIVOT_TABLE_COL_ABC.col_b, PIVOT_TABLE_COL_ABC.col_c)) AS g 
   FROM (
-  SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS col_a, events.event AS col_b, count() AS col_c 
+  SELECT has(events.properties_group_custom, %(hogql_val_0)s) ? events.properties_group_custom[%(hogql_val_0)s] : null AS col_a, events.event AS col_b, count() AS col_c 
   FROM events 
   WHERE equals(events.team_id, 420) 
-  GROUP BY replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', ''), events.event) AS PIVOT_TABLE_COL_ABC 
+  GROUP BY has(events.properties_group_custom, %(hogql_val_1)s) ? events.properties_group_custom[%(hogql_val_1)s] : null, events.event) AS PIVOT_TABLE_COL_ABC 
   GROUP BY PIVOT_TABLE_COL_ABC.col_a) AS PIVOT_FUNCTION_1 
   GROUP BY PIVOT_FUNCTION_1.col_a) AS PIVOT_FUNCTION_2 ORDER BY PIVOT_FUNCTION_2.col_a ASC 
   LIMIT 100 
@@ -1022,10 +1022,10 @@
   FROM (
   SELECT PIVOT_TABLE_COL_ABC.col_a AS col_a, groupArray(tuple(PIVOT_TABLE_COL_ABC.col_b, PIVOT_TABLE_COL_ABC.col_c)) AS g 
   FROM (
-  SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS col_a, events.event AS col_b, count() AS col_c 
+  SELECT has(events.properties_group_custom, %(hogql_val_0)s) ? events.properties_group_custom[%(hogql_val_0)s] : null AS col_a, events.event AS col_b, count() AS col_c 
   FROM events 
   WHERE equals(events.team_id, 420) 
-  GROUP BY replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', ''), events.event) AS PIVOT_TABLE_COL_ABC 
+  GROUP BY has(events.properties_group_custom, %(hogql_val_1)s) ? events.properties_group_custom[%(hogql_val_1)s] : null, events.event) AS PIVOT_TABLE_COL_ABC 
   GROUP BY PIVOT_TABLE_COL_ABC.col_a) AS PIVOT_FUNCTION_1 
   GROUP BY PIVOT_FUNCTION_1.col_a) AS PIVOT_FUNCTION_2) AS final ORDER BY final.col_a ASC 
   LIMIT 100 

posthog/hogql/test/test_modifiers.py

@@ -32,7 +32,8 @@ def test_create_default_modifiers_for_team_init(self):
         assert modifiers.personsOnEventsMode == PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS
 
     def test_team_modifiers_override(self):
-        assert self.team.modifiers is None
+        self.team.modifiers = None
+        self.team.save()
         modifiers = create_default_modifiers_for_team(self.team)
         assert modifiers.personsOnEventsMode == self.team.default_modifiers["personsOnEventsMode"]
         assert (
@@ -60,7 +61,9 @@ def test_team_modifiers_override(self):
         True,
     )
     def test_modifiers_persons_on_events_default_is_based_on_team_property(self):
-        assert self.team.modifiers is None
+        self.team.modifiers = None
+        self.team.save()
+
         modifiers = create_default_modifiers_for_team(self.team)
         assert self.team.person_on_events_mode == PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS
         assert modifiers.personsOnEventsMode == self.team.default_modifiers["personsOnEventsMode"]

posthog/hogql/test/test_query.py

@@ -18,7 +18,15 @@
 from posthog.session_recordings.queries.test.session_replay_sql import (
     produce_replay_summary,
 )
-from posthog.schema import HogQLFilters, EventPropertyFilter, DateRange, QueryTiming, SessionPropertyFilter
+from posthog.schema import (
+    HogQLFilters,
+    EventPropertyFilter,
+    DateRange,
+    HogQLQueryModifiers,
+    PropertyGroupsMode,
+    QueryTiming,
+    SessionPropertyFilter,
+)
 from posthog.settings import HOGQL_INCREASED_MAX_EXECUTION_TIME
 from posthog.test.base import (
     APIBaseTest,
@@ -996,6 +1004,7 @@ def test_property_access_with_arrays(self):
                 query,
                 team=self.team,
                 pretty=False,
+                modifiers=HogQLQueryModifiers(propertyGroupsMode=PropertyGroupsMode.DISABLED),
             )
             self.assertEqual(
                 f"SELECT "
@@ -1416,6 +1425,7 @@ def test_hogql_query_filters_alias(self):
                 team=self.team,
                 filters=filters,
                 pretty=False,
+                modifiers=HogQLQueryModifiers(propertyGroupsMode=PropertyGroupsMode.DISABLED),
             )
             self.assertEqual(
                 response.hogql,

posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation.ambr

@@ -54,9 +54,9 @@
                  FROM
                    (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp,
                            if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target,
-                           if(and(equals(e.event, 'user signed up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0)), 1, 0) AS step_0,
+                           if(and(equals(e.event, 'user signed up'), equals(e.properties_group_custom['key'], 'val')), 1, 0) AS step_0,
                            if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0,
-                           if(and(equals(e.event, 'paid'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0)), 1, 0) AS step_1,
+                           if(and(equals(e.event, 'paid'), equals(e.properties_group_custom['key'], 'val')), 1, 0) AS step_1,
                            if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1
                     FROM events AS e
                     LEFT OUTER JOIN
@@ -123,9 +123,9 @@
                  FROM
                    (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp,
                            if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target,
-                           if(and(equals(e.event, 'user signed up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0)), 1, 0) AS step_0,
+                           if(and(equals(e.event, 'user signed up'), equals(e.properties_group_custom['key'], 'val')), 1, 0) AS step_0,
                            if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0,
-                           if(and(equals(e.event, 'paid'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0)), 1, 0) AS step_1,
+                           if(and(equals(e.event, 'paid'), equals(e.properties_group_custom['key'], 'val')), 1, 0) AS step_1,
                            if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1
                     FROM events AS e
                     LEFT OUTER JOIN