Merge branch 'master' into feat/pd/report-when-player-state-is-error

hogvm/__tests__/__snapshots__/stl.hoge

@@ -24,4 +24,8 @@
 29, 2, "notEmpty", 1, 2, "print", 1, 35, 30, 2, "notEmpty", 1, 2, "print", 1, 35, 32, "", 2, "print", 1, 35, 32,
 "-- replaceAll, replaceOne --", 2, "print", 1, 35, 32, "hello world", 32, "l", 32, "L", 2, "replaceAll", 3, 2, "print",
 1, 35, 32, "hello world", 32, "l", 32, "L", 2, "replaceOne", 3, 2, "print", 1, 35, 32, "", 2, "print", 1, 35, 32,
-"-- generateUUIDv4 --", 2, "print", 1, 35, 2, "generateUUIDv4", 0, 2, "length", 1, 2, "print", 1, 35]
+"-- generateUUIDv4 --", 2, "print", 1, 35, 2, "generateUUIDv4", 0, 2, "length", 1, 2, "print", 1, 35, 32, "", 2,
+"print", 1, 35, 32, "-- isNull, isNotNull --", 2, "print", 1, 35, 31, 2, "isNull", 1, 31, 2, "isNotNull", 1, 2, "print",
+2, 35, 29, 2, "isNull", 1, 29, 2, "isNotNull", 1, 2, "print", 2, 35, 32, "banana", 2, "isNull", 1, 32, "banana", 2,
+"isNotNull", 1, 2, "print", 2, 35, 30, 2, "isNull", 1, 30, 2, "isNotNull", 1, 2, "print", 2, 35, 33, 0, 2, "isNull", 1,
+33, 0, 2, "isNotNull", 1, 2, "print", 2, 35, 33, 1, 2, "isNull", 1, 33, 1, 2, "isNotNull", 1, 2, "print", 2, 35]

hogvm/__tests__/__snapshots__/stl.js

@@ -7,6 +7,8 @@ function print (...args) { console.log(...args.map(__printHogStringOutput)) }
 function notEmpty (value) { return !empty(value) }
 function lower (value) { return value.toLowerCase() }
 function length (value) { return value.length }
+function isNull (value) { return value === null || value === undefined }
+function isNotNull (value) { return value !== null && value !== undefined }
 function generateUUIDv4 () { return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) { const r = (Math.random() * 16) | 0; const v = c === 'x' ? r : (r & 0x3) | 0x8; return v.toString(16) })}
 function encodeURLComponent (str) { return encodeURIComponent(str) }
 function empty (value) {
@@ -119,3 +121,11 @@ print(replaceOne("hello world", "l", "L"));
 print("");
 print("-- generateUUIDv4 --");
 print(length(generateUUIDv4()));
+print("");
+print("-- isNull, isNotNull --");
+print(isNull(null), isNotNull(null));
+print(isNull(true), isNotNull(true));
+print(isNull("banana"), isNotNull("banana"));
+print(isNull(false), isNotNull(false));
+print(isNull(0), isNotNull(0));
+print(isNull(1), isNotNull(1));

hogvm/__tests__/__snapshots__/stl.stdout

@@ -57,3 +57,11 @@ heLlo world
 
 -- generateUUIDv4 --
 36
+
+-- isNull, isNotNull --
+true false
+false true
+false true
+false true
+false true
+false true

hogvm/__tests__/stl.hog

@@ -57,4 +57,11 @@ print(replaceOne('hello world', 'l', 'L'))
 print('')
 print('-- generateUUIDv4 --')
 print(length(generateUUIDv4()))
-
+print('')
+print('-- isNull, isNotNull --')
+print(isNull(null), isNotNull(null))
+print(isNull(true), isNotNull(true))
+print(isNull('banana'), isNotNull('banana'))
+print(isNull(false), isNotNull(false))
+print(isNull(0), isNotNull(0))
+print(isNull(1), isNotNull(1))

hogvm/python/stl/__init__.py

@@ -449,6 +449,8 @@ def _typeof(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]]
     "toInt": STLFunction(fn=toInt, minArgs=1, maxArgs=1),
     "toFloat": STLFunction(fn=toFloat, minArgs=1, maxArgs=1),
     "ifNull": STLFunction(fn=ifNull, minArgs=2, maxArgs=2),
+    "isNull": STLFunction(fn=lambda args, team, stdout, timeout: args[0] is None, minArgs=1, maxArgs=1),
+    "isNotNull": STLFunction(fn=lambda args, team, stdout, timeout: args[0] is not None, minArgs=1, maxArgs=1),
     "length": STLFunction(fn=lambda args, team, stdout, timeout: len(args[0]), minArgs=1, maxArgs=1),
     "empty": STLFunction(fn=empty, minArgs=1, maxArgs=1),
     "notEmpty": STLFunction(

hogvm/typescript/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@posthog/hogvm",
-    "version": "1.0.64",
+    "version": "1.0.65",
     "description": "PostHog Hog Virtual Machine",
     "types": "dist/index.d.ts",
     "source": "src/index.ts",

hogvm/typescript/src/stl/stl.ts

@@ -113,6 +113,20 @@ export const STL: Record<string, STLFunction> = {
         minArgs: 2,
         maxArgs: 2,
     },
+    isNull: {
+        fn: (args) => {
+            return args[0] === null || args[0] === undefined
+        },
+        minArgs: 1,
+        maxArgs: 1,
+    },
+    isNotNull: {
+        fn: (args) => {
+            return args[0] !== null && args[0] !== undefined
+        },
+        minArgs: 1,
+        maxArgs: 1,
+    },
     length: {
         fn: (args) => {
             return args[0].length

package.json

@@ -77,7 +77,7 @@
         "@microlink/react-json-view": "^1.21.3",
         "@microsoft/fetch-event-source": "^2.0.1",
         "@monaco-editor/react": "4.6.0",
-        "@posthog/hogvm": "^1.0.64",
+        "@posthog/hogvm": "^1.0.65",
         "@posthog/icons": "0.9.2",
         "@posthog/plugin-scaffold": "^1.4.4",
         "@react-hook/size": "^2.1.2",

plugin-server/package.json

@@ -54,7 +54,7 @@
         "@maxmind/geoip2-node": "^3.4.0",
         "@posthog/clickhouse": "^1.7.0",
         "@posthog/cyclotron": "file:../rust/cyclotron-node",
-        "@posthog/hogvm": "^1.0.64",
+        "@posthog/hogvm": "^1.0.65",
         "@posthog/plugin-scaffold": "1.4.4",
         "@sentry/node": "^7.49.0",
         "@sentry/profiling-node": "^0.3.0",

posthog/hogql/compiler/javascript_stl.py

@@ -55,6 +55,14 @@
         "function ifNull (value, defaultValue) { return value !== null ? value : defaultValue } ",
         [],
     ],
+    "isNull": [
+        "function isNull (value) { return value === null || value === undefined }",
+        [],
+    ],
+    "isNotNull": [
+        "function isNotNull (value) { return value !== null && value !== undefined }",
+        [],
+    ],
     "length": [
         "function length (value) { return value.length }",
         [],

posthog/hogql/database/database.py

@@ -53,6 +53,7 @@
 from posthog.hogql.database.schema.person_distinct_ids import (
     PersonDistinctIdsTable,
     RawPersonDistinctIdsTable,
+    join_data_warehouse_experiment_table_with_person_distinct_ids_table,
 )
 from posthog.hogql.database.schema.persons import (
     PersonsTable,
@@ -457,6 +458,14 @@ def define_mappings(warehouse: dict[str, Table], get_table: Callable):
                 ),
             )
 
+            if "events" in join.joining_table_name and join.configuration.get("experiments_optimized"):
+                source_table.fields["pdi"] = LazyJoin(
+                    from_field=from_field,
+                    join_table=PersonDistinctIdsTable(),
+                    join_function=join_data_warehouse_experiment_table_with_person_distinct_ids_table,
+                )
+                source_table.fields["person"] = FieldTraverser(chain=["pdi", "person"])
+
             if join.source_table_name == "persons":
                 person_field = database.events.fields["person"]
                 if isinstance(person_field, ast.FieldTraverser):

posthog/hogql/database/schema/person_distinct_ids.py

@@ -67,6 +67,29 @@ def join_with_person_distinct_ids_table(
     return join_expr
 
 
+def join_data_warehouse_experiment_table_with_person_distinct_ids_table(
+    join_to_add: LazyJoinToAdd,
+    context: HogQLContext,
+    node: SelectQuery,
+):
+    from posthog.hogql import ast
+
+    if not join_to_add.fields_accessed:
+        raise ResolutionError("No fields requested from person_distinct_ids")
+    join_expr = ast.JoinExpr(table=select_from_person_distinct_ids_table(join_to_add.fields_accessed))
+    join_expr.join_type = "LEFT JOIN"
+    join_expr.alias = join_to_add.to_table
+    join_expr.constraint = ast.JoinConstraint(
+        expr=ast.CompareOperation(
+            op=ast.CompareOperationOp.Eq,
+            left=ast.Field(chain=[join_to_add.from_table, *join_to_add.lazy_join.from_field]),
+            right=ast.Field(chain=[join_to_add.to_table, "distinct_id"]),
+        ),
+        constraint_type="ON",
+    )
+    return join_expr
+
+
 class RawPersonDistinctIdsTable(Table):
     fields: dict[str, FieldOrTable] = {
         **PERSON_DISTINCT_IDS_FIELDS,

posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py

@@ -246,15 +246,6 @@ def create_data_warehouse_table_with_usage(self):
             field_name="events",
             configuration={"experiments_optimized": True, "experiments_timestamp_key": "ds"},
         )
-
-        DataWarehouseJoin.objects.create(
-            team=self.team,
-            source_table_name=table_name,
-            source_table_key="userid",
-            joining_table_name="persons",
-            joining_table_key="properties.$user_id",
-            field_name="person",
-        )
         return table_name
 
     @freeze_time("2020-01-01T12:00:00Z")
@@ -869,8 +860,9 @@ def test_query_runner_with_data_warehouse_series_no_end_date_and_nested_id(self)
 
         _create_person(
             team=self.team,
-            distinct_ids=["internal_test_1"],
-            properties={"email": "[email protected]", "$user_id": "internal_test_1"},
+            uuid="018f14b8-6cf3-7ffd-80bb-5ef1a9e4d328",
+            distinct_ids=["018f14b8-6cf3-7ffd-80bb-5ef1a9e4d328", "internal_test_1"],
+            properties={"email": "[email protected]"},
         )
 
         _create_event(
@@ -946,7 +938,7 @@ def test_query_runner_with_data_warehouse_series_no_end_date_and_nested_id(self)
             )
 
             # Assert the expected join condition in the clickhouse SQL
-            expected_join_condition = f"and(equals(events.team_id, {query_runner.count_query_runner.team.id}), equals(event, %(hogql_val_12)s), greaterOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull(%(hogql_val_13)s, 6, %(hogql_val_14)s))), lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull(%(hogql_val_15)s, 6, %(hogql_val_16)s))))) AS e__events ON"
+            expected_join_condition = f"and(equals(events.team_id, {query_runner.count_query_runner.team.id}), equals(event, %(hogql_val_11)s), greaterOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull(%(hogql_val_12)s, 6, %(hogql_val_13)s))), lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull(%(hogql_val_14)s, 6, %(hogql_val_15)s))))) AS e__events ON"
             self.assertIn(
                 expected_join_condition,
                 str(response.clickhouse),

posthog/management/commands/print_hog_stl_table.py

@@ -0,0 +1,33 @@
+# Print compatibility table
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    help = "Print a Hog/HogQL STL compatibility table"
+
+    def handle(self, *args, **options):
+        from posthog.hogql.functions.mapping import HOGQL_CLICKHOUSE_FUNCTIONS, HOGQL_COMPARISON_MAPPING
+
+        hogql_functions = set(HOGQL_COMPARISON_MAPPING.keys()).union(set(HOGQL_CLICKHOUSE_FUNCTIONS.keys()))
+
+        from hogvm.python.stl import STL
+        from hogvm.python.stl.bytecode import BYTECODE_STL
+
+        hog_functions = set(STL.keys()).union(set(BYTECODE_STL.keys()))
+
+        hogql_functions = {
+            fn
+            if fn not in {f.lower() for f in hog_functions}
+            else next((f for f in hog_functions if f.lower() == fn), fn)
+            for fn in hogql_functions
+        }
+
+        all_functions = sorted(hog_functions.union(hogql_functions))
+        max_length = max(len(fn) for fn in all_functions)
+
+        for fn in all_functions:
+            print(  # noqa: T201
+                fn.ljust(max_length),
+                "HogQL" if fn in hogql_functions else "     ",
+                "Hog" if fn in hog_functions else "   ",
+            )