feat(err): symbol set management (#26439)

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: David Newell <[email protected]>

frontend/src/scenes/error-tracking/ErrorTrackingConfigurationScene.tsx

@@ -1,9 +1,10 @@
-import { IconTrash } from '@posthog/icons'
-import { LemonButton, LemonCollapse, LemonTable, LemonTableColumns, LemonTabs } from '@posthog/lemon-ui'
+import { IconRevert, IconTrash, IconUpload } from '@posthog/icons'
+import { LemonButton, LemonCollapse, LemonDialog, LemonTable, LemonTableColumns, LemonTabs } from '@posthog/lemon-ui'
 import { useActions, useValues } from 'kea'
 import { stackFrameLogic } from 'lib/components/Errors/stackFrameLogic'
 import { ErrorTrackingSymbolSet } from 'lib/components/Errors/types'
 import { JSONViewer } from 'lib/components/JSONViewer'
+import { humanFriendlyDetailedTime } from 'lib/utils'
 import { useEffect, useState } from 'react'
 import { SceneExport } from 'scenes/sceneTypes'
 
@@ -43,25 +44,48 @@ const SymbolSetTable = ({
     missing?: boolean
 }): JSX.Element => {
     const { symbolSetsLoading } = useValues(errorTrackingSymbolSetLogic)
-    const { deleteSymbolSet } = useActions(errorTrackingSymbolSetLogic)
+    const { deleteSymbolSet, setUploadSymbolSetId } = useActions(errorTrackingSymbolSetLogic)
 
     const columns: LemonTableColumns<ErrorTrackingSymbolSet> = [
         { title: missing && 'Missing symbol sets', dataIndex: 'ref' },
+        { title: 'Created At', dataIndex: 'created_at', render: (data) => humanFriendlyDetailedTime(data as string) },
         {
             dataIndex: 'id',
             render: (_, { id }) => {
                 return (
-                    <div className="flex justify-end">
-                        {!missing && (
-                            <LemonButton
-                                type="secondary"
-                                size="xsmall"
-                                tooltip="Delete symbol set"
-                                icon={<IconTrash />}
-                                onClick={() => deleteSymbolSet(id)}
-                                className="py-1"
-                            />
-                        )}
+                    <div className="flex justify-end space-x-1">
+                        <LemonButton
+                            type={missing ? 'primary' : 'secondary'}
+                            size="xsmall"
+                            tooltip={missing ? 'Upload symbol set' : 'Replace symbol set'}
+                            icon={missing ? <IconUpload /> : <IconRevert />}
+                            onClick={() => setUploadSymbolSetId(id)}
+                            className="py-1"
+                        >
+                            {missing && 'Upload'}
+                        </LemonButton>
+                        <LemonButton
+                            type="secondary"
+                            size="xsmall"
+                            tooltip="Delete symbol set"
+                            icon={<IconTrash />}
+                            onClick={() =>
+                                LemonDialog.open({
+                                    title: 'Delete symbol set',
+                                    description: 'Are you sure you want to delete this symbol set?',
+                                    secondaryButton: {
+                                        type: 'secondary',
+                                        children: 'Cancel',
+                                    },
+                                    primaryButton: {
+                                        type: 'primary',
+                                        onClick: () => deleteSymbolSet(id),
+                                        children: 'Delete',
+                                    },
+                                })
+                            }
+                            className="py-1"
+                        />
                     </div>
                 )
             },

frontend/src/scenes/error-tracking/ErrorTrackingFilters.tsx

@@ -140,17 +140,19 @@ export const Options = ({ isGroup = false }: { isGroup?: boolean }): JSX.Element
                     </div>
                 )}
             </div>
-            {hasGroupActions && !isGroup && (
-                <div className="flex items-center gap-1">
-                    <span>Assigned to:</span>
-                    <MemberSelect
-                        value={assignee}
-                        onChange={(user) => {
-                            setAssignee(user?.id || null)
-                        }}
-                    />
-                </div>
-            )}
+            <div className="flex items-center gap-1">
+                {hasGroupActions && !isGroup && (
+                    <>
+                        <span>Assigned to:</span>
+                        <MemberSelect
+                            value={assignee}
+                            onChange={(user) => {
+                                setAssignee(user?.id || null)
+                            }}
+                        />
+                    </>
+                )}
+            </div>
         </div>
     )
 }

frontend/src/scenes/error-tracking/ErrorTrackingScene.tsx

@@ -8,6 +8,7 @@ import { PageHeader } from 'lib/components/PageHeader'
 import { LemonTableLink } from 'lib/lemon-ui/LemonTable/LemonTableLink'
 import { SceneExport } from 'scenes/sceneTypes'
 import { urls } from 'scenes/urls'
+import { userLogic } from 'scenes/userLogic'
 
 import { insightVizDataNodeKey } from '~/queries/nodes/InsightViz/InsightViz'
 import { Query } from '~/queries/Query/Query'
@@ -164,12 +165,25 @@ const AssigneeColumn: QueryContextColumnComponent = (props) => {
 }
 
 const Header = (): JSX.Element => {
+    const { user } = useValues(userLogic)
+
     return (
         <PageHeader
             buttons={
-                <LemonButton to={urls.errorTrackingConfiguration()} type="secondary" icon={<IconGear />}>
-                    Configure
-                </LemonButton>
+                <>
+                    {user?.is_staff ? (
+                        <LemonButton
+                            onClick={() => {
+                                throw Error('Oh my!')
+                            }}
+                        >
+                            Send an exception
+                        </LemonButton>
+                    ) : null}
+                    <LemonButton to={urls.errorTrackingConfiguration()} type="secondary" icon={<IconGear />}>
+                        Configure
+                    </LemonButton>
+                </>
             }
         />
     )

frontend/src/scenes/error-tracking/SymbolSetUploadModal.tsx

@@ -7,42 +7,61 @@ import { LemonField } from 'lib/lemon-ui/LemonField'
 import { errorTrackingSymbolSetLogic } from './errorTrackingSymbolSetLogic'
 
 export const SymbolSetUploadModal = (): JSX.Element => {
-    const { setUploadSymbolSetReference } = useActions(errorTrackingSymbolSetLogic)
-    const { uploadSymbolSetReference, isUploadSymbolSetSubmitting, uploadSymbolSet } =
-        useValues(errorTrackingSymbolSetLogic)
+    const { setUploadSymbolSetId } = useActions(errorTrackingSymbolSetLogic)
+    const { uploadSymbolSetId, isUploadSymbolSetSubmitting, uploadSymbolSet } = useValues(errorTrackingSymbolSetLogic)
 
-    const onClose = (): void => setUploadSymbolSetReference(null)
+    const onClose = (): void => setUploadSymbolSetId(null)
 
     return (
-        <LemonModal title="" onClose={onClose} isOpen={!!uploadSymbolSetReference} simple>
+        <LemonModal title="" onClose={onClose} isOpen={!!uploadSymbolSetId} simple>
             <Form logic={errorTrackingSymbolSetLogic} formKey="uploadSymbolSet" className="gap-1" enableFormOnSubmit>
                 <LemonModal.Header>
-                    <h3>Upload source map</h3>
+                    <h3>Upload javscript symbol set</h3>
                 </LemonModal.Header>
                 <LemonModal.Content className="space-y-2">
-                    <LemonField name="files">
+                    <LemonField name="minified">
                         <LemonFileInput
-                            accept="text/plain"
+                            accept="text/javascript"
                             multiple={false}
                             callToAction={
-                                <div className="flex flex-col items-center justify-center space-y-2 border border-dashed rounded p-4">
+                                <div className="flex flex-col items-center justify-center space-y-2 border border-dashed rounded p-4 w-full">
                                     <span className="flex items-center gap-2 font-semibold">
-                                        <IconUploadFile className="text-2xl" /> Add source map
+                                        <IconUploadFile className="text-2xl" /> Add minified source
                                     </span>
                                     <div>
-                                        Drag and drop your local source map here or click to open the file browser.
+                                        Drag and drop your minified source file here or click to open the file browser.
                                     </div>
                                 </div>
                             }
                         />
                     </LemonField>
+                    <LemonField name="sourcemap">
+                        <LemonFileInput
+                            accept="*"
+                            multiple={false}
+                            callToAction={
+                                <div className="flex flex-col items-center justify-center space-y-2 border border-dashed rounded p-4 w-full">
+                                    <span className="flex items-center gap-2 font-semibold">
+                                        <IconUploadFile className="text-2xl" /> Add source map
+                                    </span>
+                                    <div>Drag and drop your source map here or click to open the file browser.</div>
+                                </div>
+                            }
+                        />
+                    </LemonField>
                 </LemonModal.Content>
                 <LemonModal.Footer>
                     <LemonButton type="secondary" onClick={onClose}>
                         Cancel
                     </LemonButton>
                     <LemonButton
-                        disabledReason={uploadSymbolSet.files.length < 1 ? 'Upload a source map' : undefined}
+                        disabledReason={
+                            uploadSymbolSet.minified.length < 1
+                                ? 'Upload a minified source'
+                                : uploadSymbolSet.sourceMap.length < 1
+                                ? 'Upload a source map'
+                                : undefined
+                        }
                         type="primary"
                         status="alt"
                         htmlType="submit"

frontend/src/scenes/error-tracking/errorTrackingSymbolSetLogic.tsx

@@ -16,18 +16,25 @@ export enum ErrorGroupTab {
     Breakdowns = 'breakdowns',
 }
 
+export type SymbolSetUpload = SourceMapUpload
+
+export interface SourceMapUpload {
+    minified: File
+    sourcemap: File
+}
+
 export const errorTrackingSymbolSetLogic = kea<errorTrackingSymbolSetLogicType>([
     path(['scenes', 'error-tracking', 'errorTrackingSymbolSetLogic']),
 
     actions({
-        setUploadSymbolSetReference: (ref: ErrorTrackingSymbolSet['id'] | null) => ({ ref }),
+        setUploadSymbolSetId: (id: ErrorTrackingSymbolSet['id'] | null) => ({ id }),
     }),
 
     reducers({
-        uploadSymbolSetReference: [
+        uploadSymbolSetId: [
             null as string | null,
             {
-                setUploadSymbolSetReference: (_, { ref }) => ref,
+                setUploadSymbolSetId: (_, { id }) => id,
             },
         ],
     }),
@@ -40,10 +47,10 @@ export const errorTrackingSymbolSetLogic = kea<errorTrackingSymbolSetLogicType>(
                     const response = await api.errorTracking.symbolSets()
                     return response.results
                 },
-                deleteSymbolSet: async (ref) => {
-                    await api.errorTracking.deleteSymbolSet(ref)
+                deleteSymbolSet: async (id) => {
+                    await api.errorTracking.deleteSymbolSet(id)
                     const newValues = [...values.symbolSets]
-                    return newValues.filter((v) => v.ref !== ref)
+                    return newValues.filter((v) => v.id !== id)
                 },
             },
         ],
@@ -70,17 +77,29 @@ export const errorTrackingSymbolSetLogic = kea<errorTrackingSymbolSetLogicType>(
 
     forms(({ values, actions }) => ({
         uploadSymbolSet: {
-            defaults: { files: [] } as { files: File[] },
-            submit: async ({ files }) => {
-                if (files.length > 0 && values.uploadSymbolSetReference) {
-                    const formData = new FormData()
-                    const file = files[0]
-                    formData.append('source_map', file)
-                    await api.errorTracking.updateSymbolSet(values.uploadSymbolSetReference, formData)
-                    actions.setUploadSymbolSetReference(null)
-                    actions.loadSymbolSets()
-                    lemonToast.success('Source map uploaded')
+            defaults: { minified: [], sourceMap: [] } as { minified: File[]; sourceMap: File[] },
+            submit: async ({ minified, sourceMap }) => {
+                if (minified.length < 1 || sourceMap.length < 1) {
+                    lemonToast.error('Please select both a minified file and a source map file')
+                    return
                 }
+
+                const minifiedSrc = minified[0]
+                const sourceMapSrc = sourceMap[0]
+                const id = values.uploadSymbolSetId
+
+                if (id == null) {
+                    return
+                }
+
+                const formData = new FormData()
+                formData.append('minified', minifiedSrc)
+                formData.append('source_map', sourceMapSrc)
+                await api.errorTracking.updateSymbolSet(id, formData)
+                actions.setUploadSymbolSetId(null)
+                actions.loadSymbolSets()
+                actions.resetUploadSymbolSet()
+                lemonToast.success('Source map uploaded')
             },
         },
     })),

posthog/api/error_tracking.py

@@ -1,4 +1,6 @@
+from django.core.files.uploadedfile import UploadedFile
 import structlog
+import hashlib
 
 from rest_framework import serializers, viewsets, status
 from rest_framework.response import Response
@@ -14,7 +16,10 @@
 from posthog.storage import object_storage
 
 
-FIFTY_MEGABYTES = 50 * 1024 * 1024
+ONE_GIGABYTE = 1024 * 1024 * 1024
+JS_DATA_MAGIC = b"posthog_error_tracking"
+JS_DATA_VERSION = 1
+JS_DATA_TYPE_SOURCE_AND_MAP = 2
 
 logger = structlog.get_logger(__name__)
 
@@ -81,7 +86,7 @@ class Meta:
         read_only_fields = ["team_id"]
 
 
-class ErrorTrackingSymbolSetViewSet(TeamAndOrgViewSetMixin, viewsets.ReadOnlyModelViewSet):
+class ErrorTrackingSymbolSetViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):
     scope_object = "INTERNAL"
     queryset = ErrorTrackingSymbolSet.objects.all()
     serializer_class = ErrorTrackingSymbolSetSerializer
@@ -97,27 +102,52 @@ def destroy(self, request, *args, **kwargs):
 
     def update(self, request, *args, **kwargs) -> Response:
         symbol_set = self.get_object()
-        symbol_set.delete()
         # TODO: delete file from s3
-        storage_ptr = upload_symbol_set(request.FILES["source_map"], self.team_id)
+        minified = request.FILES["minified"]
+        source_map = request.FILES["source_map"]
+        (storage_ptr, content_hash) = upload_symbol_set(minified, source_map, self.team_id)
         symbol_set.storage_ptr = storage_ptr
+        symbol_set.content_hash = content_hash
         symbol_set.save()
+        ErrorTrackingStackFrame.objects.filter(team=self.team, symbol_set=symbol_set).delete()
         return Response({"ok": True}, status=status.HTTP_204_NO_CONTENT)
 
 
-def upload_symbol_set(file, team_id) -> str:
+def upload_symbol_set(minified: UploadedFile, source_map: UploadedFile, team_id) -> tuple[str, str]:
+    js_data = construct_js_data_object(minified.read(), source_map.read())
+    content_hash = hashlib.sha512(js_data).hexdigest()
+
     try:
         if settings.OBJECT_STORAGE_ENABLED:
-            if file.size > FIFTY_MEGABYTES:
-                raise ValidationError(code="file_too_large", detail="Source maps must be less than 50MB")
+            # TODO - maybe a gigabyte is too much?
+            if len(js_data) > ONE_GIGABYTE:
+                raise ValidationError(
+                    code="file_too_large", detail="Combined source map and symbol set must be less than 1 gigabyte"
+                )
 
             upload_path = f"{settings.OBJECT_STORAGE_ERROR_TRACKING_SOURCE_MAPS_FOLDER}/{str(uuid7())}"
-            object_storage.write(upload_path, file)
-            return upload_path
+            object_storage.write(upload_path, bytes(js_data))
+            return (upload_path, content_hash)
         else:
             raise ObjectStorageUnavailable()
     except ObjectStorageUnavailable:
         raise ValidationError(
             code="object_storage_required",
             detail="Object storage must be available to allow source map uploads.",
         )
+
+
+def construct_js_data_object(minified: bytes, source_map: bytes) -> bytearray:
+    # See rust/cymbal/hacks/js_data.rs
+    data = bytearray()
+    data.extend(JS_DATA_MAGIC)
+    data.extend(JS_DATA_VERSION.to_bytes(4, "little"))
+    data.extend((JS_DATA_TYPE_SOURCE_AND_MAP).to_bytes(4, "little"))
+    # TODO - this doesn't seem right?
+    s_bytes = minified.decode("utf-8").encode("utf-8")
+    data.extend(len(s_bytes).to_bytes(8, "little"))
+    data.extend(s_bytes)
+    sm_bytes = source_map.decode("utf-8").encode("utf-8")
+    data.extend(len(sm_bytes).to_bytes(8, "little"))
+    data.extend(sm_bytes)
+    return data