Merge pull request #804 from MuckRock/sveltekit-upload-refactor

Upload refactor
MuckRock · Nov 4, 2024 · 5fab3da · 5fab3da
2 parents ca90f75 + 25a174f
commit 5fab3da
Show file tree

Hide file tree

Showing 17 changed files with 501 additions and 362 deletions.
diff --git a/src/app.html b/src/app.html
@@ -8,7 +8,7 @@
 
     %sveltekit.head%
   </head>
-  <body data-sveltekit-preload-data="tap">
+  <body data-sveltekit-preload-data="hover">
     <div style="display: contents">%sveltekit.body%</div>
   </body>
 </html>
diff --git a/src/hooks.client.ts b/src/hooks.client.ts
@@ -14,7 +14,10 @@ Sentry.init({
   replaysOnErrorSampleRate: 0.5,
 
   // If you don't want to use Session Replay, just remove the line below:
-  integrations: [replayIntegration()],
+  integrations: [
+    replayIntegration(),
+    Sentry.captureConsoleIntegration({ levels: ["error", "warn"] }),
+  ],
 });
 
 // If you have a custom error handler, pass it to `handleErrorWithSentry`

diff --git a/src/hooks.server.ts b/src/hooks.server.ts
@@ -11,6 +11,9 @@ import { DC_BASE } from "./config/config.js";
 
 Sentry.init({
   dsn: env.SENTRY_DSN,
+  integrations: [
+    Sentry.captureConsoleIntegration({ levels: ["error", "warn"] }),
+  ],
   tracesSampleRate: 1,
 });
 

diff --git a/src/langs/json/en.json b/src/langs/json/en.json
@@ -12,7 +12,8 @@
     "add": "Add",
     "edit": "Edit",
     "search": "Search",
-    "feedback": "Feedback"
+    "feedback": "Feedback",
+    "title": "Title"
   },
   "search": {
     "matchingResults": "{n, plural, one {# matching result} other {# matching results}}"
@@ -383,7 +384,7 @@
     "fileLimitWarning": "You can only upload {limit} files at once.",
     "beginUpload": "Begin Upload",
     "selectFiles": "Select files",
-    "dragDrop": "Select, drag and drop, or paste files to upload",
+    "dragDrop": "Select, paste or drag-and-drop files to upload",
     "publicMsg": "Document will be publicly visible.",
     "collabMsg": "Document will be visible to your organization.",
     "privateMsg": "Document will be visible to you alone.",
@@ -409,7 +410,14 @@
     "revisionControlHelp": "All previous versions to the document will be preserved and available for download.",
     "tesseract": "<a target=\"_blank\" rel=\"noopener noreferrer\" href=\"https://tesseract-ocr.github.io/tessdoc/\">Tesseract</a> is an open source OCR engine available free to all DocumentCloud accounts.",
     "textract": "<a target=\"_blank\" rel=\"noopener noreferrer\" href=\"https://aws.amazon.com/textract/\">Textract</a> is a more powerful OCR engine that can better handle messy text, handwriting and fuzzy scans.",
-    "empty": "Get started by selecting, pasting or dragging-and-dropping files"
+    "empty": "Get started by selecting, pasting or dragging-and-dropping files",
+    "steps": {
+      "ready": "Ready",
+      "created": "Created",
+      "uploading": "Uploading",
+      "processing": "Processing",
+      "done": "Done"
+    }
   },
   "uploadOptions": {
     "documentLang": "Document language:",

diff --git a/src/lib/api/documents.ts b/src/lib/api/documents.ts
@@ -194,17 +194,17 @@ export async function textPositions(
 }
 
 /**
- * Create new documents in a batch (or a batch of one).
+ * Create a new document.
  *
- * If documents contain a `file_url` property, the server will attempt to fetch and upload that file.
+ * If document contains a `file_url` property, the server will attempt to fetch and upload that file.
  * Otherwise, the response will contain all documents fields plus a `presigned_url` field, which should
  * be passed to `upload` to store the actual file.
  */
 export async function create(
-  documents: DocumentUpload[],
+  doc: DocumentUpload,
   csrf_token: string,
   fetch = globalThis.fetch,
-): Promise<APIResponse<Document[], unknown>> {
+): Promise<APIResponse<Document, unknown>> {
   const endpoint = new URL("documents/", BASE_API_URL);
 
   const resp = await fetch(endpoint, {
@@ -215,10 +215,10 @@ export async function create(
       [CSRF_HEADER_NAME]: csrf_token,
       Referer: APP_URL,
     },
-    body: JSON.stringify(documents),
+    body: JSON.stringify(doc),
   }).catch(console.error);
 
-  return getApiResponse<Document[], unknown>(resp);
+  return getApiResponse<Document, unknown>(resp);
 }
 
 /**
@@ -241,7 +241,7 @@ export async function upload(
 }
 
 /**
- * Tell the backend to begin processing a batch of documents.
+ * Tell the backend to begin processing (or reprocessing) a batch of documents.
  * Only errors are returned here. A null response means success.
  */
 export async function process(

diff --git a/src/lib/api/tests/documents.test.ts b/src/lib/api/tests/documents.test.ts
@@ -51,12 +51,12 @@ const test = base.extend({
     await use(documents as DocumentResults);
   },
 
-  created: async ({}, use: Use<Document[]>) => {
+  created: async ({}, use: Use<Document>) => {
     const { default: created } = await import(
       "@/test/fixtures/documents/create.json"
     );
 
-    await use(created as Document[]);
+    await use(created[0] as Document);
   },
 
   pending: async ({}, use: Use<Pending[]>) => {
@@ -262,12 +262,12 @@ describe("document uploads and processing", () => {
       },
     }));
 
-    const docs: DocumentUpload[] = created.map((d) => ({
-      title: d.title,
-      access: d.access,
-      language: d.language,
-      original_extension: d.original_extension,
-    }));
+    const docs: DocumentUpload = {
+      title: created.title,
+      access: created.access,
+      language: created.language,
+      original_extension: created.original_extension,
+    };
 
     const { data } = await documents.create(docs, "token", mockFetch);
     expect(data).toMatchObject(created);
@@ -288,31 +288,29 @@ describe("document uploads and processing", () => {
   });
 
   test("documents.upload", async ({ created }) => {
-    created.forEach(async (doc) => {
-      const mockFetch = vi.fn().mockImplementation(async () => ({
-        ok: true,
-      }));
+    const mockFetch = vi.fn().mockImplementation(async () => ({
+      ok: true,
+    }));
 
-      const file = new File(
-        ["test content"],
-        "finalseasonal-allergies-pollen-and-mold-2023-en.pdf",
-        { type: "application/pdf" },
-      );
+    const file = new File(
+      ["test content"],
+      "finalseasonal-allergies-pollen-and-mold-2023-en.pdf",
+      { type: "application/pdf" },
+    );
 
-      const resp = await documents.upload(
-        new URL(doc.presigned_url),
-        file,
-        mockFetch,
-      );
+    const resp = await documents.upload(
+      new URL(created.presigned_url),
+      file,
+      mockFetch,
+    );
 
-      expect(resp.ok).toBeTruthy();
-      expect(mockFetch).toHaveBeenCalledWith(new URL(doc.presigned_url), {
-        body: file,
-        headers: {
-          "Content-Type": file.type,
-        },
-        method: "PUT",
-      });
+    expect(resp.ok).toBeTruthy();
+    expect(mockFetch).toHaveBeenCalledWith(new URL(created.presigned_url), {
+      body: file,
+      headers: {
+        "Content-Type": file.type,
+      },
+      method: "PUT",
     });
   });
 
@@ -325,7 +323,7 @@ describe("document uploads and processing", () => {
     }));
 
     const { data } = await documents.process(
-      created.map((d) => ({ id: d.id })),
+      [{ id: created.id }],
       "csrf_token",
       mockFetch,
     );
@@ -334,7 +332,7 @@ describe("document uploads and processing", () => {
     expect(mockFetch).toHaveBeenCalledWith(
       new URL("/api/documents/process/", DC_BASE),
       {
-        body: JSON.stringify(created.map((d) => ({ id: d.id }))),
+        body: JSON.stringify([{ id: created.id }]),
         credentials: "include",
         headers: {
           "Content-type": "application/json",