From cae77334f4988fed25dba6d4d361813d055b3c6b Mon Sep 17 00:00:00 2001
From: Aofei Sheng <aofei@aofeisheng.com>
Date: Tue, 23 Jul 2024 18:00:51 +0800
Subject: [PATCH] spx-gui: implement batched object URL processing and optimize
 lazy loading

- Introduced `util.batchedMakeObjectUrls` to queue and merge object URL
  requests.
- Updated `cloud.universalUrlToWebUrl` to use batched processing.
- Refactored `cloud.getFiles` and file creation process to support lazy
  URL resolution, addressing potential URL expiry issues.

Fixes #653
---
 spx-gui/src/apis/util.ts           | 38 ++++++++++++++++++++++++++-
 spx-gui/src/models/common/cloud.ts | 41 ++++++++++++++----------------
 2 files changed, 56 insertions(+), 23 deletions(-)

diff --git a/spx-gui/src/apis/util.ts b/spx-gui/src/apis/util.ts
index dd738bfe9..010e95680 100644
--- a/spx-gui/src/apis/util.ts
+++ b/spx-gui/src/apis/util.ts
@@ -2,6 +2,7 @@
  * @desc util-related APIs of spx-backend
  */
 
+import Mutex from '@/utils/mutex'
 import { client, type UniversalUrl, type UniversalToWebUrlMap } from './common'
 
 export interface FormatError {
@@ -36,9 +37,44 @@ export function getUpInfo() {
   return client.get('/util/upinfo') as Promise<UpInfo>
 }
 
-export async function makeObjectUrls(objects: UniversalUrl[]) {
+export async function makeObjectUrls(objects: UniversalUrl[]): Promise<UniversalToWebUrlMap> {
   const res = (await client.post('/util/fileurls', { objects: objects })) as {
     objectUrls: UniversalToWebUrlMap
   }
   return res.objectUrls
 }
+
+export const batchedMakeObjectUrls = (() => {
+  const batch = new Set<UniversalUrl>()
+  let pendingPromise: Promise<UniversalToWebUrlMap> | null = null
+  const mutex = new Mutex()
+
+  const processBatch = async () => {
+    let currentBatch!: UniversalUrl[]
+    await mutex.runExclusive(() => {
+      currentBatch = Array.from(batch)
+      batch.clear()
+      pendingPromise = null
+    })
+
+    try {
+      return await makeObjectUrls(currentBatch)
+    } catch (e) {
+      throw Object.fromEntries(currentBatch.map((url) => [url, e]))
+    }
+  }
+
+  return async (objects: UniversalUrl[]): Promise<UniversalToWebUrlMap> => {
+    let promise!: Promise<UniversalToWebUrlMap>
+    await mutex.runExclusive(() => {
+      objects.forEach((url) => batch.add(url))
+      if (pendingPromise == null) {
+        pendingPromise = new Promise((resolve) => setTimeout(() => resolve(processBatch()), 1))
+      }
+      promise = pendingPromise
+    })
+
+    const objectUrls = await promise
+    return Object.fromEntries(objects.map((url) => [url, objectUrls[url]]))
+  }
+})()
diff --git a/spx-gui/src/models/common/cloud.ts b/spx-gui/src/models/common/cloud.ts
index 9d04a55ff..c7ebede32 100644
--- a/spx-gui/src/models/common/cloud.ts
+++ b/spx-gui/src/models/common/cloud.ts
@@ -1,9 +1,13 @@
 import * as qiniu from 'qiniu-js'
 import { filename } from '@/utils/path'
-import type { WebUrl, UniversalUrl, FileCollection, UniversalToWebUrlMap } from '@/apis/common'
+import type { WebUrl, UniversalUrl, FileCollection } from '@/apis/common'
 import type { ProjectData } from '@/apis/project'
 import { IsPublic, addProject, getProject, updateProject } from '@/apis/project'
-import { getUpInfo as getRawUpInfo, makeObjectUrls, type UpInfo as RawUpInfo } from '@/apis/util'
+import {
+  batchedMakeObjectUrls,
+  getUpInfo as getRawUpInfo,
+  type UpInfo as RawUpInfo
+} from '@/apis/util'
 import { DefaultException } from '@/utils/exception'
 import type { Metadata } from '../project'
 import { File, toNativeFile, toText, type Files } from './file'
@@ -57,24 +61,10 @@ export async function saveFiles(
 }
 
 export async function getFiles(fileCollection: FileCollection): Promise<Files> {
-  const objectUniversalUrls = Object.values(fileCollection).filter(
-    (url) => new URL(url).protocol === fileUniversalUrlSchemes.kodo
-  )
-  const objectUrls: UniversalToWebUrlMap = objectUniversalUrls.length
-    ? await makeObjectUrls(objectUniversalUrls)
-    : {}
-
   const files: Files = {}
-  Object.keys(fileCollection).forEach((path) => {
+  Object.keys(fileCollection).forEach(async (path) => {
     const universalUrl = fileCollection[path]
-    let webUrl = universalUrl
-
-    const objectUrl = objectUrls[universalUrl]
-    if (objectUrl) {
-      webUrl = objectUrl
-    }
-
-    const file = createFileWithWebUrl(webUrl, filename(path))
+    const file = createFileWithUniversalUrl(universalUrl, filename(path))
     setUniversalUrl(file, universalUrl)
     files[path] = file
   })
@@ -92,11 +82,18 @@ function getUniversalUrl(file: File): UniversalUrl | null {
   return file.meta.universalUrl ?? null
 }
 
-export function createFileWithWebUrl(webUrl: WebUrl, name = filename(webUrl)) {
+function createFileWithUniversalUrl(url: UniversalUrl, name = filename(url)) {
   return new File(name, async () => {
+    const webUrl = await universalUrlToWebUrl(url)
     const resp = await fetch(webUrl)
-    const blob = await resp.blob()
-    return blob.arrayBuffer()
+    return resp.arrayBuffer()
+  })
+}
+
+export function createFileWithWebUrl(url: WebUrl, name = filename(url)) {
+  return new File(name, async () => {
+    const resp = await fetch(url)
+    return resp.arrayBuffer()
   })
 }
 
@@ -108,7 +105,7 @@ export async function getWebUrl(file: File) {
 async function universalUrlToWebUrl(universalUrl: UniversalUrl) {
   const { protocol } = new URL(universalUrl)
   if (protocol !== fileUniversalUrlSchemes.kodo) return universalUrl
-  const map = await makeObjectUrls([universalUrl])
+  const map = await batchedMakeObjectUrls([universalUrl])
   return map[universalUrl]
 }