spx-gui: implement batched object URL processing and optimize lazy lo…

…ading

- Extended `util.makeObjectUrls` with batching to queue and merge object
  URL requests.
- Updated `cloud.universalUrlToWebUrl` to use batched processing.
- Refactored `cloud.getFiles` and file creation process to support lazy
  URL resolution, addressing potential URL expiry issues.

Fixes goplus#653

spx-gui/src/apis/util.test.ts

@@ -0,0 +1,101 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
+import { makeObjectUrls } from './util'
+import { client, type UniversalToWebUrlMap } from './common'
+
+vi.mock('./common', () => ({
+  client: {
+    post: vi.fn()
+  }
+}))
+
+describe('makeObjectUrls', () => {
+  beforeEach(() => {
+    vi.useFakeTimers()
+  })
+
+  afterEach(() => {
+    vi.useRealTimers()
+    vi.resetAllMocks()
+  })
+
+  it('should make object URLs correctly with immediate flag', async () => {
+    const mockResponse = {
+      objectUrls: {
+        'kodo://bucket/key1': 'https://bucket.example.com/key1',
+        'kodo://bucket/key2': 'https://bucket.example.com/key2'
+      } as UniversalToWebUrlMap
+    }
+    vi.mocked(client.post).mockResolvedValue(mockResponse)
+
+    const objects = ['kodo://bucket/key1', 'kodo://bucket/key2']
+    const objectUrls = await makeObjectUrls(objects, true)
+
+    expect(client.post).toHaveBeenCalledWith('/util/fileurls', { objects })
+    expect(objectUrls).toEqual(mockResponse.objectUrls)
+  })
+
+  it('should make multiple requests when immediate flag is true', async () => {
+    const mockResponse1 = {
+      objectUrls: {
+        'kodo://bucket/key1': 'https://bucket.example.com/key1',
+        'kodo://bucket/key2': 'https://bucket.example.com/key2'
+      } as UniversalToWebUrlMap
+    }
+    const mockResponse2 = {
+      objectUrls: {
+        'kodo://bucket/key3': 'https://bucket.example.com/key3'
+      } as UniversalToWebUrlMap
+    }
+    vi.mocked(client.post).mockResolvedValueOnce(mockResponse1).mockResolvedValueOnce(mockResponse2)
+
+    const objects1 = ['kodo://bucket/key1', 'kodo://bucket/key2']
+    const objects2 = ['kodo://bucket/key3']
+    const [objectUrls1, objectUrls2] = await Promise.all([
+      makeObjectUrls(objects1, true),
+      makeObjectUrls(objects2, true)
+    ])
+
+    expect(client.post).toHaveBeenCalledTimes(2)
+    expect(client.post).toHaveBeenNthCalledWith(1, '/util/fileurls', { objects: objects1 })
+    expect(client.post).toHaveBeenNthCalledWith(2, '/util/fileurls', { objects: objects2 })
+    expect(objectUrls1).toEqual(mockResponse1.objectUrls)
+    expect(objectUrls2).toEqual(mockResponse2.objectUrls)
+  })
+
+  it('should batch requests when immediate flag is false', async () => {
+    const mockResponse = {
+      objectUrls: {
+        'kodo://bucket/key1': 'https://bucket.example.com/key1',
+        'kodo://bucket/key2': 'https://bucket.example.com/key2',
+        'kodo://bucket/key3': 'https://bucket.example.com/key3'
+      } as UniversalToWebUrlMap
+    }
+    vi.mocked(client.post).mockResolvedValue(mockResponse)
+
+    const objects1 = ['kodo://bucket/key1', 'kodo://bucket/key2']
+    const objects2 = ['kodo://bucket/key3']
+    const promise1 = makeObjectUrls(objects1)
+    const promise2 = makeObjectUrls(objects2)
+    vi.advanceTimersByTime(2)
+    const [objectUrls1, objectUrls2] = await Promise.all([promise1, promise2])
+
+    expect(client.post).toHaveBeenCalledTimes(1)
+    expect(client.post).toHaveBeenCalledWith('/util/fileurls', {
+      objects: [...objects1, ...objects2]
+    })
+    expect(objectUrls1).toEqual(
+      Object.fromEntries(objects1.map((key) => [key, mockResponse.objectUrls[key]]))
+    )
+    expect(objectUrls2).toEqual(
+      Object.fromEntries(objects2.map((key) => [key, mockResponse.objectUrls[key]]))
+    )
+  })
+
+  it('should handle errors correctly', async () => {
+    const error = new Error('network error')
+    vi.mocked(client.post).mockRejectedValue(error)
+
+    const objects = ['kodo://bucket/key']
+    await expect(makeObjectUrls(objects, true)).rejects.toThrow(error)
+  })
+})

spx-gui/src/apis/util.ts

@@ -36,9 +36,31 @@ export function getUpInfo() {
   return client.get('/util/upinfo') as Promise<UpInfo>
 }
 
-export async function makeObjectUrls(objects: UniversalUrl[]) {
-  const res = (await client.post('/util/fileurls', { objects: objects })) as {
-    objectUrls: UniversalToWebUrlMap
+export const makeObjectUrls = (() => {
+  const make = async (objects: UniversalUrl[]) => {
+    const res = (await client.post('/util/fileurls', { objects: objects })) as {
+      objectUrls: UniversalToWebUrlMap
+    }
+    return res.objectUrls
   }
-  return res.objectUrls
-}
+
+  const batch = new Set<UniversalUrl>()
+  let batchPromise: Promise<UniversalToWebUrlMap> | null = null
+  const processBatch = () => {
+    const currentBatch = Array.from(batch)
+    batch.clear()
+    batchPromise = null
+    return make(currentBatch)
+  }
+
+  return async (objects: UniversalUrl[], immediate = false): Promise<UniversalToWebUrlMap> => {
+    if (immediate) return make(objects)
+
+    objects.forEach((url) => batch.add(url))
+    if (batchPromise == null) {
+      batchPromise = new Promise((resolve) => setTimeout(() => resolve(processBatch()), 15))
+    }
+    const objectUrls = await batchPromise
+    return Object.fromEntries(objects.map((url) => [url, objectUrls[url]]))
+  }
+})()

spx-gui/src/models/common/cloud.ts

@@ -1,6 +1,6 @@
 import * as qiniu from 'qiniu-js'
 import { filename } from '@/utils/path'
-import type { WebUrl, UniversalUrl, FileCollection, UniversalToWebUrlMap } from '@/apis/common'
+import type { WebUrl, UniversalUrl, FileCollection } from '@/apis/common'
 import type { ProjectData } from '@/apis/project'
 import { IsPublic, addProject, getProject, updateProject } from '@/apis/project'
 import { getUpInfo as getRawUpInfo, makeObjectUrls, type UpInfo as RawUpInfo } from '@/apis/util'
@@ -57,24 +57,10 @@ export async function saveFiles(
 }
 
 export async function getFiles(fileCollection: FileCollection): Promise<Files> {
-  const objectUniversalUrls = Object.values(fileCollection).filter(
-    (url) => new URL(url).protocol === fileUniversalUrlSchemes.kodo
-  )
-  const objectUrls: UniversalToWebUrlMap = objectUniversalUrls.length
-    ? await makeObjectUrls(objectUniversalUrls)
-    : {}
-
   const files: Files = {}
   Object.keys(fileCollection).forEach((path) => {
     const universalUrl = fileCollection[path]
-    let webUrl = universalUrl
-
-    const objectUrl = objectUrls[universalUrl]
-    if (objectUrl) {
-      webUrl = objectUrl
-    }
-
-    const file = createFileWithWebUrl(webUrl, filename(path))
+    const file = createFileWithUniversalUrl(universalUrl, filename(path))
     setUniversalUrl(file, universalUrl)
     files[path] = file
   })
@@ -92,11 +78,18 @@ function getUniversalUrl(file: File): UniversalUrl | null {
   return file.meta.universalUrl ?? null
 }
 
-export function createFileWithWebUrl(webUrl: WebUrl, name = filename(webUrl)) {
+function createFileWithUniversalUrl(url: UniversalUrl, name = filename(url)) {
   return new File(name, async () => {
+    const webUrl = await universalUrlToWebUrl(url)
     const resp = await fetch(webUrl)
-    const blob = await resp.blob()
-    return blob.arrayBuffer()
+    return resp.arrayBuffer()
+  })
+}
+
+export function createFileWithWebUrl(url: WebUrl, name = filename(url)) {
+  return new File(name, async () => {
+    const resp = await fetch(url)
+    return resp.arrayBuffer()
   })
 }