From 834f65863425a5e7762b8ca200854bf7b98711f9 Mon Sep 17 00:00:00 2001 From: Aofei Sheng Date: Mon, 13 May 2024 17:56:33 +0800 Subject: [PATCH] Switch downloading Kodo objects from unsigned to signed URLs Updates #412 --- spx-backend/cmd/spx-backend/gop_autogen.go | 52 +++++++++++++++---- spx-backend/cmd/spx-backend/project_yap.gox | 17 ++++++ spx-backend/internal/controller/controller.go | 42 ++++++++++++--- spx-gui/src/apis/util.ts | 11 +++- spx-gui/src/models/common/asset.ts | 6 +-- spx-gui/src/models/common/cloud.ts | 17 +++--- spx-gui/src/models/project.ts | 2 +- 7 files changed, 117 insertions(+), 30 deletions(-) diff --git a/spx-backend/cmd/spx-backend/gop_autogen.go b/spx-backend/cmd/spx-backend/gop_autogen.go index 1a003c95b..40611f7bd 100644 --- a/spx-backend/cmd/spx-backend/gop_autogen.go +++ b/spx-backend/cmd/spx-backend/gop_autogen.go @@ -502,29 +502,59 @@ func (this *project) MainEntry() { //line cmd/spx-backend/project_yap.gox:296:1 replyWithData(ctx, upInfo) }) -//line cmd/spx-backend/project_yap.gox:299:1 - var err error //line cmd/spx-backend/project_yap.gox:300:1 - logger := log.GetLogger() + this.Post("/util/signfiles", func(ctx *yap.Context) { //line cmd/spx-backend/project_yap.gox:301:1 - this.ctrl, err = controller.NewController(context.Background()) + if +//line cmd/spx-backend/project_yap.gox:301:1 + _, ok := ensureUser(ctx); !ok { //line cmd/spx-backend/project_yap.gox:302:1 + return + } +//line cmd/spx-backend/project_yap.gox:304:1 + files := &controller.SignedFiles{} +//line cmd/spx-backend/project_yap.gox:305:1 + if +//line cmd/spx-backend/project_yap.gox:305:1 + ok := parseJson(ctx, files); !ok { +//line cmd/spx-backend/project_yap.gox:306:1 + return + } +//line cmd/spx-backend/project_yap.gox:308:1 + signedFiles, err := this.ctrl.SignFiles(utils.GetCtx(ctx), files) +//line cmd/spx-backend/project_yap.gox:309:1 + if err != nil { +//line cmd/spx-backend/project_yap.gox:310:1 + handlerInnerError(ctx, err) +//line cmd/spx-backend/project_yap.gox:311:1 + return + } +//line cmd/spx-backend/project_yap.gox:313:1 + replyWithData(ctx, signedFiles) + }) +//line cmd/spx-backend/project_yap.gox:316:1 + var err error +//line cmd/spx-backend/project_yap.gox:317:1 + logger := log.GetLogger() +//line cmd/spx-backend/project_yap.gox:318:1 + this.ctrl, err = controller.NewController(context.Background()) +//line cmd/spx-backend/project_yap.gox:319:1 if err != nil { -//line cmd/spx-backend/project_yap.gox:303:1 +//line cmd/spx-backend/project_yap.gox:320:1 logger.Fatalln("New controller failed:", err) } -//line cmd/spx-backend/project_yap.gox:305:1 +//line cmd/spx-backend/project_yap.gox:322:1 user.CasdoorConfigInit() -//line cmd/spx-backend/project_yap.gox:306:1 +//line cmd/spx-backend/project_yap.gox:323:1 port := os.Getenv("PORT") -//line cmd/spx-backend/project_yap.gox:307:1 +//line cmd/spx-backend/project_yap.gox:324:1 if port == "" { -//line cmd/spx-backend/project_yap.gox:308:1 +//line cmd/spx-backend/project_yap.gox:325:1 port = ":8080" } -//line cmd/spx-backend/project_yap.gox:310:1 +//line cmd/spx-backend/project_yap.gox:327:1 logger.Printf("Listening to %s", port) -//line cmd/spx-backend/project_yap.gox:311:1 +//line cmd/spx-backend/project_yap.gox:328:1 this.Run(port, UserMiddleware, ReqIDMiddleware, CorsMiddleware) } func (this *project) Main() { diff --git a/spx-backend/cmd/spx-backend/project_yap.gox b/spx-backend/cmd/spx-backend/project_yap.gox index 16b3aef2f..0c45e5fe1 100644 --- a/spx-backend/cmd/spx-backend/project_yap.gox +++ b/spx-backend/cmd/spx-backend/project_yap.gox @@ -296,6 +296,23 @@ get "/util/upinfo", ctx => { replyWithData(ctx, upInfo) } +// Sign files for downloading +post "/util/signfiles", ctx => { + if _, ok := ensureUser(ctx); !ok { + return + } + files := &controller.SignedFiles{} + if ok := parseJson(ctx, files); !ok { + return + } + signedFiles, err := ctrl.SignFiles(utils.getCtx(ctx), files) + if err != nil { + handlerInnerError(ctx, err) + return + } + replyWithData(ctx, signedFiles) +} + var err error logger := log.GetLogger() ctrl, err = controller.NewController(context.Background()) diff --git a/spx-backend/internal/controller/controller.go b/spx-backend/internal/controller/controller.go index cfba0d245..2ffa734b5 100644 --- a/spx-backend/internal/controller/controller.go +++ b/spx-backend/internal/controller/controller.go @@ -7,6 +7,8 @@ import ( "fmt" "os" "regexp" + "strings" + "time" _ "image/png" @@ -35,8 +37,7 @@ type Controller struct { } type kodoConfig struct { - ak string - sk string + cred *qiniuAuth.Credentials bucket string bucketRegion string baseUrl string @@ -60,8 +61,10 @@ func NewController(ctx context.Context) (ret *Controller, err error) { } kc := &kodoConfig{ - ak: mustEnv(logger, "KODO_AK"), - sk: mustEnv(logger, "KODO_SK"), + cred: qiniuAuth.New( + mustEnv(logger, "KODO_AK"), + mustEnv(logger, "KODO_SK"), + ), bucket: mustEnv(logger, "KODO_BUCKET"), bucketRegion: mustEnv(logger, "KODO_BUCKET_REGION"), baseUrl: mustEnv(logger, "KODO_BASE_URL"), @@ -519,8 +522,7 @@ func (ctrl *Controller) GetUpInfo(ctx context.Context) (*UpInfo, error) { // the future. FsizeLimit: 25 << 20, // 25 MiB } - mac := qiniuAuth.New(ctrl.kodo.ak, ctrl.kodo.sk) - upToken := putPolicy.UploadToken(mac) + upToken := putPolicy.UploadToken(ctrl.kodo.cred) return &UpInfo{ Token: upToken, Expires: putPolicy.Expires, @@ -529,3 +531,31 @@ func (ctrl *Controller) GetUpInfo(ctx context.Context) (*UpInfo, error) { BaseUrl: ctrl.kodo.baseUrl, }, nil } + +type SignedFiles struct { + Files model.FileCollection `json:"files"` +} + +func (ctrl *Controller) SignFiles(ctx context.Context, files *SignedFiles) (*SignedFiles, error) { + const expires = 2 * 24 * 3600 // 2 days + logger := log.GetReqLogger(ctx) + signedFiles := &SignedFiles{ + Files: make(model.FileCollection, len(files.Files)), + } + for k, v := range files.Files { + if !strings.HasPrefix(v, ctrl.kodo.baseUrl) { + err := fmt.Errorf("unrecognized file url: %s", v) + logger.Printf("%v", err) + return nil, err + } + + // INFO: Workaround for browser caching issue with signed URLs, causing redundant downloads. + now := time.Now().UTC() + e := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC).Unix() + expires + + signedURL := fmt.Sprintf("%s?e=%d", v, e) + signedURL += "&token=" + ctrl.kodo.cred.Sign([]byte(signedURL)) + signedFiles.Files[k] = signedURL + } + return signedFiles, nil +} diff --git a/spx-gui/src/apis/util.ts b/spx-gui/src/apis/util.ts index 0fe5505a2..08fb9cdca 100644 --- a/spx-gui/src/apis/util.ts +++ b/spx-gui/src/apis/util.ts @@ -2,7 +2,7 @@ * @desc util-related APIs of spx-backend */ -import { client } from './common' +import { client, type FileUrls } from './common' export interface FormatError { column: number @@ -35,3 +35,12 @@ export type UpInfo = { export function getUpInfo() { return client.get('/util/upinfo') as Promise } + +export type SignedFiles = { + /** Signed files */ + files: FileUrls +} + +export function signFiles(files: FileUrls) { + return client.post('/util/signfiles', { files: files }) as Promise +} diff --git a/spx-gui/src/models/common/asset.ts b/spx-gui/src/models/common/asset.ts index 468db6128..725963660 100644 --- a/spx-gui/src/models/common/asset.ts +++ b/spx-gui/src/models/common/asset.ts @@ -28,7 +28,7 @@ export async function sprite2Asset(sprite: Sprite): Promise { } export async function asset2Sprite(assetData: PartialAssetData) { - const files = getFiles(assetData.files) + const files = await getFiles(assetData.files) const sprites = await Sprite.loadAll(files) if (sprites.length === 0) throw new Error('no sprite loaded') return sprites[0] @@ -49,7 +49,7 @@ export async function backdrop2Asset(backdrop: Backdrop): Promise { } export async function asset2Sound(assetData: PartialAssetData) { - const files = getFiles(assetData.files) + const files = await getFiles(assetData.files) const sounds = await Sound.loadAll(files) if (sounds.length === 0) throw new Error('no sound loaded') return sounds[0] diff --git a/spx-gui/src/models/common/cloud.ts b/spx-gui/src/models/common/cloud.ts index ef4faf2ee..dcf9b05fa 100644 --- a/spx-gui/src/models/common/cloud.ts +++ b/spx-gui/src/models/common/cloud.ts @@ -3,13 +3,13 @@ import { filename } from '@/utils/path' import { File, toNativeFile, type Files } from './file' import type { FileCollection, ProjectData } from '@/apis/project' import { IsPublic, addProject, getProject, updateProject } from '@/apis/project' -import { getUpInfo as getRawUpInfo, type UpInfo as RawUpInfo } from '@/apis/util' +import { getUpInfo as getRawUpInfo, type UpInfo as RawUpInfo, signFiles } from '@/apis/util' import { DefaultException } from '@/utils/exception' import type { Metadata } from '../project' export async function load(owner: string, name: string) { const projectData = await getProject(owner, name) - return parseProjectData(projectData) + return await parseProjectData(projectData) } export async function save(metadata: Metadata, files: Files) { @@ -21,11 +21,11 @@ export async function save(metadata: Metadata, files: Files) { const projectData = await (id != null ? updateProject(owner, name, { isPublic, files: fileUrls }) : addProject({ name, isPublic, files: fileUrls })) - return parseProjectData(projectData) + return await parseProjectData(projectData) } -export function parseProjectData({ files: fileUrls, ...metadata }: ProjectData) { - const files = getFiles(fileUrls) +export async function parseProjectData({ files: fileUrls, ...metadata }: ProjectData) { + const files = await getFiles(fileUrls) return { metadata, files } } @@ -40,10 +40,11 @@ export async function uploadFiles(files: Files): Promise { return fileUrls } -export function getFiles(fileUrls: FileCollection): Files { +export async function getFiles(fileUrls: FileCollection): Promise { + const { files: signedFileUrls } = await signFiles(fileUrls) const files: Files = {} - Object.keys(fileUrls).forEach((path) => { - const url = fileUrls[path] + Object.keys(signedFileUrls).forEach((path) => { + const url = signedFileUrls[path] files[path] = createFileWithUrl(filename(path), url) }) return files diff --git a/spx-gui/src/models/project.ts b/spx-gui/src/models/project.ts index 04df42d70..58b81c0e0 100644 --- a/spx-gui/src/models/project.ts +++ b/spx-gui/src/models/project.ts @@ -226,7 +226,7 @@ export class Project extends Disposble { const { metadata, files } = typeof ownerOrProjectData === 'string' ? await cloudHelper.load(ownerOrProjectData, name!) - : cloudHelper.parseProjectData(ownerOrProjectData) + : await cloudHelper.parseProjectData(ownerOrProjectData) await this.load(metadata, files) }