From 153329f73adbd028c895c77783b46cc29f1de561 Mon Sep 17 00:00:00 2001 From: bir Date: Thu, 19 Dec 2024 17:43:19 +0100 Subject: [PATCH] Request multiple blobs of different instances in same request #8069 Co-authored-by: ivk --- packages/tutanota-utils/lib/MapUtils.ts | 12 + packages/tutanota-utils/lib/index.ts | 2 +- .../api/worker/facades/lazy/BlobFacade.ts | 136 ++++------ .../worker/facades/lazy/MailExportFacade.ts | 15 +- .../api/worker/facades/BlobFacadeTest.ts | 253 +++++++++++++++++- .../worker/facades/MailExportFacadeTest.ts | 6 +- 6 files changed, 327 insertions(+), 97 deletions(-) diff --git a/packages/tutanota-utils/lib/MapUtils.ts b/packages/tutanota-utils/lib/MapUtils.ts index f7c28928a62f..a6e4190555d3 100644 --- a/packages/tutanota-utils/lib/MapUtils.ts +++ b/packages/tutanota-utils/lib/MapUtils.ts @@ -41,3 +41,15 @@ export function deleteMapEntry(map: ReadonlyMap, key: K): Map newMap.delete(key) return newMap } + +/** + * Convert values of {@param map} using {@param mapper} like {@link Array.prototype.map}, + */ +export function mapMap(map: ReadonlyMap, mapper: (value: V) => R): Map { + const resultMap = new Map() + for (const [key, oldValue] of map) { + const newValue = mapper(oldValue) + resultMap.set(key, newValue) + } + return resultMap +} diff --git a/packages/tutanota-utils/lib/index.ts b/packages/tutanota-utils/lib/index.ts index 9538df411d57..9de0d98f5aee 100644 --- a/packages/tutanota-utils/lib/index.ts +++ b/packages/tutanota-utils/lib/index.ts @@ -93,7 +93,7 @@ export { } from "./Encoding.js" export type { Base64, Base64Ext, Base64Url, Hex } from "./Encoding.js" export { LazyLoaded } from "./LazyLoaded.js" -export { mergeMaps, getFromMap, addMapEntry, deleteMapEntry } from "./MapUtils.js" +export { mergeMaps, getFromMap, addMapEntry, deleteMapEntry, mapMap } from "./MapUtils.js" export { pMap } from "./PromiseMap.js" export type { Mapper } from "./PromiseMap.js" export { mapInCallContext, promiseMap, promiseMapCompat, PromisableWrapper, delay, tap, ofClass, promiseFilter, settledThen } from "./PromiseUtils.js" diff --git a/src/common/api/worker/facades/lazy/BlobFacade.ts b/src/common/api/worker/facades/lazy/BlobFacade.ts index 6df05cde446e..8047cbb39870 100644 --- a/src/common/api/worker/facades/lazy/BlobFacade.ts +++ b/src/common/api/worker/facades/lazy/BlobFacade.ts @@ -8,6 +8,7 @@ import { getFirstOrThrow, groupBy, isEmpty, + mapMap, neverNull, promiseMap, splitUint8ArrayInChunks, @@ -26,7 +27,7 @@ import type { AesApp } from "../../../../native/worker/AesApp.js" import { InstanceMapper } from "../../crypto/InstanceMapper.js" import { Blob, BlobReferenceTokenWrapper, createBlobReferenceTokenWrapper } from "../../../entities/sys/TypeRefs.js" import { FileReference } from "../../../common/utils/FileUtils.js" -import { handleRestError } from "../../../common/error/RestError.js" +import { handleRestError, NotFoundError } from "../../../common/error/RestError.js" import { ProgrammingError } from "../../../common/error/ProgrammingError.js" import { IServiceExecutor } from "../../../common/ServiceRequest.js" import { BlobGetInTypeRef, BlobPostOut, BlobPostOutTypeRef, BlobServerAccessInfo, createBlobGetIn, createBlobId } from "../../../entities/storage/TypeRefs.js" @@ -153,59 +154,53 @@ export class BlobFacade { } /** - * Downloads multiple blobs, decrypts and joins them to unencrypted binary data. - * - * @param archiveDataType - * @param referencingInstances that directly references the blobs - * @returns Uint8Array unencrypted binary data + * Downloads blobs of all {@param referencingInstances}, decrypts them and joins them to unencrypted binaries. + * If some blobs are not found the result will contain {@code null}. + * @returns Map from instance id to the decrypted and concatenated contents of the referenced blobs */ - async downloadAndDecryptMultipleInstances( + async downloadAndDecryptBlobsOfMultipleInstances( archiveDataType: ArchiveDataType, referencingInstances: BlobReferencingInstance[], blobLoadOptions: BlobLoadOptions = {}, - ): Promise>> { - if (isEmpty(referencingInstances)) { - return new Map() - } - if (referencingInstances.length === 1) { - const downloaded = this.downloadAndDecrypt(archiveDataType, referencingInstances[0], blobLoadOptions) - return new Map([[referencingInstances[0].elementId, downloaded]]) - } - + ): Promise> { // If a mail has multiple attachments, we cannot assume they are all on the same archive. - const allArchives: Map = new Map() - for (const instance of referencingInstances) { - const archiveId = instance.blobs[0].archiveId - const archive = allArchives.get(archiveId) ?? [] - archive.push(instance) - allArchives.set(archiveId, archive) - } - - // file to data - const result: Map> = new Map() - - for (const [archive, instances] of allArchives.entries()) { - let latestAccessInfo: Promise | null = null - - for (const instance of instances) { + // But all blobs of a single attachment should be in the same archive + const instancesByArchive = groupBy(referencingInstances, (instance) => getFirstOrThrow(instance.blobs).archiveId) + + // instance id to data + const result: Map = new Map() + + for (const [_, instances] of instancesByArchive.entries()) { + // request a token for all instances of the archive + // download all blobs from all instances for this archive + const allBlobs = instances.flatMap((instance) => instance.blobs) + const doBlobRequest = async () => { + const accessInfo = await this.blobAccessTokenFacade.requestReadTokenMultipleInstances(archiveDataType, instances, blobLoadOptions) + return this.downloadBlobsOfOneArchive(allBlobs, accessInfo, blobLoadOptions) + } + const doEvictToken = () => { + for (const instance of instances) { + this.blobAccessTokenFacade.evictReadBlobsToken(instance) + } + } + const encryptedBlobsOfAllInstances = await doBlobRequestWithRetry(doBlobRequest, doEvictToken) + // sort blobs by the instance + instanceLoop: for (const instance of instances) { + // get the key of the instance const sessionKey = await this.resolveSessionKey(instance.entity) - - const doBlobRequest = async () => { - if (latestAccessInfo == null) { - latestAccessInfo = this.blobAccessTokenFacade.requestReadTokenMultipleInstances(ArchiveDataType.Attachments, instances, blobLoadOptions) + // decrypt blobs of the instance and concatenate them + const decryptedChunks: Uint8Array[] = [] + for (const blob of instance.blobs) { + const encryptedChunk = encryptedBlobsOfAllInstances.get(blob.blobId) + if (encryptedChunk == null) { + result.set(instance.elementId, null) + continue instanceLoop } - const accessInfoToUse = await latestAccessInfo - return promiseMap(instance.blobs, (blob) => this.downloadAndDecryptChunk(blob, accessInfoToUse, sessionKey, blobLoadOptions)) + decryptedChunks.push(aesDecrypt(sessionKey, encryptedChunk)) } - const doEvictToken = () => { - this.blobAccessTokenFacade.evictReadBlobsToken(instance) - latestAccessInfo = null - } - const request = doBlobRequestWithRetry(doBlobRequest, doEvictToken) - result.set( - instance.elementId, - request.then((data) => concat(...data)), - ) + const decryptedData = concat(...decryptedChunks) + // return Map of instance id -> blob data + result.set(instance.elementId, decryptedData) } } @@ -349,44 +344,23 @@ export class BlobFacade { return createBlobReferenceTokenWrapper({ blobReferenceToken }) } - private async downloadAndDecryptChunk( - blob: Blob, + private async downloadAndDecryptMultipleBlobsOfArchive( + blobs: readonly Blob[], blobServerAccessInfo: BlobServerAccessInfo, sessionKey: AesKey, blobLoadOptions: BlobLoadOptions, - ): Promise { - const { archiveId, blobId } = blob - const getData = createBlobGetIn({ - archiveId, - blobId, - blobIds: [], - }) - const BlobGetInTypeModel = await resolveTypeReference(BlobGetInTypeRef) - const literalGetData = await this.instanceMapper.encryptAndMapToLiteral(BlobGetInTypeModel, getData, null) - const body = JSON.stringify(literalGetData) - const queryParams = await this.blobAccessTokenFacade.createQueryParams(blobServerAccessInfo, {}, BlobGetInTypeRef) - return tryServers( - blobServerAccessInfo.servers, - async (serverUrl) => { - const data = await this.restClient.request(BLOB_SERVICE_REST_PATH, HttpMethod.GET, { - queryParams: queryParams, - body, - responseType: MediaType.Binary, - baseUrl: serverUrl, - noCORS: true, - headers: blobLoadOptions.extraHeaders, - suspensionBehavior: blobLoadOptions.suspensionBehavior, - }) - return aesDecrypt(sessionKey, data) - }, - `can't download from server `, - ) + ): Promise> { + const mapWithEncryptedBlobs = await this.downloadBlobsOfOneArchive(blobs, blobServerAccessInfo, blobLoadOptions) + return mapMap(mapWithEncryptedBlobs, (blob) => aesDecrypt(sessionKey, blob)) } - private async downloadAndDecryptMultipleBlobsOfArchive( + /** + * Download blobs of a single archive in a single request + * @return map from blob id to the data + */ + private async downloadBlobsOfOneArchive( blobs: readonly Blob[], blobServerAccessInfo: BlobServerAccessInfo, - sessionKey: AesKey, blobLoadOptions: BlobLoadOptions, ): Promise> { if (isEmpty(blobs)) { @@ -420,13 +394,7 @@ export class BlobFacade { }, `can't download from server `, ) - const mapWithEncryptedBlobs = parseMultipleBlobsResponse(concatBinaryData) - const mapWithDecryptedBlobs = new Map() - for (const [blobId, encryptedData] of mapWithEncryptedBlobs) { - const decryptedData = aesDecrypt(sessionKey, encryptedData) - mapWithDecryptedBlobs.set(blobId, decryptedData) - } - return mapWithDecryptedBlobs + return parseMultipleBlobsResponse(concatBinaryData) } private async downloadAndDecryptChunkNative(blob: Blob, blobServerAccessInfo: BlobServerAccessInfo, sessionKey: AesKey): Promise { diff --git a/src/common/api/worker/facades/lazy/MailExportFacade.ts b/src/common/api/worker/facades/lazy/MailExportFacade.ts index 0ffef8d06fe1..f3740b6dd89d 100644 --- a/src/common/api/worker/facades/lazy/MailExportFacade.ts +++ b/src/common/api/worker/facades/lazy/MailExportFacade.ts @@ -7,7 +7,7 @@ import { BlobFacade } from "./BlobFacade.js" import { CryptoFacade } from "../../crypto/CryptoFacade.js" import { createReferencingInstance } from "../../../common/utils/BlobUtils.js" import { MailExportTokenFacade } from "./MailExportTokenFacade.js" -import { assertNotNull, isNotNull, promiseMap } from "@tutao/tutanota-utils" +import { assertNotNull, isNotNull } from "@tutao/tutanota-utils" import { NotFoundError } from "../../../common/error/RestError" import { elementIdPart } from "../../../common/utils/EntityUtils" @@ -50,14 +50,17 @@ export class MailExportFacade { const downloads = await this.mailExportTokenFacade.loadWithToken((token) => { const referencingInstances = attachmentsWithKeys.map(createReferencingInstance) - return this.blobFacade.downloadAndDecryptMultipleInstances(ArchiveDataType.Attachments, referencingInstances, this.options(token)) + return this.blobFacade.downloadAndDecryptBlobsOfMultipleInstances(ArchiveDataType.Attachments, referencingInstances, this.options(token)) }) - const attachmentData = await promiseMap(downloads.entries(), async ([fileId, download]) => { + const attachmentData = Array.from(downloads.entries()).map(([fileId, bytes]) => { try { - const bytes = await download - const attachment = assertNotNull(attachmentsWithKeys.find((attachment) => elementIdPart(attachment._id) === fileId)) - return convertToDataFile(attachment, bytes) + if (bytes == null) { + return null + } else { + const attachment = assertNotNull(attachmentsWithKeys.find((attachment) => elementIdPart(attachment._id) === fileId)) + return convertToDataFile(attachment, bytes) + } } catch (e) { if (e instanceof NotFoundError) { return null diff --git a/test/tests/api/worker/facades/BlobFacadeTest.ts b/test/tests/api/worker/facades/BlobFacadeTest.ts index 533d8057ea70..859adbfd0f9e 100644 --- a/test/tests/api/worker/facades/BlobFacadeTest.ts +++ b/test/tests/api/worker/facades/BlobFacadeTest.ts @@ -1,6 +1,6 @@ import o from "@tutao/otest" import { BLOB_SERVICE_REST_PATH, BlobFacade, parseMultipleBlobsResponse } from "../../../../../src/common/api/worker/facades/lazy/BlobFacade.js" -import { RestClient } from "../../../../../src/common/api/worker/rest/RestClient.js" +import { RestClient, RestClientOptions } from "../../../../../src/common/api/worker/rest/RestClient.js" import { SuspensionHandler } from "../../../../../src/common/api/worker/SuspensionHandler.js" import { NativeFileApp } from "../../../../../src/common/native/common/FileApp.js" import { AesApp } from "../../../../../src/common/native/worker/AesApp.js" @@ -18,11 +18,11 @@ import { CryptoFacade } from "../../../../../src/common/api/worker/crypto/Crypto import { FileReference } from "../../../../../src/common/api/common/utils/FileUtils.js" import { assertThrows } from "@tutao/tutanota-test-utils" import { ProgrammingError } from "../../../../../src/common/api/common/error/ProgrammingError.js" -import { BlobPostOutTypeRef, BlobServerAccessInfoTypeRef, BlobServerUrlTypeRef } from "../../../../../src/common/api/entities/storage/TypeRefs.js" +import { BlobGetIn, BlobPostOutTypeRef, BlobServerAccessInfoTypeRef, BlobServerUrlTypeRef } from "../../../../../src/common/api/entities/storage/TypeRefs.js" import type { AuthDataProvider } from "../../../../../src/common/api/worker/facades/UserFacade.js" import { BlobAccessTokenFacade } from "../../../../../src/common/api/worker/facades/BlobAccessTokenFacade.js" import { DateProvider } from "../../../../../src/common/api/common/DateProvider.js" -import { elementIdPart, listIdPart } from "../../../../../src/common/api/common/utils/EntityUtils.js" +import { elementIdPart, getElementId, listIdPart } from "../../../../../src/common/api/common/utils/EntityUtils.js" import { createTestEntity } from "../../../TestUtils.js" import { DefaultEntityRestCache } from "../../../../../src/common/api/worker/rest/DefaultEntityRestCache.js" import { BlobReferencingInstance } from "../../../../../src/common/api/common/utils/BlobUtils.js" @@ -50,6 +50,7 @@ o.spec("BlobFacade test", function () { let cryptoFacadeMock: CryptoFacade let dateProvider: DateProvider let file: TutanotaFile + let anotherFile: TutanotaFile o.beforeEach(function () { authDataProvider = object() @@ -65,6 +66,7 @@ o.spec("BlobFacade test", function () { const mimeType = "text/plain" const name = "fileName" file = createTestEntity(FileTypeRef, { name, mimeType, _id: ["fileListId", "fileElementId"] }) + anotherFile = createTestEntity(FileTypeRef, { name, mimeType, _id: ["fileListId", "anotherFileElementId"] }) blobFacade = new BlobFacade( authDataProvider, @@ -331,6 +333,251 @@ o.spec("BlobFacade test", function () { }) }) + o.spec("downloadAndDecryptBlobsOfMultipleInstances", function () { + o.test("when passed multiple instances of the same archives it downloads and decrypts the data", async function () { + const sessionKey = aes256RandomKey() + const anothersessionKey = aes256RandomKey() + const blobData1 = new Uint8Array([1, 2, 3]) + const blobId1 = "--------0s-1" + file.blobs.push(createTestEntity(BlobTypeRef, { blobId: blobId1, size: String(65) })) + const encryptedBlobData1 = aesEncrypt(sessionKey, blobData1, generateIV(), true, true) + + const blobData2 = new Uint8Array([4, 5, 6, 7, 8, 9]) + const blobId2 = "--------0s-2" + file.blobs.push(createTestEntity(BlobTypeRef, { blobId: blobId2, size: String(65) })) + const encryptedBlobData2 = aesEncrypt(sessionKey, blobData2, generateIV(), true, true) + + const blobData3 = new Uint8Array([10, 11, 12, 13, 14, 15]) + const blobId3 = "--------0s-3" + anotherFile.blobs.push(createTestEntity(BlobTypeRef, { blobId: blobId3, size: String(65) })) + const encryptedBlobData3 = aesEncrypt(anothersessionKey, blobData3, generateIV(), true, true) + + const blobAccessInfo = createTestEntity(BlobServerAccessInfoTypeRef, { + blobAccessToken: "123", + servers: [createTestEntity(BlobServerUrlTypeRef, { url: "someBaseUrl" })], + }) + when( + blobAccessTokenFacade.requestReadTokenMultipleInstances( + archiveDataType, + [wrapTutanotaFile(file), wrapTutanotaFile(anotherFile)], + matchers.anything(), + ), + ).thenResolve(blobAccessInfo) + when(blobAccessTokenFacade.createQueryParams(blobAccessInfo, anything(), anything())).thenResolve({ + baseUrl: "someBaseUrl", + blobAccessToken: blobAccessInfo.blobAccessToken, + }) + when(cryptoFacadeMock.resolveSessionKeyForInstance(file)).thenResolve(sessionKey) + when(cryptoFacadeMock.resolveSessionKeyForInstance(anotherFile)).thenResolve(anothersessionKey) + const requestBody = { "request-body": true } + when(instanceMapperMock.encryptAndMapToLiteral(anything(), anything(), anything())).thenResolve(requestBody) + // data size is 65 (16 data block, 16 iv, 32 hmac, 1 byte for mac marking) + const blobSizeBinary = new Uint8Array([0, 0, 0, 65]) + const blobResponse = concat( + // blob id + base64ToUint8Array(base64ExtToBase64(blobId1)), + // blob hash + new Uint8Array([1, 2, 3, 4, 5, 6]), + // blob size + blobSizeBinary, + // blob data + encryptedBlobData1, + // blob id + base64ToUint8Array(base64ExtToBase64(blobId2)), + // blob hash + new Uint8Array([6, 5, 4, 3, 2, 1]), + // blob size + blobSizeBinary, + // blob data + encryptedBlobData2, + //blodId + base64ToUint8Array(base64ExtToBase64(blobId3)), + // blob hash + new Uint8Array([7, 8, 9, 10, 11, 12]), + // blob size + blobSizeBinary, + // blob data + encryptedBlobData3, + ) + when(restClientMock.request(BLOB_SERVICE_REST_PATH, HttpMethod.GET, anything())).thenResolve(blobResponse) + + const result = await blobFacade.downloadAndDecryptBlobsOfMultipleInstances(archiveDataType, [wrapTutanotaFile(file), wrapTutanotaFile(anotherFile)]) + + o(result).deepEquals( + new Map([ + [getElementId(file), concat(blobData1, blobData2)], + [getElementId(anotherFile), blobData3], + ]), + ) + }) + o.test("when passed multiple instances of the different archives it downloads and decrypts the data", async function () { + const sessionKey = aes256RandomKey() + const anothersessionKey = aes256RandomKey() + const blobData1 = new Uint8Array([1, 2, 3]) + const blobId1 = "--------0s-1" + file.blobs.push(createTestEntity(BlobTypeRef, { blobId: blobId1, size: String(65), archiveId: "archiveId1" })) + const encryptedBlobData1 = aesEncrypt(sessionKey, blobData1, generateIV(), true, true) + + const blobData2 = new Uint8Array([4, 5, 6, 7, 8, 9]) + const blobId2 = "--------0s-2" + file.blobs.push(createTestEntity(BlobTypeRef, { blobId: blobId2, size: String(65), archiveId: "archiveId1" })) + const encryptedBlobData2 = aesEncrypt(sessionKey, blobData2, generateIV(), true, true) + + const blobData3 = new Uint8Array([10, 11, 12, 13, 14, 15]) + const blobId3 = "--------0s-3" + anotherFile.blobs.push(createTestEntity(BlobTypeRef, { blobId: blobId3, size: String(65), archiveId: "archiveId2" })) + const encryptedBlobData3 = aesEncrypt(anothersessionKey, blobData3, generateIV(), true, true) + + const blobAccessInfo = createTestEntity(BlobServerAccessInfoTypeRef, { + blobAccessToken: "123", + servers: [createTestEntity(BlobServerUrlTypeRef, { url: "someBaseUrl" })], + }) + when(blobAccessTokenFacade.requestReadTokenMultipleInstances(archiveDataType, [wrapTutanotaFile(file)], matchers.anything())).thenResolve( + blobAccessInfo, + ) + when(blobAccessTokenFacade.requestReadTokenMultipleInstances(archiveDataType, [wrapTutanotaFile(anotherFile)], matchers.anything())).thenResolve( + blobAccessInfo, + ) + when(blobAccessTokenFacade.createQueryParams(blobAccessInfo, anything(), anything())).thenResolve({ + baseUrl: "someBaseUrl", + blobAccessToken: blobAccessInfo.blobAccessToken, + }) + when(cryptoFacadeMock.resolveSessionKeyForInstance(file)).thenResolve(sessionKey) + when(cryptoFacadeMock.resolveSessionKeyForInstance(anotherFile)).thenResolve(anothersessionKey) + const requestBody1 = { body: 1 } + when( + instanceMapperMock.encryptAndMapToLiteral( + anything(), + matchers.argThat((inData: BlobGetIn) => inData.archiveId === "archiveId1" && inData.blobIds.length == 2), + anything(), + ), + ).thenResolve(requestBody1) + const requestBody2 = { body: 2 } + when( + instanceMapperMock.encryptAndMapToLiteral( + anything(), + matchers.argThat((inData: BlobGetIn) => inData.archiveId === "archiveId2" && inData.blobIds.length == 1), + anything(), + ), + ).thenResolve(requestBody2) + // data size is 65 (16 data block, 16 iv, 32 hmac, 1 byte for mac marking) + const blobSizeBinary = new Uint8Array([0, 0, 0, 65]) + const blobResponse1 = concat( + // blob id + base64ToUint8Array(base64ExtToBase64(blobId1)), + // blob hash + new Uint8Array([1, 2, 3, 4, 5, 6]), + // blob size + blobSizeBinary, + // blob data + encryptedBlobData1, + // blob id + base64ToUint8Array(base64ExtToBase64(blobId2)), + // blob hash + new Uint8Array([6, 5, 4, 3, 2, 1]), + // blob size + blobSizeBinary, + // blob data + encryptedBlobData2, + ) + + const blobResponse2 = concat( + //blodId + base64ToUint8Array(base64ExtToBase64(blobId3)), + // blob hash + new Uint8Array([7, 8, 9, 10, 11, 12]), + // blob size + blobSizeBinary, + // blob data + encryptedBlobData3, + ) + when( + restClientMock.request( + BLOB_SERVICE_REST_PATH, + HttpMethod.GET, + matchers.argThat((options: RestClientOptions) => options.body && JSON.parse(options.body as string).body === 1), + ), + ).thenResolve(blobResponse1) + when( + restClientMock.request( + BLOB_SERVICE_REST_PATH, + HttpMethod.GET, + matchers.argThat((options: RestClientOptions) => options.body && JSON.parse(options.body as string).body === 2), + ), + ).thenResolve(blobResponse2) + + const result = await blobFacade.downloadAndDecryptBlobsOfMultipleInstances(archiveDataType, [wrapTutanotaFile(file), wrapTutanotaFile(anotherFile)]) + + o(result).deepEquals( + new Map([ + [getElementId(file), concat(blobData1, blobData2)], + [getElementId(anotherFile), blobData3], + ]), + ) + }) + o.test("when passed multiple instances of the same archives but one blob is missing it downloads and decrypts the rest", async function () { + const sessionKey = aes256RandomKey() + const anothersessionKey = aes256RandomKey() + const blobData1 = new Uint8Array([1, 2, 3]) + const blobId1 = "--------0s-1" + file.blobs.push(createTestEntity(BlobTypeRef, { blobId: blobId1, size: String(65) })) + const encryptedBlobData1 = aesEncrypt(sessionKey, blobData1, generateIV(), true, true) + + const blobData2 = new Uint8Array([4, 5, 6, 7, 8, 9]) + const blobId2 = "--------0s-2" + file.blobs.push(createTestEntity(BlobTypeRef, { blobId: blobId2, size: String(65) })) + const encryptedBlobData2 = aesEncrypt(sessionKey, blobData2, generateIV(), true, true) + + const blobId3 = "--------0s-3" + anotherFile.blobs.push(createTestEntity(BlobTypeRef, { blobId: blobId3, size: String(65) })) + + const blobAccessInfo = createTestEntity(BlobServerAccessInfoTypeRef, { + blobAccessToken: "123", + servers: [createTestEntity(BlobServerUrlTypeRef, { url: "someBaseUrl" })], + }) + when( + blobAccessTokenFacade.requestReadTokenMultipleInstances( + archiveDataType, + [wrapTutanotaFile(file), wrapTutanotaFile(anotherFile)], + matchers.anything(), + ), + ).thenResolve(blobAccessInfo) + when(blobAccessTokenFacade.createQueryParams(blobAccessInfo, anything(), anything())).thenResolve({ + baseUrl: "someBaseUrl", + blobAccessToken: blobAccessInfo.blobAccessToken, + }) + when(cryptoFacadeMock.resolveSessionKeyForInstance(file)).thenResolve(sessionKey) + when(cryptoFacadeMock.resolveSessionKeyForInstance(anotherFile)).thenResolve(anothersessionKey) + const requestBody = { "request-body": true } + when(instanceMapperMock.encryptAndMapToLiteral(anything(), anything(), anything())).thenResolve(requestBody) + // data size is 65 (16 data block, 16 iv, 32 hmac, 1 byte for mac marking) + const blobSizeBinary = new Uint8Array([0, 0, 0, 65]) + const blobResponse = concat( + // blob id + base64ToUint8Array(base64ExtToBase64(blobId1)), + // blob hash + new Uint8Array([1, 2, 3, 4, 5, 6]), + // blob size + blobSizeBinary, + // blob data + encryptedBlobData1, + // blob id + base64ToUint8Array(base64ExtToBase64(blobId2)), + // blob hash + new Uint8Array([6, 5, 4, 3, 2, 1]), + // blob size + blobSizeBinary, + // blob data + encryptedBlobData2, + ) + when(restClientMock.request(BLOB_SERVICE_REST_PATH, HttpMethod.GET, anything())).thenResolve(blobResponse) + + const result = await blobFacade.downloadAndDecryptBlobsOfMultipleInstances(archiveDataType, [wrapTutanotaFile(file), wrapTutanotaFile(anotherFile)]) + + o(result).deepEquals(new Map([[getElementId(file), concat(blobData1, blobData2)]])) + }) + }) + o.spec("parseMultipleBlobsResponse", function () { o.test("parses two blobs", function () { // Blob id OETv4XP----0 hash [3, -112, 88, -58, -14, -64] bytes [1, 2, 3] diff --git a/test/tests/api/worker/facades/MailExportFacadeTest.ts b/test/tests/api/worker/facades/MailExportFacadeTest.ts index 6305673b51f7..91cc6819549b 100644 --- a/test/tests/api/worker/facades/MailExportFacadeTest.ts +++ b/test/tests/api/worker/facades/MailExportFacadeTest.ts @@ -73,7 +73,7 @@ o.spec("MailExportFacade", () => { when(cryptoFacade.enforceSessionKeyUpdateIfNeeded(mail1, mailAttachments)).thenResolve(mailAttachments) when( - blobFacade.downloadAndDecryptMultipleInstances( + blobFacade.downloadAndDecryptBlobsOfMultipleInstances( ArchiveDataType.Attachments, [createReferencingInstance(mailAttachments[0]), createReferencingInstance(mailAttachments[1])], { @@ -82,8 +82,8 @@ o.spec("MailExportFacade", () => { ), ).thenResolve( new Map([ - ["id1", Promise.resolve(dataByteMail1)], - ["id2", Promise.resolve(dataByteMail2)], + ["id1", dataByteMail1], + ["id2", dataByteMail2], ]), )