From d7cb39f9f493a5b14a0c9830b7986dc2929ad49c Mon Sep 17 00:00:00 2001 From: Mohamed Marrouchi Date: Sat, 11 Jan 2025 11:14:16 +0100 Subject: [PATCH] fix: message attachment id --- api/src/migration/migration.module.ts | 2 + api/src/migration/migration.service.spec.ts | 8 ++ api/src/migration/migration.service.ts | 3 + .../1735836154221-v-2-2-0.migration.ts | 104 ++++++++++++++++++ api/src/migration/types.ts | 2 + 5 files changed, 119 insertions(+) diff --git a/api/src/migration/migration.module.ts b/api/src/migration/migration.module.ts index 1f09f473..08885a0c 100644 --- a/api/src/migration/migration.module.ts +++ b/api/src/migration/migration.module.ts @@ -12,6 +12,7 @@ import { HttpModule } from '@nestjs/axios'; import { Module } from '@nestjs/common'; import { MongooseModule } from '@nestjs/mongoose'; +import { AttachmentModule } from '@/attachment/attachment.module'; import { LoggerModule } from '@/logger/logger.module'; import { MigrationCommand } from './migration.command'; @@ -23,6 +24,7 @@ import { MigrationService } from './migration.service'; MongooseModule.forFeature([MigrationModel]), LoggerModule, HttpModule, + AttachmentModule, ], providers: [ MigrationService, diff --git a/api/src/migration/migration.service.spec.ts b/api/src/migration/migration.service.spec.ts index 87a64ba3..31827bae 100644 --- a/api/src/migration/migration.service.spec.ts +++ b/api/src/migration/migration.service.spec.ts @@ -14,6 +14,7 @@ import { EventEmitter2 } from '@nestjs/event-emitter'; import { getModelToken, MongooseModule } from '@nestjs/mongoose'; import { Test, TestingModule } from '@nestjs/testing'; +import { AttachmentService } from '@/attachment/services/attachment.service'; import { LoggerService } from '@/logger/logger.service'; import { MetadataRepository } from '@/setting/repositories/metadata.repository'; import { Metadata, MetadataModel } from '@/setting/schemas/metadata.schema'; @@ -54,6 +55,10 @@ describe('MigrationService', () => { provide: HttpService, useValue: {}, }, + { + provide: AttachmentService, + useValue: {}, + }, { provide: ModuleRef, useValue: { @@ -278,6 +283,7 @@ describe('MigrationService', () => { }); expect(loadMigrationFileSpy).toHaveBeenCalledWith('v2.1.9'); expect(migrationMock.up).toHaveBeenCalledWith({ + attachmentService: service['attachmentService'], logger: service['logger'], http: service['httpService'], }); @@ -308,6 +314,7 @@ describe('MigrationService', () => { }); expect(loadMigrationFileSpy).toHaveBeenCalledWith('v2.1.9'); expect(migrationMock.up).toHaveBeenCalledWith({ + attachmentService: service['attachmentService'], logger: service['logger'], http: service['httpService'], }); @@ -338,6 +345,7 @@ describe('MigrationService', () => { }); expect(loadMigrationFileSpy).toHaveBeenCalledWith('v2.1.9'); expect(migrationMock.up).toHaveBeenCalledWith({ + attachmentService: service['attachmentService'], logger: service['logger'], http: service['httpService'], }); diff --git a/api/src/migration/migration.service.ts b/api/src/migration/migration.service.ts index e74a9b0e..88744095 100644 --- a/api/src/migration/migration.service.ts +++ b/api/src/migration/migration.service.ts @@ -19,6 +19,7 @@ import leanDefaults from 'mongoose-lean-defaults'; import leanGetters from 'mongoose-lean-getters'; import leanVirtuals from 'mongoose-lean-virtuals'; +import { AttachmentService } from '@/attachment/services/attachment.service'; import { config } from '@/config'; import { LoggerService } from '@/logger/logger.service'; import { MetadataService } from '@/setting/services/metadata.service'; @@ -43,6 +44,7 @@ export class MigrationService implements OnApplicationBootstrap { private readonly logger: LoggerService, private readonly metadataService: MetadataService, private readonly httpService: HttpService, + private readonly attachmentService: AttachmentService, @InjectModel(Migration.name) private readonly migrationModel: Model, ) {} @@ -253,6 +255,7 @@ module.exports = { const result = await migration[action]({ logger: this.logger, http: this.httpService, + attachmentService: this.attachmentService, }); if (result) { diff --git a/api/src/migration/migrations/1735836154221-v-2-2-0.migration.ts b/api/src/migration/migrations/1735836154221-v-2-2-0.migration.ts index bf4572e3..843ed4ae 100644 --- a/api/src/migration/migrations/1735836154221-v-2-2-0.migration.ts +++ b/api/src/migration/migrations/1735836154221-v-2-2-0.migration.ts @@ -10,11 +10,13 @@ import { existsSync } from 'fs'; import { join, resolve } from 'path'; import mongoose, { HydratedDocument } from 'mongoose'; +import { v4 as uuidv4 } from 'uuid'; import attachmentSchema, { Attachment, } from '@/attachment/schemas/attachment.schema'; import blockSchema, { Block } from '@/chat/schemas/block.schema'; +import messageSchema, { Message } from '@/chat/schemas/message.schema'; import subscriberSchema, { Subscriber } from '@/chat/schemas/subscriber.schema'; import { StdOutgoingAttachmentMessage } from '@/chat/schemas/types/message'; import contentSchema, { Content } from '@/cms/schemas/content.schema'; @@ -372,12 +374,114 @@ const migrateAttachmentContents = async ( } }; +/** + * Updates message documents that contain attachment "message.attachment" + * to apply one of the following operation: + * - Rename 'attachment_id' to 'id' + * - Parse internal url for to get the 'id' + * - Fetch external url, stores the attachment and store the 'id' + * + * @returns Resolves when the migration process is complete. + */ +const migrateAttachmentMessages = async ({ + logger, + http, + attachmentService, +}: MigrationServices) => { + const MessageModel = mongoose.model(Message.name, messageSchema); + + // Find blocks where "message.attachment" exists + const cursor = MessageModel.find({ + 'message.attachment.payload': { $exists: true }, + 'message.attachment.payload.id': { $exists: false }, + }).cursor(); + + // Helper function to update the attachment ID in the database + const updateAttachmentId = async ( + messageId: mongoose.Types.ObjectId, + attachmentId: string | null, + ) => { + await MessageModel.updateOne( + { _id: messageId }, + { $set: { 'message.attachment.payload.id': attachmentId } }, + ); + }; + + for await (const msg of cursor) { + try { + if ( + 'attachment' in msg.message && + 'payload' in msg.message.attachment && + msg.message.attachment.payload + ) { + if ('attachment_id' in msg.message.attachment.payload) { + await updateAttachmentId( + msg._id, + msg.message.attachment.payload.attachment_id as string, + ); + } else if ('url' in msg.message.attachment.payload) { + const url = msg.message.attachment.payload.url; + const regex = + /^https?:\/\/[\w.-]+\/attachment\/download\/([a-f\d]{24})\/.+$/; + // Test the URL and extract the ID + const match = url.match(regex); + if (match) { + const [, attachmentId] = match; + await updateAttachmentId(msg._id, attachmentId); + } else if (url) { + logger.log( + `Migrate message ${msg._id}: Handling an external url ...`, + ); + const response = await http.axiosRef.get(url, { + responseType: 'arraybuffer', // Ensures the response is returned as a Buffer + }); + const fileBuffer = Buffer.from(response.data); + const attachment = await attachmentService.store(fileBuffer, { + name: uuidv4(), + size: fileBuffer.length, + type: response.headers['content-type'], + channel: {}, + }); + await updateAttachmentId(msg._id, attachment.id); + } + } else { + logger.warn( + `Unable to migrate message ${msg._id}: No ID nor URL was found`, + ); + + throw new Error( + 'Unable to process message attachment: No ID or URL to be processed', + ); + } + } else { + throw new Error( + 'Unable to process message attachment: Invalid Payload', + ); + } + } catch (error) { + logger.error( + `Failed to update message ${msg._id}: ${error.message}, defaulting to null`, + ); + try { + await updateAttachmentId(msg._id, null); + } catch (err) { + logger.error( + `Failed to update message ${msg._id}: ${error.message}, unable to default to null`, + ); + } + } + } +}; + module.exports = { async up(services: MigrationServices) { await populateSubscriberAvatar(services); await updateOldAvatarsPath(services); await migrateAttachmentBlocks(MigrationAction.UP, services); await migrateAttachmentContents(MigrationAction.UP, services); + // Given the complexity and inconsistency data, this method does not have + // a revert equivalent, at the same time, thus, it doesn't "unset" any attribute + await migrateAttachmentMessages(services); return true; }, async down(services: MigrationServices) { diff --git a/api/src/migration/types.ts b/api/src/migration/types.ts index 2bf93cdd..e2a37c0f 100644 --- a/api/src/migration/types.ts +++ b/api/src/migration/types.ts @@ -8,6 +8,7 @@ import { HttpService } from '@nestjs/axios'; +import { AttachmentService } from '@/attachment/services/attachment.service'; import { LoggerService } from '@/logger/logger.service'; import { MigrationDocument } from './migration.schema'; @@ -34,4 +35,5 @@ export interface MigrationSuccessCallback extends MigrationRunParams { export type MigrationServices = { logger: LoggerService; http: HttpService; + attachmentService: AttachmentService; };