diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 1082186475..cbffad5fc0 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -69,7 +69,7 @@ import { registerTxAdapterFactory } from '@hcengineering/server-pipeline' import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token' -import { FileModelLogger } from '@hcengineering/server-tool' +import { FileModelLogger, buildModel } from '@hcengineering/server-tool' import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service' import path from 'path' @@ -143,7 +143,7 @@ import { moveFromMongoToPG, moveWorkspaceFromMongoToPG } from './db' -import { restoreControlledDocContentMongo, restoreWikiContentMongo } from './markup' +import { restoreControlledDocContentMongo, restoreWikiContentMongo, restoreMarkupRefsMongo } from './markup' import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin' import { fixAccountEmails, renameAccount } from './renameAccount' import { copyToDatalake, moveFiles, showLostFiles } from './storage' @@ -1345,6 +1345,61 @@ export function devTool ( }) }) + program + .command('restore-markup-ref-mongo') + .description('restore markup document content refs') + .option('-w, --workspace ', 'Selected workspace only', '') + .option('-f, --force', 'Force update', false) + .action(async (cmd: { workspace: string, force: boolean }) => { + const { txes, version } = prepareTools() + + const { hierarchy } = await buildModel(toolCtx, txes) + + let workspaces: Workspace[] = [] + await withAccountDatabase(async (db) => { + workspaces = await listWorkspacesPure(db) + workspaces = workspaces + .filter((p) => isActiveMode(p.mode)) + .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) + .sort((a, b) => b.lastVisit - a.lastVisit) + }) + + console.log('found workspaces', workspaces.length) + + await withStorage(async (storageAdapter) => { + const mongodbUri = getMongoDBUrl() + const client = getMongoClient(mongodbUri) + const _client = await client.getClient() + + try { + const count = workspaces.length + let index = 0 + for (const workspace of workspaces) { + index++ + + toolCtx.info('processing workspace', { + workspace: workspace.workspace, + version: workspace.version, + index, + count + }) + + if (!cmd.force && (workspace.version === undefined || !deepEqual(workspace.version, version))) { + console.log(`upgrade to ${versionToString(version)} is required`) + continue + } + + const workspaceId = getWorkspaceId(workspace.workspace) + const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) + + await restoreMarkupRefsMongo(toolCtx, wsDb, workspaceId, hierarchy, storageAdapter) + } + } finally { + client.close() + } + }) + }) + program .command('confirm-email ') .description('confirm user email') diff --git a/dev/tool/src/markup.ts b/dev/tool/src/markup.ts index d939e2cb48..776be29e79 100644 --- a/dev/tool/src/markup.ts +++ b/dev/tool/src/markup.ts @@ -13,10 +13,17 @@ // limitations under the License. // -import { loadCollabYdoc, saveCollabYdoc, yDocCopyXmlField } from '@hcengineering/collaboration' +import { + loadCollabYdoc, + saveCollabJson, + saveCollabYdoc, + yDocCopyXmlField, + yDocFromBuffer +} from '@hcengineering/collaboration' import core, { type Blob, type Doc, + type Hierarchy, type MeasureContext, type Ref, type TxCreateDoc, @@ -24,6 +31,7 @@ import core, { type WorkspaceId, DOMAIN_TX, SortingOrder, + makeCollabId, makeCollabYdocId, makeDocCollabId } from '@hcengineering/core' @@ -290,3 +298,65 @@ export async function restoreControlledDocContentForDoc ( return true } + +export async function restoreMarkupRefsMongo ( + ctx: MeasureContext, + db: Db, + workspaceId: WorkspaceId, + hierarchy: Hierarchy, + storageAdapter: StorageAdapter +): Promise { + const classes = hierarchy.getDescendants(core.class.Doc) + for (const _class of classes) { + const domain = hierarchy.findDomain(_class) + if (domain === undefined) continue + + const allAttributes = hierarchy.getAllAttributes(_class) + const attributes = Array.from(allAttributes.values()).filter((attribute) => { + return hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeDoc) + }) + + if (attributes.length === 0) continue + if (hierarchy.isMixin(_class) && attributes.every((p) => p.attributeOf !== _class)) continue + + ctx.info('processing', { _class, attributes: attributes.map((p) => p.name) }) + + const collection = db.collection(domain) + const iterator = collection.find({ _class }) + try { + while (true) { + const doc = await iterator.next() + if (doc === null) { + break + } + + for (const attribute of attributes) { + const isMixin = hierarchy.isMixin(attribute.attributeOf) + + const attributeName = isMixin ? `${attribute.attributeOf}.${attribute.name}` : attribute.name + + const value = isMixin + ? ((doc as any)[attribute.attributeOf]?.[attribute.name] as string) + : ((doc as any)[attribute.name] as string) + + if (typeof value === 'string') { + continue + } + + const collabId = makeCollabId(doc._class, doc._id, attribute.name) + const ydocId = makeCollabYdocId(collabId) + + try { + const buffer = await storageAdapter.read(ctx, workspaceId, ydocId) + const ydoc = yDocFromBuffer(Buffer.concat(buffer as any)) + + const jsonId = await saveCollabJson(ctx, storageAdapter, workspaceId, collabId, ydoc) + await collection.updateOne({ _id: doc._id }, { $set: { [attributeName]: jsonId } }) + } catch {} + } + } + } finally { + await iterator.close() + } + } +} diff --git a/plugins/text-editor-resources/src/provider/hocuspocus.ts b/plugins/text-editor-resources/src/provider/hocuspocus.ts index 4e711eef72..52036f2db3 100644 --- a/plugins/text-editor-resources/src/provider/hocuspocus.ts +++ b/plugins/text-editor-resources/src/provider/hocuspocus.ts @@ -33,7 +33,7 @@ export class HocuspocusCollabProvider extends HocuspocusProvider implements Prov const parameters: Record = {} const content = configuration.parameters?.content - if (content !== null && content !== '') { + if (content !== null && content !== undefined && content !== '') { parameters.content = content }