UBERF-8581: Optimise session data (#7582)

dev/prod/src/platform.ts

@@ -420,7 +420,9 @@ export async function configurePlatform() {
   setMetadata(login.metadata.TransactorOverride, config.TRANSACTOR_OVERRIDE)
 
   // Use binary response transfer for faster performance and small transfer sizes.
-  setMetadata(client.metadata.UseBinaryProtocol, config.USE_BINARY_PROTOCOL ?? true)
+  const binaryOverride = localStorage.getItem(client.metadata.UseBinaryProtocol)
+  setMetadata(client.metadata.UseBinaryProtocol, binaryOverride != null ? binaryOverride === 'true' : (config.USE_BINARY_PROTOCOL ?? true))
+
   // Disable for now, since it causes performance issues on linux/docker/kubernetes boxes for now.
   setMetadata(client.metadata.UseProtocolCompression, true)
 

dev/tool/package.json

@@ -160,7 +160,6 @@
     "csv-parse": "~5.1.0",
     "email-addresses": "^5.0.0",
     "fast-equals": "^5.0.1",
-    "got": "^11.8.3",
     "libphonenumber-js": "^1.9.46",
     "mime-types": "~2.1.34",
     "mongodb": "^6.12.0",

packages/core/src/classes.ts

@@ -661,6 +661,7 @@ export type WorkspaceMode =
   | 'pending-deletion' // -> 'deleting'
   | 'deleting' // -> "deleted"
   | 'active'
+  | 'deleted'
   | 'archiving-pending-backup' // -> 'cleaning'
   | 'archiving-backup' // -> 'archiving-pending-clean'
   | 'archiving-pending-clean' // -> 'archiving-clean'

plugins/client-resources/src/connection.ts

@@ -549,6 +549,7 @@ class Connection implements ClientConnection {
     once?: boolean // Require handleResult to retrieve result
     measure?: (time: number, result: any, serverTime: number, queue: number, toRecieve: number) => void
     allowReconnect?: boolean
+    overrideId?: number
   }): Promise<any> {
     return this.ctx.newChild('send-request', {}).with(data.method, {}, async (ctx) => {
       if (this.closed) {
@@ -566,7 +567,7 @@ class Connection implements ClientConnection {
         }
       }
 
-      const id = this.lastId++
+      const id = data.overrideId ?? this.lastId++
       const promise = new RequestPromise(data.method, data.params, data.handleResult)
       promise.handleTime = data.measure
 
@@ -725,7 +726,7 @@ class Connection implements ClientConnection {
   }
 
   sendForceClose (): Promise<void> {
-    return this.sendRequest({ method: 'forceClose', params: [], allowReconnect: false })
+    return this.sendRequest({ method: 'forceClose', params: [], allowReconnect: false, overrideId: -2, once: true })
   }
 }
 

pods/server/src/server.ts

@@ -14,22 +14,16 @@
 // limitations under the License.
 //
 
-import {
-  type Branding,
-  type BrandingMap,
-  type MeasureContext,
-  type Tx,
-  type WorkspaceIdWithUrl
-} from '@hcengineering/core'
+import { type BrandingMap, type MeasureContext, type Tx } from '@hcengineering/core'
 import { buildStorageFromConfig } from '@hcengineering/server-storage'
 
 import { ClientSession, startSessionManager } from '@hcengineering/server'
 import {
-  type Pipeline,
   type ServerFactory,
   type Session,
   type SessionManager,
-  type StorageConfiguration
+  type StorageConfiguration,
+  type Workspace
 } from '@hcengineering/server-core'
 import { type Token } from '@hcengineering/server-token'
 
@@ -42,9 +36,9 @@ import {
   registerTxAdapterFactory
 } from '@hcengineering/server-pipeline'
 
-import { readFileSync } from 'node:fs'
 import { createMongoAdapter, createMongoDestroyAdapter, createMongoTxAdapter } from '@hcengineering/mongo'
 import { createPostgreeDestroyAdapter, createPostgresAdapter, createPostgresTxAdapter } from '@hcengineering/postgres'
+import { readFileSync } from 'node:fs'
 const model = JSON.parse(readFileSync(process.env.MODEL_JSON ?? 'model.json').toString()) as Tx[]
 
 registerStringLoaders()
@@ -93,13 +87,8 @@ export function start (
     { ...opt, externalStorage, adapterSecurity: dbUrl.startsWith('postgresql') },
     {}
   )
-  const sessionFactory = (
-    token: Token,
-    pipeline: Pipeline,
-    workspaceId: WorkspaceIdWithUrl,
-    branding: Branding | null
-  ): Session => {
-    return new ClientSession(token, pipeline, workspaceId, branding, token.extra?.mode === 'backup')
+  const sessionFactory = (token: Token, workspace: Workspace): Session => {
+    return new ClientSession(token, workspace, token.extra?.mode === 'backup')
   }
 
   const { shutdown: onClose, sessionManager } = startSessionManager(metrics, {

server/account/src/operations.ts

@@ -1712,7 +1712,7 @@ export async function getAllWorkspaces (
   }
 
   return (await db.workspace.find({})).map((it) => {
-    it.accounts = (it.accounts ?? []).map((it) => it.toString())
+    ;(it as any).accounts = (it.accounts ?? []).length
     return it
   })
 }

server/core/src/types.ts

@@ -43,7 +43,7 @@ import {
 } from '@hcengineering/core'
 import type { Asset, Resource } from '@hcengineering/platform'
 import type { LiveQuery } from '@hcengineering/query'
-import type { Request, Response } from '@hcengineering/rpc'
+import type { ReqId, Request, Response } from '@hcengineering/rpc'
 import type { Token } from '@hcengineering/server-token'
 import { type Readable } from 'stream'
 import type { DbAdapter, DomainHelper } from './adapter'
@@ -496,32 +496,21 @@ export interface SessionRequest {
 
 export interface ClientSessionCtx {
   ctx: MeasureContext
-  sendResponse: (msg: any) => Promise<void>
+
+  pipeline: Pipeline
+
+  requestId: ReqId | undefined
+  sendResponse: (id: ReqId | undefined, msg: any) => Promise<void>
   sendPong: () => void
-  sendError: (msg: any, error: any) => Promise<void>
+  sendError: (id: ReqId | undefined, msg: any, error: any) => Promise<void>
 }
 
 /**
  * @public
  */
 export interface Session {
+  workspace: Workspace
   createTime: number
-  getUser: () => string
-  pipeline: () => Pipeline
-  ping: (ctx: ClientSessionCtx) => Promise<void>
-  findAll: <T extends Doc>(
-    ctx: ClientSessionCtx,
-    _class: Ref<Class<T>>,
-    query: DocumentQuery<T>,
-    options?: FindOptions<T>
-  ) => Promise<void>
-  findAllRaw: <T extends Doc>(
-    ctx: MeasureContext,
-    _class: Ref<Class<T>>,
-    query: DocumentQuery<T>,
-    options?: FindOptions<T>
-  ) => Promise<FindResult<T>>
-  tx: (ctx: ClientSessionCtx, tx: Tx) => Promise<void>
 
   // Session restore information
   sessionId: string
@@ -544,6 +533,28 @@ export interface Session {
   getMode: () => string
 
   broadcast: (ctx: MeasureContext, socket: ConnectionSocket, tx: Tx[]) => void
+
+  // Client methods
+  ping: (ctx: ClientSessionCtx) => Promise<void>
+  getUser: () => string
+
+  loadModel: (ctx: ClientSessionCtx, lastModelTx: Timestamp, hash?: string) => Promise<void>
+  getAccount: (ctx: ClientSessionCtx) => Promise<void>
+
+  getRawAccount: (pipeline: Pipeline) => Account
+  findAll: <T extends Doc>(
+    ctx: ClientSessionCtx,
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options?: FindOptions<T>
+  ) => Promise<void>
+  searchFulltext: (ctx: ClientSessionCtx, query: SearchQuery, options: SearchOptions) => Promise<void>
+  tx: (ctx: ClientSessionCtx, tx: Tx) => Promise<void>
+  loadChunk: (ctx: ClientSessionCtx, domain: Domain, idx?: number) => Promise<void>
+  closeChunk: (ctx: ClientSessionCtx, idx: number) => Promise<void>
+  loadDocs: (ctx: ClientSessionCtx, domain: Domain, docs: Ref<Doc>[]) => Promise<void>
+  upload: (ctx: ClientSessionCtx, domain: Domain, docs: Doc[]) => Promise<void>
+  clean: (ctx: ClientSessionCtx, domain: Domain, docs: Ref<Doc>[]) => Promise<void>
 }
 
 /**
@@ -587,7 +598,7 @@ export interface Workspace {
   context: MeasureContext
   id: string
   token: string // Account workspace update token.
-  pipeline: Promise<Pipeline>
+  pipeline: Promise<Pipeline> | Pipeline
   tickHash: number
 
   tickHandlers: Map<string, TickHandler>
@@ -599,7 +610,7 @@ export interface Workspace {
   softShutdown: number
   workspaceInitCompleted: boolean
 
-  workspaceId: WorkspaceId
+  workspaceId: WorkspaceIdWithUrl
   workspaceName: string
   workspaceUuid?: string
   branding: Branding | null
@@ -622,12 +633,7 @@ export interface SessionManager {
   workspaces: Map<string, Workspace>
   sessions: Map<string, { session: Session, socket: ConnectionSocket }>
 
-  createSession: (
-    token: Token,
-    pipeline: Pipeline,
-    workspaceId: WorkspaceIdWithUrl,
-    branding: Branding | null
-  ) => Session
+  createSession: (token: Token, workspace: Workspace) => Session
 
   addSession: (
     ctx: MeasureContext,

server/core/src/utils.ts

@@ -167,20 +167,51 @@ export function getUser (modelDb: ModelDb, userEmail: string | undefined, admin?
 
 export class SessionDataImpl implements SessionData {
   _account: Account | undefined
+  _removedMap: Map<Ref<Doc>, Doc> | undefined
+  _contextCache: Map<string, any> | undefined
+  _broadcast: SessionData['broadcast'] | undefined
 
   constructor (
     readonly userEmail: string,
     readonly sessionId: string,
     readonly admin: boolean | undefined,
-    readonly broadcast: SessionData['broadcast'],
+    _broadcast: SessionData['broadcast'] | undefined,
     readonly workspace: WorkspaceIdWithUrl,
     readonly branding: Branding | null,
     readonly isAsyncContext: boolean,
-    readonly removedMap: Map<Ref<Doc>, Doc>,
-    readonly contextCache: Map<string, any>,
+    _removedMap: Map<Ref<Doc>, Doc> | undefined,
+    _contextCache: Map<string, any> | undefined,
     readonly modelDb: ModelDb,
     readonly rawAccount?: Account
-  ) {}
+  ) {
+    this._removedMap = _removedMap
+    this._contextCache = _contextCache
+    this._broadcast = _broadcast
+  }
+
+  get broadcast (): SessionData['broadcast'] {
+    if (this._broadcast === undefined) {
+      this._broadcast = {
+        targets: {},
+        txes: []
+      }
+    }
+    return this._broadcast
+  }
+
+  get removedMap (): Map<Ref<Doc>, Doc> {
+    if (this._removedMap === undefined) {
+      this._removedMap = new Map()
+    }
+    return this._removedMap
+  }
+
+  get contextCache (): Map<string, any> {
+    if (this._contextCache === undefined) {
+      this._contextCache = new Map()
+    }
+    return this._contextCache
+  }
 
   get account (): Account {
     this._account = this.rawAccount ?? this._account ?? getUser(this.modelDb, this.userEmail, this.admin)
@@ -234,8 +265,8 @@ export function wrapPipeline (
     wsUrl,
     null,
     true,
-    new Map(),
-    new Map(),
+    undefined,
+    undefined,
     pipeline.context.modelDb
   )
   ctx.contextData = contextData

server/indexer/src/indexer/indexer.ts

@@ -759,12 +759,12 @@ export class FullTextIndexPipeline implements FullTextPipeline {
       systemAccountEmail,
       '',
       true,
-      { targets: {}, txes: [] },
+      undefined,
       this.workspace,
       null,
       false,
-      new Map(),
-      new Map(),
+      undefined,
+      undefined,
       this.model
     )
   }