diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index dd0bfc9ebfa..dd7aeb18c10 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,6 +1 @@ -* @matrix-org/element-web -/.github/workflows/** @matrix-org/element-web-app-team -/package.json @matrix-org/element-web-app-team -/yarn.lock @matrix-org/element-web-app-team -/src/webrtc @matrix-org/element-call-reviewers -/spec/*/webrtc @matrix-org/element-call-reviewers +* @matrix-org/element-call-reviewers diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index d4d50b458d7..41ab56c84dd 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -485,6 +485,7 @@ export class MockCallMatrixClient extends TypedEventEmitter().mockReturnValue([]); public getRoom = jest.fn(); + public getFoci = jest.fn(); public supportsThreads(): boolean { return true; diff --git a/spec/unit/embedded.spec.ts b/spec/unit/embedded.spec.ts index caab40ac056..115e0a419ca 100644 --- a/spec/unit/embedded.spec.ts +++ b/spec/unit/embedded.spec.ts @@ -23,7 +23,14 @@ limitations under the License. // eslint-disable-next-line no-restricted-imports import { EventEmitter } from "events"; import { MockedObject } from "jest-mock"; -import { WidgetApi, WidgetApiToWidgetAction, MatrixCapabilities, ITurnServer, IRoomEvent } from "matrix-widget-api"; +import { + WidgetApi, + WidgetApiToWidgetAction, + MatrixCapabilities, + ITurnServer, + IRoomEvent, + IOpenIDCredentials, +} from "matrix-widget-api"; import { createRoomWidgetClient, MsgType } from "../../src/matrix"; import { MatrixClient, ClientEvent, ITurnServer as IClientTurnServer } from "../../src/client"; @@ -33,6 +40,12 @@ import { MatrixEvent } from "../../src/models/event"; import { ToDeviceBatch } from "../../src/models/ToDeviceMessage"; import { DeviceInfo } from "../../src/crypto/deviceinfo"; +const testOIDCToken = { + access_token: "12345678", + expires_in: "10", + matrix_server_name: "homeserver.oabc", + token_type: "Bearer", +}; class MockWidgetApi extends EventEmitter { public start = jest.fn(); public requestCapability = jest.fn(); @@ -49,8 +62,15 @@ class MockWidgetApi extends EventEmitter { public sendRoomEvent = jest.fn(() => ({ event_id: `$${Math.random()}` })); public sendStateEvent = jest.fn(); public sendToDevice = jest.fn(); + public requestOpenIDConnectToken = jest.fn(() => { + return testOIDCToken; + return new Promise(() => { + return testOIDCToken; + }); + }); public readStateEvents = jest.fn(() => []); public getTurnServers = jest.fn(() => []); + public sendContentLoaded = jest.fn(); public transport = { reply: jest.fn() }; } @@ -285,7 +305,12 @@ describe("RoomWidgetClient", () => { expect(await emittedSync).toEqual(SyncState.Syncing); }); }); - + describe("oidc token", () => { + it("requests an oidc token", async () => { + await makeClient({}); + expect(await client.getOpenIdToken()).toStrictEqual(testOIDCToken); + }); + }); it("gets TURN servers", async () => { const server1: ITurnServer = { uris: [ diff --git a/spec/unit/webrtc/groupCallEventHandler.spec.ts b/spec/unit/webrtc/groupCallEventHandler.spec.ts index c8d65538c60..7fe5cb6a014 100644 --- a/spec/unit/webrtc/groupCallEventHandler.spec.ts +++ b/spec/unit/webrtc/groupCallEventHandler.spec.ts @@ -71,7 +71,8 @@ describe("Group Call Event Handler", function () { getMember: (userId: string) => (userId === FAKE_USER_ID ? mockMember : null), } as unknown as Room; - (mockClient as any).getRoom = jest.fn().mockReturnValue(mockRoom); + mockClient.getRoom = jest.fn().mockReturnValue(mockRoom); + mockClient.getFoci.mockReturnValue([{}]); }); describe("reacts to state changes", () => { diff --git a/src/client.ts b/src/client.ts index 87464cf640e..42e3146a83e 100644 --- a/src/client.ts +++ b/src/client.ts @@ -380,6 +380,8 @@ export interface ICreateClientOpts { */ useE2eForGroupCall?: boolean; + livekitServiceURL?: string; + /** * Crypto callbacks provided by the application */ @@ -397,6 +399,12 @@ export interface ICreateClientOpts { * Default: false. */ isVoipWithNoMediaAllowed?: boolean; + + /** + * If true, group calls will not establish media connectivity and only create the signaling events, + * so that livekit media can be used in the application layert (js-sdk contains no livekit code). + */ + useLivekitForGroupCalls?: boolean; } export interface IMatrixClientCreateOpts extends ICreateClientOpts { @@ -1213,6 +1221,8 @@ export class MatrixClient extends TypedEventEmitter { @@ -1362,6 +1375,8 @@ export class MatrixClient extends TypedEventEmitter { @@ -239,6 +252,18 @@ export class RoomWidgetClient extends MatrixClient { return {}; } + public async getOpenIdToken(): Promise { + const token = await this.widgetApi.requestOpenIDConnectToken(); + // the IOpenIDCredentials from the widget-api and IOpenIDToken form the matrix-js-sdk are compatible. + // we still recreate the token to make this transparent and catch'able by the linter in case the types change in the future. + return { + access_token: token.access_token, + expires_in: token.expires_in, + matrix_server_name: token.matrix_server_name, + token_type: token.token_type, + }; + } + public async queueToDevice({ eventType, batch }: ToDeviceBatch): Promise { // map: user Id → device Id → payload const contentMap: MapWithDefault> = new MapWithDefault(() => new Map()); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 4c3224435cd..4307a7a25ed 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -170,6 +170,8 @@ export interface IGroupCallRoomState { // TODO: Specify data-channels "dataChannelsEnabled"?: boolean; "dataChannelOptions"?: IGroupCallDataChannelOptions; + + "io.element.livekit_service_url"?: string; } export interface IGroupCallRoomMemberFeed { @@ -250,6 +252,7 @@ export class GroupCall extends TypedEventEmitter< private initWithAudioMuted = false; private initWithVideoMuted = false; private initCallFeedPromise?: Promise; + private _livekitServiceURL?: string; private stats: GroupCallStats | undefined; /** @@ -268,10 +271,16 @@ export class GroupCall extends TypedEventEmitter< private dataChannelsEnabled?: boolean, private dataChannelOptions?: IGroupCallDataChannelOptions, isCallWithoutVideoAndAudio?: boolean, + // this tells the js-sdk not to actually establish any calls to exchange media and just to + // create the group call signaling events, with the intention that the actual media will be + // handled using livekit. The js-sdk doesn't contain any code to do the actual livekit call though. + private useLivekit = false, + livekitServiceURL?: string, ) { super(); this.reEmitter = new ReEmitter(this); this.groupCallId = groupCallId ?? genCallID(); + this._livekitServiceURL = livekitServiceURL; this.creationTs = room.currentState.getStateEvents(EventType.GroupCallPrefix, this.groupCallId)?.getTs() ?? null; this.updateParticipants(); @@ -320,6 +329,12 @@ export class GroupCall extends TypedEventEmitter< this.client.groupCallEventHandler!.groupCalls.set(this.room.roomId, this); this.client.emit(GroupCallEventHandlerEvent.Outgoing, this); + await this.sendCallStateEvent(); + + return this; + } + + private async sendCallStateEvent(): Promise { const groupCallState: IGroupCallRoomState = { "m.intent": this.intent, "m.type": this.type, @@ -328,10 +343,20 @@ export class GroupCall extends TypedEventEmitter< "dataChannelsEnabled": this.dataChannelsEnabled, "dataChannelOptions": this.dataChannelsEnabled ? this.dataChannelOptions : undefined, }; + if (this.livekitServiceURL) { + groupCallState["io.element.livekit_service_url"] = this.livekitServiceURL; + } await this.client.sendStateEvent(this.room.roomId, EventType.GroupCallPrefix, groupCallState, this.groupCallId); + } - return this; + public get livekitServiceURL(): string | undefined { + return this._livekitServiceURL; + } + + public updateLivekitServiceURL(newURL: string): Promise { + this._livekitServiceURL = newURL; + return this.sendCallStateEvent(); } private _state = GroupCallState.LocalCallFeedUninitialized; @@ -442,6 +467,11 @@ export class GroupCall extends TypedEventEmitter< } public async initLocalCallFeed(): Promise { + if (this.useLivekit) { + logger.info("Livekit group call: not starting local call feed."); + return; + } + if (this.state !== GroupCallState.LocalCallFeedUninitialized) { throw new Error(`Cannot initialize local call feed in the "${this.state}" state.`); } @@ -537,11 +567,13 @@ export class GroupCall extends TypedEventEmitter< this.onIncomingCall(call); } - this.retryCallLoopInterval = setInterval(this.onRetryCallLoop, this.retryCallInterval); + if (!this.useLivekit) { + this.retryCallLoopInterval = setInterval(this.onRetryCallLoop, this.retryCallInterval); - this.activeSpeaker = undefined; - this.onActiveSpeakerLoop(); - this.activeSpeakerLoopInterval = setInterval(this.onActiveSpeakerLoop, this.activeSpeakerInterval); + this.activeSpeaker = undefined; + this.onActiveSpeakerLoop(); + this.activeSpeakerLoopInterval = setInterval(this.onActiveSpeakerLoop, this.activeSpeakerInterval); + } } private dispose(): void { @@ -923,6 +955,11 @@ export class GroupCall extends TypedEventEmitter< return; } + if (this.useLivekit) { + logger.info("Received incoming call whilst in signaling-only mode! Ignoring."); + return; + } + const deviceMap = this.calls.get(opponentUserId) ?? new Map(); const prevCall = deviceMap.get(newCall.getOpponentDeviceId()!); @@ -1629,7 +1666,7 @@ export class GroupCall extends TypedEventEmitter< } }); - if (this.state === GroupCallState.Entered) this.placeOutgoingCalls(); + if (this.state === GroupCallState.Entered && !this.useLivekit) this.placeOutgoingCalls(); // Update the participants stored in the stats object }; diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 08487bdd234..ea8a8cc909e 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -189,6 +189,8 @@ export class GroupCallEventHandler { content?.dataChannelsEnabled || this.client.isVoipWithNoMediaAllowed, dataChannelOptions, this.client.isVoipWithNoMediaAllowed, + this.client.useLivekitForGroupCalls, + content["io.element.livekit_service_url"], ); this.groupCalls.set(room.roomId, groupCall);