Skip to content
This repository has been archived by the owner on Sep 11, 2024. It is now read-only.

Commit

Permalink
Implement MSC3819: Allowing widgets to send/receive to-device messages (
Browse files Browse the repository at this point in the history
#8885)

* Implement MSC3819: Allowing widgets to send/receive to-device messages

* Don't change the room events and state events drivers

* Update to latest matrix-widget-api changes

* Support sending encrypted to-device messages

* Use queueToDevice for better reliability

* Update types for latest WidgetDriver changes

* Upgrade matrix-widget-api

* Add tests

* Test StopGapWidget

* Fix a potential memory leak
  • Loading branch information
robintown authored Aug 10, 2022
1 parent 3d0982e commit 103b60d
Show file tree
Hide file tree
Showing 9 changed files with 322 additions and 24 deletions.
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@
"matrix-encrypt-attachment": "^1.0.3",
"matrix-events-sdk": "^0.0.1-beta.7",
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#develop",
"matrix-widget-api": "^0.1.0-beta.18",
"matrix-widget-api": "^1.0.0",
"minimist": "^1.2.5",
"opus-recorder": "^8.0.3",
"pako": "^2.0.3",
Expand Down
37 changes: 24 additions & 13 deletions src/stores/widgets/StopGapWidget.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import {
WidgetKind,
} from "matrix-widget-api";
import { EventEmitter } from "events";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { MatrixEvent, MatrixEventEvent } from "matrix-js-sdk/src/models/event";
import { logger } from "matrix-js-sdk/src/logger";
import { ClientEvent } from "matrix-js-sdk/src/client";
Expand Down Expand Up @@ -148,6 +149,7 @@ export class ElementWidget extends Widget {
}

export class StopGapWidget extends EventEmitter {
private client: MatrixClient;
private messaging: ClientWidgetApi;
private mockWidget: ElementWidget;
private scalarToken: string;
Expand All @@ -157,12 +159,13 @@ export class StopGapWidget extends EventEmitter {

constructor(private appTileProps: IAppTileProps) {
super();
let app = appTileProps.app;
this.client = MatrixClientPeg.get();

let app = appTileProps.app;
// Backwards compatibility: not all old widgets have a creatorUserId
if (!app.creatorUserId) {
app = objectShallowClone(app); // clone to prevent accidental mutation
app.creatorUserId = MatrixClientPeg.get().getUserId();
app.creatorUserId = this.client.getUserId();
}

this.mockWidget = new ElementWidget(app);
Expand Down Expand Up @@ -203,7 +206,7 @@ export class StopGapWidget extends EventEmitter {
const fromCustomisation = WidgetVariableCustomisations?.provideVariables?.() ?? {};
const defaults: ITemplateParams = {
widgetRoomId: this.roomId,
currentUserId: MatrixClientPeg.get().getUserId(),
currentUserId: this.client.getUserId(),
userDisplayName: OwnProfileStore.instance.displayName,
userHttpAvatarUrl: OwnProfileStore.instance.getHttpAvatarUrl(),
clientId: ELEMENT_CLIENT_ID,
Expand Down Expand Up @@ -260,8 +263,10 @@ export class StopGapWidget extends EventEmitter {
*/
public startMessaging(iframe: HTMLIFrameElement): any {
if (this.started) return;

const allowedCapabilities = this.appTileProps.whitelistCapabilities || [];
const driver = new StopGapWidgetDriver(allowedCapabilities, this.mockWidget, this.kind, this.roomId);

this.messaging = new ClientWidgetApi(this.mockWidget, iframe, driver);
this.messaging.on("preparing", () => this.emit("preparing"));
this.messaging.on("ready", () => this.emit("ready"));
Expand Down Expand Up @@ -302,7 +307,7 @@ export class StopGapWidget extends EventEmitter {
// Populate the map of "read up to" events for this widget with the current event in every room.
// This is a bit inefficient, but should be okay. We do this for all rooms in case the widget
// requests timeline capabilities in other rooms down the road. It's just easier to manage here.
for (const room of MatrixClientPeg.get().getRooms()) {
for (const room of this.client.getRooms()) {
// Timelines are most recent last
const events = room.getLiveTimeline()?.getEvents() || [];
const roomEvent = events[events.length - 1];
Expand All @@ -311,8 +316,9 @@ export class StopGapWidget extends EventEmitter {
}

// Attach listeners for feeding events - the underlying widget classes handle permissions for us
MatrixClientPeg.get().on(ClientEvent.Event, this.onEvent);
MatrixClientPeg.get().on(MatrixEventEvent.Decrypted, this.onEventDecrypted);
this.client.on(ClientEvent.Event, this.onEvent);
this.client.on(MatrixEventEvent.Decrypted, this.onEventDecrypted);
this.client.on(ClientEvent.ToDeviceEvent, this.onToDeviceEvent);

this.messaging.on(`action:${WidgetApiFromWidgetAction.UpdateAlwaysOnScreen}`,
(ev: CustomEvent<IStickyActionRequest>) => {
Expand Down Expand Up @@ -363,7 +369,7 @@ export class StopGapWidget extends EventEmitter {

// noinspection JSIgnoredPromiseFromCall
IntegrationManagers.sharedInstance().getPrimaryManager().open(
MatrixClientPeg.get().getRoom(RoomViewStore.instance.getRoomId()),
this.client.getRoom(RoomViewStore.instance.getRoomId()),
`type_${integType}`,
integId,
);
Expand Down Expand Up @@ -428,14 +434,13 @@ export class StopGapWidget extends EventEmitter {
WidgetMessagingStore.instance.stopMessaging(this.mockWidget, this.roomId);
this.messaging = null;

if (MatrixClientPeg.get()) {
MatrixClientPeg.get().off(ClientEvent.Event, this.onEvent);
MatrixClientPeg.get().off(MatrixEventEvent.Decrypted, this.onEventDecrypted);
}
this.client.off(ClientEvent.Event, this.onEvent);
this.client.off(MatrixEventEvent.Decrypted, this.onEventDecrypted);
this.client.off(ClientEvent.ToDeviceEvent, this.onToDeviceEvent);
}

private onEvent = (ev: MatrixEvent) => {
MatrixClientPeg.get().decryptEventIfNeeded(ev);
this.client.decryptEventIfNeeded(ev);
if (ev.isBeingDecrypted() || ev.isDecryptionFailure()) return;
this.feedEvent(ev);
};
Expand All @@ -445,6 +450,12 @@ export class StopGapWidget extends EventEmitter {
this.feedEvent(ev);
};

private onToDeviceEvent = async (ev: MatrixEvent) => {
await this.client.decryptEventIfNeeded(ev);
if (ev.isDecryptionFailure()) return;
await this.messaging.feedToDevice(ev.getEffectiveEvent(), ev.isEncrypted());
};

private feedEvent(ev: MatrixEvent) {
if (!this.messaging) return;

Expand All @@ -465,7 +476,7 @@ export class StopGapWidget extends EventEmitter {

// Timelines are most recent last, so reverse the order and limit ourselves to 100 events
// to avoid overusing the CPU.
const timeline = MatrixClientPeg.get().getRoom(ev.getRoomId()).getLiveTimeline();
const timeline = this.client.getRoom(ev.getRoomId()).getLiveTimeline();
const events = arrayFastClone(timeline.getEvents()).reverse().slice(0, 100);

for (const timelineEvent of events) {
Expand Down
48 changes: 46 additions & 2 deletions src/stores/widgets/StopGapWidgetDriver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import {
IOpenIDCredentials,
IOpenIDUpdate,
ISendEventDetails,
IRoomEvent,
MatrixCapabilities,
OpenIDRequestState,
SimpleObservable,
Expand Down Expand Up @@ -182,6 +183,49 @@ export class StopGapWidgetDriver extends WidgetDriver {
return { roomId, eventId: r.event_id };
}

public async sendToDevice(
eventType: string,
encrypted: boolean,
contentMap: { [userId: string]: { [deviceId: string]: object } },
): Promise<void> {
const client = MatrixClientPeg.get();

if (encrypted) {
const deviceInfoMap = await client.crypto.deviceList.downloadKeys(Object.keys(contentMap), false);

await Promise.all(
Object.entries(contentMap).flatMap(([userId, userContentMap]) =>
Object.entries(userContentMap).map(async ([deviceId, content]) => {
if (deviceId === "*") {
// Send the message to all devices we have keys for
await client.encryptAndSendToDevices(
Object.values(deviceInfoMap[userId]).map(deviceInfo => ({
userId, deviceInfo,
})),
content,
);
} else {
// Send the message to a specific device
await client.encryptAndSendToDevices(
[{ userId, deviceInfo: deviceInfoMap[userId][deviceId] }],
content,
);
}
}),
),
);
} else {
await client.queueToDevice({
eventType,
batch: Object.entries(contentMap).flatMap(([userId, userContentMap]) =>
Object.entries(userContentMap).map(([deviceId, content]) =>
({ userId, deviceId, payload: content }),
),
),
});
}
}

private pickRooms(roomIds: (string | Symbols.AnyRoom)[] = null): Room[] {
const client = MatrixClientPeg.get();
if (!client) throw new Error("Not attached to a client");
Expand All @@ -197,7 +241,7 @@ export class StopGapWidgetDriver extends WidgetDriver {
msgtype: string | undefined,
limitPerRoom: number,
roomIds: (string | Symbols.AnyRoom)[] = null,
): Promise<object[]> {
): Promise<IRoomEvent[]> {
limitPerRoom = limitPerRoom > 0 ? Math.min(limitPerRoom, Number.MAX_SAFE_INTEGER) : Number.MAX_SAFE_INTEGER; // relatively arbitrary

const rooms = this.pickRooms(roomIds);
Expand All @@ -224,7 +268,7 @@ export class StopGapWidgetDriver extends WidgetDriver {
stateKey: string | undefined,
limitPerRoom: number,
roomIds: (string | Symbols.AnyRoom)[] = null,
): Promise<object[]> {
): Promise<IRoomEvent[]> {
limitPerRoom = limitPerRoom > 0 ? Math.min(limitPerRoom, Number.MAX_SAFE_INTEGER) : Number.MAX_SAFE_INTEGER; // relatively arbitrary

const rooms = this.pickRooms(roomIds);
Expand Down
9 changes: 6 additions & 3 deletions src/widgets/CapabilityText.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
import {
Capability,
EventDirection,
EventKind,
getTimelineRoomIDFromCapability,
isTimelineCapability,
isTimelineCapabilityFor,
Expand Down Expand Up @@ -134,7 +135,7 @@ export class CapabilityText {
};

private static bylineFor(eventCap: WidgetEventCapability): TranslatedString {
if (eventCap.isState) {
if (eventCap.kind === EventKind.State) {
return !eventCap.keyStr
? _t("with an empty state key")
: _t("with state key %(stateKey)s", { stateKey: eventCap.keyStr });
Expand All @@ -143,6 +144,8 @@ export class CapabilityText {
}

public static for(capability: Capability, kind: WidgetKind): TranslatedCapabilityText {
// TODO: Support MSC3819 (to-device capabilities)

// First see if we have a super simple line of text to provide back
if (CapabilityText.simpleCaps[capability]) {
const textForKind = CapabilityText.simpleCaps[capability];
Expand Down Expand Up @@ -184,13 +187,13 @@ export class CapabilityText {
// Special case room messages so they show up a bit cleaner to the user. Result is
// effectively "Send images" instead of "Send messages... of type images" if we were
// to handle the msgtype nuances in this function.
if (!eventCap.isState && eventCap.eventType === EventType.RoomMessage) {
if (eventCap.kind === EventKind.Event && eventCap.eventType === EventType.RoomMessage) {
return CapabilityText.forRoomMessageCap(eventCap, kind);
}

// See if we have a static line of text to provide for the given event type and
// direction. The hope is that we do for common event types for friendlier copy.
const evSendRecv = eventCap.isState
const evSendRecv = eventCap.kind === EventKind.State
? CapabilityText.stateSendRecvCaps
: CapabilityText.nonStateSendRecvCaps;
if (evSendRecv[eventCap.eventType]) {
Expand Down
70 changes: 70 additions & 0 deletions test/stores/widgets/StopGapWidget-test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/*
Copyright 2022 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import { mocked, MockedObject } from "jest-mock";
import { MatrixClient, ClientEvent } from "matrix-js-sdk/src/client";
import { ClientWidgetApi } from "matrix-widget-api";

import { stubClient, mkRoom, mkEvent } from "../../test-utils";
import { MatrixClientPeg } from "../../../src/MatrixClientPeg";
import { StopGapWidget } from "../../../src/stores/widgets/StopGapWidget";

jest.mock("matrix-widget-api/lib/ClientWidgetApi");

describe("StopGapWidget", () => {
let client: MockedObject<MatrixClient>;
let widget: StopGapWidget;
let messaging: MockedObject<ClientWidgetApi>;

beforeEach(() => {
stubClient();
client = mocked(MatrixClientPeg.get());

widget = new StopGapWidget({
app: {
id: "test",
creatorUserId: "@alice:example.org",
type: "example",
url: "https://example.org",
},
room: mkRoom(client, "!1:example.org"),
userId: "@alice:example.org",
creatorUserId: "@alice:example.org",
waitForIframeLoad: true,
userWidget: false,
});
// Start messaging without an iframe, since ClientWidgetApi is mocked
widget.startMessaging(null as unknown as HTMLIFrameElement);
messaging = mocked(mocked(ClientWidgetApi).mock.instances[0]);
});

afterEach(() => {
widget.stopMessaging();
});

it("feeds incoming to-device messages to the widget", async () => {
const event = mkEvent({
event: true,
type: "org.example.foo",
user: "@alice:example.org",
content: { hello: "world" },
});

client.emit(ClientEvent.ToDeviceEvent, event);
await Promise.resolve(); // flush promises
expect(messaging.feedToDevice).toHaveBeenCalledWith(event.getEffectiveEvent(), false);
});
});
Loading

0 comments on commit 103b60d

Please sign in to comment.