Run Playwright tests against Dendrite & Pinecone periodically

.github/workflows/end-to-end-tests.yaml

@@ -114,6 +114,8 @@ jobs:
                     - Chrome
                     - Firefox
                     - WebKit
+                    - Dendrite
+                    - Pinecone
                 runAllTests:
                     - ${{ github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'X-Run-All-Tests') }}
                 # Skip the Firefox & Safari runs unless this was a cron trigger or PR has X-Run-All-Tests label
@@ -122,6 +124,10 @@ jobs:
                       project: Firefox
                     - runAllTests: false
                       project: WebKit
+                    - runAllTests: false
+                      project: Dendrite
+                    - runAllTests: false
+                      project: Pinecone
         steps:
             - uses: actions/checkout@v4
               with:

docs/playwright.md

@@ -66,11 +66,11 @@ as is typical for Playwright tests. Likewise, tests live in `playwright/e2e`.
 of Synapse/Dendrite. These servers are what Element-web runs against in the tests.
 
 Synapse can be launched with different configurations in order to test element
-in different configurations. You can specify `synapseConfigOptions` as such:
+in different configurations. You can specify `synapseConfig` as such:
 
 ```typescript
 test.use({
-    synapseConfigOptions: {
+    synapseConfig: {
         // The config options to pass to the Synapse instance
     },
 });

playwright.config.ts

@@ -8,19 +8,25 @@ Please see LICENSE files in the repository root for full details.
 
 import { defineConfig, devices } from "@playwright/test";
 
+import { Options } from "./playwright/services";
+
 const baseURL = process.env["BASE_URL"] ?? "http://localhost:8080";
 
-export default defineConfig({
+const chromeProject = {
+    ...devices["Desktop Chrome"],
+    channel: "chromium",
+    permissions: ["clipboard-write", "clipboard-read", "microphone"],
+    launchOptions: {
+        args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--mute-audio"],
+    },
+};
+
+export default defineConfig<Options>({
     projects: [
         {
             name: "Chrome",
             use: {
-                ...devices["Desktop Chrome"],
-                channel: "chromium",
-                permissions: ["clipboard-write", "clipboard-read", "microphone"],
-                launchOptions: {
-                    args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--mute-audio"],
-                },
+                ...chromeProject,
             },
         },
         {
@@ -48,6 +54,22 @@ export default defineConfig({
             },
             ignoreSnapshots: true,
         },
+        {
+            name: "Dendrite",
+            use: {
+                ...chromeProject,
+                homeserverType: "dendrite",
+            },
+            ignoreSnapshots: true,
+        },
+        {
+            name: "Pinecone",
+            use: {
+                ...chromeProject,
+                homeserverType: "pinecone",
+            },
+            ignoreSnapshots: true,
+        },
     ],
     use: {
         viewport: { width: 1280, height: 720 },

playwright/e2e/create-room/create-room.spec.ts

@@ -27,7 +27,7 @@ test.describe("Create Room", () => {
         // Submit
         await dialog.getByRole("button", { name: "Create room" }).click();
 
-        await expect(page).toHaveURL(/\/#\/room\/#test-room-1:localhost/);
+        await expect(page).toHaveURL(new RegExp(`/#/room/#test-room-1:${user.homeServer}`));
         const header = page.locator(".mx_RoomHeader");
         await expect(header).toContainText(name);
     });

playwright/e2e/crypto/backups.spec.ts

@@ -9,6 +9,7 @@ Please see LICENSE files in the repository root for full details.
 import { type Page } from "@playwright/test";
 
 import { test, expect } from "../../element-web-test";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
 
 async function expectBackupVersionToBe(page: Page, version: string) {
     await expect(page.locator(".mx_SecureBackupPanel_statusList tr:nth-child(5) td")).toHaveText(
@@ -19,6 +20,7 @@ async function expectBackupVersionToBe(page: Page, version: string) {
 }
 
 test.describe("Backups", () => {
+    test.skip(isDendrite, "Dendrite lacks support for MSC3967 so requires additional auth here");
     test.use({
         displayName: "Hanako",
     });

playwright/e2e/crypto/complete-security.spec.ts

@@ -8,8 +8,10 @@ Please see LICENSE files in the repository root for full details.
 
 import { test, expect } from "../../element-web-test";
 import { logIntoElement } from "./utils";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
 
 test.describe("Complete security", () => {
+    test.skip(isDendrite, "Dendrite lacks support for MSC3967 so requires additional auth here");
     test.use({
         displayName: "Jeff",
     });

playwright/e2e/crypto/crypto.spec.ts

@@ -11,6 +11,7 @@ import { expect, test } from "../../element-web-test";
 import { autoJoin, copyAndContinue, createSharedRoomWithUser, enableKeyBackup, verify } from "./utils";
 import { Bot } from "../../pages/bot";
 import { ElementAppPage } from "../../pages/ElementAppPage";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
 
 const checkDMRoom = async (page: Page) => {
     const body = page.locator(".mx_RoomView_body");
@@ -67,6 +68,7 @@ const bobJoin = async (page: Page, bob: Bot) => {
 };
 
 test.describe("Cryptography", function () {
+    test.skip(isDendrite, "Dendrite lacks support for MSC3967 so requires additional auth here");
     test.use({
         displayName: "Alice",
         botCreateOpts: {

playwright/e2e/crypto/decryption-failure-messages.spec.ts

@@ -1,4 +1,4 @@
 /*
 Copyright 2024 New Vector Ltd.
 Copyright 2022-2024 The Matrix.org Foundation C.I.C.
 
@@ -28,6 +28,8 @@
     });
 
     test.describe("decryption failure messages", () => {
+        test.skip(isDendrite, "Dendrite lacks support for MSC3967 so requires additional auth here");
+
         test("should handle device-relative historical messages", async ({
             homeserver,
             page,
@@ -63,7 +65,7 @@
             // Finally, log out again, and back in, skipping verification for now, and see what we see.
             await logOutOfElement(page);
             await logIntoElement(page, credentials);
             await page.locator(".mx_AuthPage").getByRole("button", { name: "Skip verification for now" }).click();
             await page.locator(".mx_AuthPage").getByRole("button", { name: "I'll verify later" }).click();
             await app.viewRoomByName("Test room");
 

playwright/e2e/crypto/device-verification.spec.ts

@@ -219,9 +219,10 @@ test.describe("Device verification", { tag: "@no-webkit" }, () => {
         /* And we're all done! */
         const infoDialog = page.locator(".mx_InfoDialog");
         await infoDialog.getByRole("button", { name: "They match" }).click();
-        await expect(
-            infoDialog.getByText(`You've successfully verified (${aliceBotClient.credentials.deviceId})!`),
-        ).toBeVisible();
+        // We don't assert the full string as the device name is unset on Synapse but set to the user ID on Dendrite
+        await expect(infoDialog.getByText(`You've successfully verified`)).toContainText(
+            `(${aliceBotClient.credentials.deviceId})`,
+        );
         await infoDialog.getByRole("button", { name: "Got it" }).click();
     });
 });

playwright/e2e/crypto/logout.spec.ts

@@ -8,8 +8,10 @@ Please see LICENSE files in the repository root for full details.
 
 import { test, expect } from "../../element-web-test";
 import { createRoom, enableKeyBackup, logIntoElement, sendMessageInCurrentRoom } from "./utils";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
 
 test.describe("Logout tests", () => {
+    test.skip(isDendrite, "Dendrite lacks support for MSC3967 so requires additional auth here");
     test.beforeEach(async ({ page, homeserver, credentials }) => {
         await logIntoElement(page, credentials);
     });

playwright/e2e/editing/editing.spec.ts

@@ -12,6 +12,7 @@ import type { EventType, IContent, ISendEventResponse, MsgType, Visibility } fro
 import { expect, test } from "../../element-web-test";
 import { ElementAppPage } from "../../pages/ElementAppPage";
 import { SettingLevel } from "../../../src/settings/SettingLevel";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
 
 async function sendEvent(app: ElementAppPage, roomId: string): Promise<ISendEventResponse> {
     return app.client.sendEvent(roomId, null, "m.room.message" as EventType, {
@@ -31,6 +32,8 @@ function mkPadding(n: number): IContent {
 }
 
 test.describe("Editing", () => {
+    test.skip(isDendrite, "due to a Dendrite bug https://github.com/element-hq/dendrite/issues/3488");
+
     // Edit "Message"
     const editLastMessage = async (page: Page, edit: string) => {
         const eventTile = page.locator(".mx_RoomView_MessageList .mx_EventTile_last");

playwright/e2e/knock/create-knock-room.spec.ts

@@ -9,8 +9,10 @@ Please see LICENSE files in the repository root for full details.
 import { test, expect } from "../../element-web-test";
 import { waitForRoom } from "../utils";
 import { Filter } from "../../pages/Spotlight";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
 
 test.describe("Create Knock Room", () => {
+    test.skip(isDendrite, "Dendrite does not have support for knocking");
     test.use({
         displayName: "Alice",
         labsFlags: ["feature_ask_to_join"],

playwright/e2e/knock/knock-into-room.spec.ts

@@ -13,8 +13,10 @@ import { type Visibility } from "matrix-js-sdk/src/matrix";
 import { test, expect } from "../../element-web-test";
 import { waitForRoom } from "../utils";
 import { Filter } from "../../pages/Spotlight";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
 
 test.describe("Knock Into Room", () => {
+    test.skip(isDendrite, "Dendrite does not have support for knocking");
     test.use({
         displayName: "Alice",
         labsFlags: ["feature_ask_to_join"],

playwright/e2e/knock/manage-knocks.spec.ts

@@ -10,8 +10,10 @@ Please see LICENSE files in the repository root for full details.
 
 import { test, expect } from "../../element-web-test";
 import { waitForRoom } from "../utils";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
 
 test.describe("Manage Knocks", () => {
+    test.skip(isDendrite, "Dendrite does not have support for knocking");
     test.use({
         displayName: "Alice",
         labsFlags: ["feature_ask_to_join"],

playwright/e2e/lazy-loading/lazy-loading.spec.ts

@@ -10,8 +10,12 @@ import { Bot } from "../../pages/bot";
 import type { Locator, Page } from "@playwright/test";
 import type { ElementAppPage } from "../../pages/ElementAppPage";
 import { test, expect } from "../../element-web-test";
+import { Credentials } from "../../plugins/homeserver";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
 
 test.describe("Lazy Loading", () => {
+    test.skip(isDendrite, "due to a Dendrite bug https://github.com/element-hq/dendrite/issues/3488");
+
     const charlies: Bot[] = [];
 
     test.use({
@@ -35,12 +39,18 @@ test.describe("Lazy Loading", () => {
     });
 
     const name = "Lazy Loading Test";
-    const alias = "#lltest:localhost";
     const charlyMsg1 = "hi bob!";
     const charlyMsg2 = "how's it going??";
     let roomId: string;
 
-    async function setupRoomWithBobAliceAndCharlies(page: Page, app: ElementAppPage, bob: Bot, charlies: Bot[]) {
+    async function setupRoomWithBobAliceAndCharlies(
+        page: Page,
+        app: ElementAppPage,
+        user: Credentials,
+        bob: Bot,
+        charlies: Bot[],
+    ) {
+        const alias = `#lltest:${user.homeServer}`;
         const visibility = await page.evaluate(() => (window as any).matrixcs.Visibility.Public);
         roomId = await bob.createRoom({
             name,
@@ -95,7 +105,13 @@ test.describe("Lazy Loading", () => {
         }
     }
 
-    async function joinCharliesWhileAliceIsOffline(page: Page, app: ElementAppPage, charlies: Bot[]) {
+    async function joinCharliesWhileAliceIsOffline(
+        page: Page,
+        app: ElementAppPage,
+        user: Credentials,
+        charlies: Bot[],
+    ) {
+        const alias = `#lltest:${user.homeServer}`;
         await app.client.network.goOffline();
         for (const charly of charlies) {
             await charly.joinRoom(alias);
@@ -107,19 +123,19 @@ test.describe("Lazy Loading", () => {
         await app.client.waitForNextSync();
     }
 
-    test("should handle lazy loading properly even when offline", async ({ page, app, bot }) => {
+    test("should handle lazy loading properly even when offline", async ({ page, app, bot, user }) => {
         test.slow();
         const charly1to5 = charlies.slice(0, 5);
         const charly6to10 = charlies.slice(5);
 
         // Set up room with alice, bob & charlies 1-5
-        await setupRoomWithBobAliceAndCharlies(page, app, bot, charly1to5);
+        await setupRoomWithBobAliceAndCharlies(page, app, user, bot, charly1to5);
         // Alice should see 2 messages from every charly with the correct display name
         await checkPaginatedDisplayNames(app, charly1to5);
 
         await openMemberlist(app);
         await checkMemberList(page, charly1to5);
-        await joinCharliesWhileAliceIsOffline(page, app, charly6to10);
+        await joinCharliesWhileAliceIsOffline(page, app, user, charly6to10);
         await checkMemberList(page, charly6to10);
 
         for (const charly of charlies) {

playwright/e2e/login/login-consent.spec.ts

@@ -12,6 +12,7 @@ import { expect, test } from "../../element-web-test";
 import { selectHomeserver } from "../utils";
 import { Credentials, HomeserverInstance } from "../../plugins/homeserver";
 import { consentHomeserver } from "../../plugins/homeserver/synapse/consentHomeserver.ts";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
 
 // This test requires fixed credentials for the device signing keys below to work
 const username = "user1234";
@@ -113,6 +114,8 @@ test.use({
 
 test.describe("Login", () => {
     test.describe("Password login", () => {
+        test.skip(isDendrite, "Dendrite lacks support for MSC3967 so requires additional auth here");
+
         test("Loads the welcome page by default; then logs in with an existing account and lands on the home screen", async ({
             credentials,
             page,

playwright/e2e/oidc/oidc-native.spec.ts

@@ -9,12 +9,10 @@ Please see LICENSE files in the repository root for full details.
 import { test, expect } from "../../element-web-test.ts";
 import { registerAccountMas } from ".";
 import { ElementAppPage } from "../../pages/ElementAppPage.ts";
-import { isDendrite } from "../../plugins/homeserver/dendrite";
 import { masHomeserver } from "../../plugins/homeserver/synapse/masHomeserver.ts";
 
 test.use(masHomeserver);
 test.describe("OIDC Native", { tag: ["@no-firefox", "@no-webkit"] }, () => {
-    test.skip(isDendrite, "does not yet support MAS");
     test.slow(); // trace recording takes a while here
 
     test("can register the oauth2 client and an account", async ({

playwright/e2e/one-to-one-chat/one-to-one-chat.spec.ts

@@ -9,12 +9,15 @@ Please see LICENSE files in the repository root for full details.
 
 import { test as base, expect } from "../../element-web-test";
 import { Credentials } from "../../plugins/homeserver";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
 
 const test = base.extend<{
     user2?: Credentials;
 }>({});
 
 test.describe("1:1 chat room", () => {
+    test.skip(isDendrite, "due to a Dendrite bug https://github.com/element-hq/dendrite/issues/3492");
+
     test.use({
         displayName: "Jeff",
         user2: async ({ homeserver }, use, testInfo) => {

playwright/e2e/permalinks/permalinks.spec.ts

@@ -31,7 +31,7 @@ test.describe("permalinks", () => {
         await charlotte.prepareClient();
 
         // We don't use a bot for danielle as we want a stable MXID.
-        const danielleId = "@danielle:localhost";
+        const danielleId = `@danielle:${user.homeServer}`;
 
         const room1Id = await app.client.createRoom({ name: room1Name });
         const room2Id = await app.client.createRoom({ name: room2Name });

playwright/e2e/polls/polls.spec.ts

@@ -11,8 +11,11 @@ import { Bot } from "../../pages/bot";
 import { SettingLevel } from "../../../src/settings/SettingLevel";
 import { Layout } from "../../../src/settings/enums/Layout";
 import type { Locator, Page } from "@playwright/test";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
 
 test.describe("Polls", () => {
+    test.skip(isDendrite, "due to a Dendrite bug https://github.com/element-hq/dendrite/issues/3492");
+
     type CreatePollOptions = {
         title: string;
         options: string[];

playwright/e2e/read-receipts/editing-messages-in-threads.spec.ts

@@ -9,8 +9,11 @@ Please see LICENSE files in the repository root for full details.
 /* See readme.md for tips on writing these tests. */
 
 import { test } from ".";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
 
 test.describe("Read receipts", { tag: "@mergequeue" }, () => {
+    test.skip(isDendrite, "due to Dendrite bug https://github.com/element-hq/dendrite/issues/2970");
+
     test.describe("editing messages", () => {
         test.describe("in threads", () => {
             test("An edit of a threaded message makes the room unread", async ({

playwright/e2e/read-receipts/editing-messages-main-timeline.spec.ts

@@ -9,8 +9,11 @@ Please see LICENSE files in the repository root for full details.
 /* See readme.md for tips on writing these tests. */
 
 import { test } from ".";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
 
 test.describe("Read receipts", { tag: "@mergequeue" }, () => {
+    test.skip(isDendrite, "due to Dendrite bug https://github.com/element-hq/dendrite/issues/2970");
+
     test.describe("editing messages", () => {
         test.describe("in the main timeline", () => {
             test("Editing a message leaves a room read", async ({ roomAlpha: room1, roomBeta: room2, util, msg }) => {