diff --git a/spec/unit/matrixrtc/MatrixRTCSession.spec.ts b/spec/unit/matrixrtc/MatrixRTCSession.spec.ts
index 046dea947a1..545c3923b85 100644
--- a/spec/unit/matrixrtc/MatrixRTCSession.spec.ts
+++ b/spec/unit/matrixrtc/MatrixRTCSession.spec.ts
@@ -17,6 +17,7 @@ limitations under the License.
 import { EventTimeline, EventType, MatrixClient, MatrixError, MatrixEvent, Room } from "../../../src";
 import { CallMembershipData } from "../../../src/matrixrtc/CallMembership";
 import { MatrixRTCSession, MatrixRTCSessionEvent } from "../../../src/matrixrtc/MatrixRTCSession";
+import { EncryptionKeysEventContent } from "../../../src/matrixrtc/types";
 import { randomString } from "../../../src/randomstring";
 import { makeMockRoom, makeMockRoomState, mockRTCEvent } from "./mocks";
 
@@ -420,6 +421,55 @@ describe("MatrixRTCSession", () => {
             }
         });
 
+        it("Rotates key if a member leaves", async () => {
+            jest.useFakeTimers();
+            try {
+                const member2 = Object.assign({}, membershipTemplate, {
+                    device_id: "BBBBBBB",
+                });
+                const mockRoom = makeMockRoom([membershipTemplate, member2]);
+                sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+
+                const onMyEncryptionKeyChanged = jest.fn();
+                sess.on(
+                    MatrixRTCSessionEvent.EncryptionKeyChanged,
+                    (_key: Uint8Array, _idx: number, participantId: string) => {
+                        if (participantId === `${client.getUserId()}:${client.getDeviceId()}`) {
+                            onMyEncryptionKeyChanged();
+                        }
+                    },
+                );
+
+                const keysSentPromise1 = new Promise<EncryptionKeysEventContent>((resolve) => {
+                    sendEventMock.mockImplementation((_roomId, _evType, payload) => resolve(payload));
+                });
+
+                sess.joinRoomSession([mockFocus], true);
+                const firstKeysPayload = await keysSentPromise1;
+                expect(firstKeysPayload.keys).toHaveLength(1);
+
+                sendEventMock.mockClear();
+
+                const keysSentPromise2 = new Promise<EncryptionKeysEventContent>((resolve) => {
+                    sendEventMock.mockImplementation((_roomId, _evType, payload) => resolve(payload));
+                });
+
+                mockRoom.getLiveTimeline().getState = jest
+                    .fn()
+                    .mockReturnValue(makeMockRoomState([membershipTemplate], mockRoom.roomId, undefined));
+                sess.onMembershipUpdate();
+
+                jest.advanceTimersByTime(10000);
+
+                const secondKeysPayload = await keysSentPromise2;
+
+                expect(secondKeysPayload.keys).toHaveLength(2);
+                expect(onMyEncryptionKeyChanged).toHaveBeenCalledTimes(2);
+            } finally {
+                jest.useRealTimers();
+            }
+        });
+
         it("Doesn't re-send key immediately", async () => {
             const realSetImmediate = setImmediate;
             jest.useFakeTimers();
@@ -643,4 +693,26 @@ describe("MatrixRTCSession", () => {
         expect(bobKeys[3]).toBeFalsy();
         expect(bobKeys[4]).toEqual(Buffer.from("this is the key", "utf-8"));
     });
+
+    it("ignores keys event for the local participant", () => {
+        const mockRoom = makeMockRoom([membershipTemplate]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        sess.onCallEncryption({
+            getType: jest.fn().mockReturnValue("io.element.call.encryption_keys"),
+            getContent: jest.fn().mockReturnValue({
+                device_id: client.getDeviceId(),
+                call_id: "",
+                keys: [
+                    {
+                        index: 4,
+                        key: "dGhpcyBpcyB0aGUga2V5",
+                    },
+                ],
+            }),
+            getSender: jest.fn().mockReturnValue(client.getUserId()),
+        } as unknown as MatrixEvent);
+
+        const myKeys = sess.getKeysForParticipant(client.getUserId()!, client.getDeviceId()!)!;
+        expect(myKeys).toBeFalsy();
+    });
 });
diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts
index 43348ff3ec2..bba60704379 100644
--- a/src/matrixrtc/MatrixRTCSession.ts
+++ b/src/matrixrtc/MatrixRTCSession.ts
@@ -26,13 +26,21 @@ import { MatrixError, MatrixEvent } from "../matrix";
 import { randomString, secureRandomBase64Url } from "../randomstring";
 import { EncryptionKeysEventContent } from "./types";
 import { decodeBase64, encodeUnpaddedBase64 } from "../base64";
-import { isNumber } from "../utils";
 
 const MEMBERSHIP_EXPIRY_TIME = 60 * 60 * 1000;
 const MEMBER_EVENT_CHECK_PERIOD = 2 * 60 * 1000; // How often we check to see if we need to re-send our member event
 const CALL_MEMBER_EVENT_RETRY_DELAY_MIN = 3000;
 const UPDATE_ENCRYPTION_KEY_THROTTLE = 3000;
 
+// A delay after a member leaves before we create and publish a new key, because people
+// tend to leave calls at the same time
+const MAKE_KEY_DELAY = 3000;
+// The delay between creating and sending a new key and starting to encrypt with it. This gives others
+// a chance to receive the new key to minimise the chance they don't get media they can't decrypt.
+// The total time between a member leaving and the call switching to new keys is therefore
+// MAKE_KEY_DELAY + SEND_KEY_DELAY
+const USE_KEY_DELAY = 5000;
+
 const getParticipantId = (userId: string, deviceId: string): string => `${userId}:${deviceId}`;
 const getParticipantIdFromMembership = (m: CallMembership): string => getParticipantId(m.sender!, m.deviceId);
 
@@ -85,6 +93,8 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
     private memberEventTimeout?: ReturnType<typeof setTimeout>;
     private expiryTimeout?: ReturnType<typeof setTimeout>;
     private keysEventUpdateTimeout?: ReturnType<typeof setTimeout>;
+    private makeNewKeyTimeout?: ReturnType<typeof setTimeout>;
+    private setNewKeyTimeouts = new Set<ReturnType<typeof setTimeout>>();
 
     private activeFoci: Focus[] | undefined;
 
@@ -253,6 +263,15 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
         // as they may still be using the same ones.
         this.encryptionKeys.set(getParticipantId(userId, deviceId), []);
 
+        if (this.makeNewKeyTimeout !== undefined) {
+            clearTimeout(this.makeNewKeyTimeout);
+            this.makeNewKeyTimeout = undefined;
+        }
+        for (const t of this.setNewKeyTimeouts) {
+            clearTimeout(t);
+        }
+        this.setNewKeyTimeouts.clear();
+
         logger.info(`Leaving call session in room ${this.room.roomId}`);
         this.relativeExpiry = undefined;
         this.activeFoci = undefined;
@@ -297,11 +316,24 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
         return (this.getKeysForParticipant(userId, deviceId)?.length ?? 0) % 16;
     }
 
+    /**
+     * Sets an encryption key at a specified index for a participant.
+     * The encryption keys for the local participanmt are also stored here under the
+     * user and device ID of the local participant.
+     * @param userId - The user ID of the participant
+     * @param deviceId - Device ID of the participant
+     * @param encryptionKeyIndex - The index of the key to set
+     * @param encryptionKeyString - The string represenation of the key to set in base64
+     * @param delayBeforeuse - If true, delay before emitting a key changed event. Useful when setting
+     *                         encryption keys for the local participant to allow time for the key to
+     *                         be distributed.
+     */
     private setEncryptionKey(
         userId: string,
         deviceId: string,
         encryptionKeyIndex: number,
         encryptionKeyString: string,
+        delayBeforeuse = false,
     ): void {
         const keyBin = decodeBase64(encryptionKeyString);
 
@@ -312,13 +344,24 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
 
         encryptionKeys[encryptionKeyIndex] = keyBin;
         this.encryptionKeys.set(participantId, encryptionKeys);
-        this.emit(MatrixRTCSessionEvent.EncryptionKeyChanged, keyBin, encryptionKeyIndex, participantId);
+        if (delayBeforeuse) {
+            const useKeyTimeout = setTimeout(() => {
+                this.setNewKeyTimeouts.delete(useKeyTimeout);
+                logger.info(`Delayed-emitting key changed event for ${participantId} idx ${encryptionKeyIndex}`);
+                this.emit(MatrixRTCSessionEvent.EncryptionKeyChanged, keyBin, encryptionKeyIndex, participantId);
+            }, USE_KEY_DELAY);
+            this.setNewKeyTimeouts.add(useKeyTimeout);
+        } else {
+            this.emit(MatrixRTCSessionEvent.EncryptionKeyChanged, keyBin, encryptionKeyIndex, participantId);
+        }
     }
 
     /**
      * Generate a new sender key and add it at the next available index
+     * @param delayBeforeUse - If true, wait for a short period before settign the key for the
+     *                         media encryptor to use. If false, set the key immediately.
      */
-    private makeNewSenderKey(): void {
+    private makeNewSenderKey(delayBeforeUse = false): void {
         const userId = this.client.getUserId();
         const deviceId = this.client.getDeviceId();
 
@@ -328,7 +371,7 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
         const encryptionKey = secureRandomBase64Url(16);
         const encryptionKeyIndex = this.getNewEncryptionKeyIndex();
         logger.info("Generated new key at index " + encryptionKeyIndex);
-        this.setEncryptionKey(userId, deviceId, encryptionKeyIndex, encryptionKey);
+        this.setEncryptionKey(userId, deviceId, encryptionKeyIndex, encryptionKey, delayBeforeUse);
     }
 
     /**
@@ -393,6 +436,7 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
 
             logger.debug(
                 `Embedded-E2EE-LOG updateEncryptionKeyEvent participantId=${userId}:${deviceId} numSent=${myKeys.length}`,
+                this.encryptionKeys,
             );
         } catch (error) {
             const matrixError = error as MatrixError;
@@ -462,9 +506,17 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
             return;
         }
 
+        if (userId === this.client.getUserId() && deviceId === this.client.getDeviceId()) {
+            // We store our own sender key in the same set along with keys from others, so it's
+            // important we don't allow our own keys to be set by one of these events (apart from
+            // the fact that we don't need it anyway because we already know our own keys).
+            logger.info("Ignoring our own keys event");
+            return;
+        }
+
         for (const key of content.keys) {
-            if (!key.key || !isNumber(key.index)) {
-                logger.warn(`Received m.call.encryption_keys with invalid entry: callId=${callId}`);
+            if (!key) {
+                logger.info("Ignoring false-y key in keys event");
                 continue;
             }
 
@@ -510,20 +562,25 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
 
         const isMyMembership = (m: CallMembership): boolean =>
             m.sender === this.client.getUserId() && m.deviceId === this.client.getDeviceId();
-        const callMembersChanged =
-            oldMemberships
-                .filter((m) => !isMyMembership(m))
-                .map(getParticipantIdFromMembership)
-                .sort()
-                .join() !==
-            this.memberships
-                .filter((m) => !isMyMembership(m))
-                .map(getParticipantIdFromMembership)
-                .sort()
-                .join();
-
-        if (callMembersChanged && this.isJoined()) {
-            this.requestKeyEventSend();
+
+        if (this.isJoined() && this.makeNewKeyTimeout === undefined) {
+            const oldMebershipIds = new Set(
+                oldMemberships.filter((m) => !isMyMembership(m)).map(getParticipantIdFromMembership),
+            );
+            const newMebershipIds = new Set(
+                this.memberships.filter((m) => !isMyMembership(m)).map(getParticipantIdFromMembership),
+            );
+
+            const anyLeft = Array.from(oldMebershipIds).some((x) => !newMebershipIds.has(x));
+            const anyJoined = Array.from(newMebershipIds).some((x) => !oldMebershipIds.has(x));
+
+            if (anyLeft) {
+                logger.debug(`Member(s) have left: queueing sender key rotation`);
+                this.makeNewKeyTimeout = setTimeout(this.onRotateKeyTimeout, MAKE_KEY_DELAY);
+            } else if (anyJoined) {
+                logger.debug(`New member(s) have joined: re-sending keys`);
+                this.requestKeyEventSend();
+            }
         }
 
         this.setExpiryTimer();
@@ -708,4 +765,13 @@ export class MatrixRTCSession extends TypedEventEmitter<MatrixRTCSessionEvent, M
             await this.triggerCallMembershipEventUpdate();
         }
     }
+
+    private onRotateKeyTimeout = (): void => {
+        this.makeNewKeyTimeout = undefined;
+        logger.info("Making new sender key for key rotation");
+        this.makeNewSenderKey(true);
+        // send immediately: if we're about to start sending with a new key, it's
+        // important we get it out to others as soon as we can.
+        this.sendEncryptionKeysEvent();
+    };
 }
diff --git a/src/matrixrtc/MatrixRTCSessionManager.ts b/src/matrixrtc/MatrixRTCSessionManager.ts
index ba1eb0fa1dd..e0ca3142b69 100644
--- a/src/matrixrtc/MatrixRTCSessionManager.ts
+++ b/src/matrixrtc/MatrixRTCSessionManager.ts
@@ -121,7 +121,9 @@ export class MatrixRTCSessionManager extends TypedEventEmitter<MatrixRTCSessionM
             return;
         }
 
-        this.refreshRoom(room);
+        if (event.getType() == EventType.GroupCallMemberPrefix) {
+            this.refreshRoom(room);
+        }
     };
 
     private refreshRoom(room: Room): void {