feat: support document apply remote events

frontend/appflowy_web_app/jest.config.cjs

@@ -17,4 +17,5 @@ module.exports = {
     '(.*)/node_modules/nanoid/.+\\.(j|t)sx?$': 'ts-jest',
   },
   'transformIgnorePatterns': [`/node_modules/(?!${esModules})`],
+  testMatch: ['**/*.test.ts'],
 };

frontend/appflowy_web_app/package.json

@@ -86,6 +86,7 @@
     "slate": "^0.101.4",
     "slate-history": "^0.100.0",
     "slate-react": "^0.101.3",
+    "smooth-scroll-into-view-if-needed": "^2.0.2",
     "ts-results": "^3.3.0",
     "unsplash-js": "^7.0.19",
     "utf8": "^3.0.0",

frontend/appflowy_web_app/src/application/folder-yjs/folder.type.ts

@@ -1,8 +1,9 @@
 export enum CoverType {
   NormalColor = 'color',
   GradientColor = 'gradient',
-  BuildInImage = 'none',
+  BuildInImage = 'built_in',
   CustomImage = 'custom',
   LocalImage = 'local',
   UpsplashImage = 'unsplash',
+  None = 'none',
 }

frontend/appflowy_web_app/src/application/slate-yjs/plugins/withYjs.ts

@@ -1,6 +1,5 @@
 import { CollabOrigin, YjsEditorKey, YSharedRoot } from '@/application/collab.type';
-import { applySlateOp } from '@/application/slate-yjs/utils/applySlateOpts';
-import { translateYjsEvent } from 'src/application/slate-yjs/utils/translateYjsEvent';
+import { applyToYjs } from '@/application/slate-yjs/utils/applyToYjs';
 import { Editor, Operation, Descendant } from 'slate';
 import Y, { YEvent, Transaction } from 'yjs';
 import { yDocToSlateContent } from '@/application/slate-yjs/utils/convert';
@@ -57,12 +56,11 @@ export const YjsEditor = {
 export function withYjs<T extends Editor>(
   editor: T,
   doc: Y.Doc,
-  {
-    localOrigin,
-  }: {
+  opts?: {
     localOrigin: CollabOrigin;
   }
 ): T & YjsEditor {
+  const { localOrigin = CollabOrigin.Local } = opts ?? {};
   const e = editor as T & YjsEditor;
   const { apply, onChange } = e;
 
@@ -76,23 +74,34 @@ export function withYjs<T extends Editor>(
     }
 
     e.children = content.children;
+
+    console.log('initializeDocumentContent', doc.getMap(YjsEditorKey.data_section).toJSON(), e.children);
     Editor.normalize(editor, { force: true });
   };
 
-  e.applyRemoteEvents = (events: Array<YEvent<YSharedRoot>>, _: Transaction) => {
+  const applyIntercept = (op: Operation) => {
+    if (YjsEditor.connected(e)) {
+      YjsEditor.storeLocalChange(e, op);
+    }
+
+    apply(op);
+  };
+
+  const applyRemoteIntercept = (op: Operation) => {
+    apply(op);
+  };
+
+  e.applyRemoteEvents = (_events: Array<YEvent<YSharedRoot>>, _: Transaction) => {
+    // Flush local changes to ensure all local changes are applied before processing remote events
     YjsEditor.flushLocalChanges(e);
+    // Replace the apply function to avoid storing remote changes as local changes
+    e.apply = applyRemoteIntercept;
 
-    // TODO: handle remote events
-    // This is a temporary implementation to apply remote events to slate
+    // Initialize or update the document content to ensure it is in the correct state before applying remote events
     initializeDocumentContent();
-    Editor.withoutNormalizing(editor, () => {
-      events.forEach((event) => {
-        translateYjsEvent(e.sharedRoot, editor, event).forEach((op) => {
-          // apply remote events to slate, don't call e.apply here because e.apply has been overridden.
-          apply(op);
-        });
-      });
-    });
+
+    // Restore the apply function to store local changes after applying remote changes
+    e.apply = applyIntercept;
   };
 
   const handleYEvents = (events: Array<YEvent<YSharedRoot>>, transaction: Transaction) => {
@@ -133,18 +142,12 @@ export function withYjs<T extends Editor>(
     // parse changes and apply to ydoc
     doc.transact(() => {
       changes.forEach((change) => {
-        applySlateOp(doc, { children: change.slateContent }, change.op);
+        applyToYjs(doc, { children: change.slateContent }, change.op);
       });
     }, localOrigin);
   };
 
-  e.apply = (op) => {
-    if (YjsEditor.connected(e)) {
-      YjsEditor.storeLocalChange(e, op);
-    }
-
-    apply(op);
-  };
+  e.apply = applyIntercept;
 
   e.onChange = () => {
     if (YjsEditor.connected(e)) {

frontend/appflowy_web_app/src/application/slate-yjs/utils/__tests__/applyRemoteEvents.ts

@@ -0,0 +1,67 @@
+import {
+  CollabOrigin,
+  YBlocks,
+  YChildrenMap,
+  YjsEditorKey,
+  YMeta,
+  YSharedRoot,
+  YTextMap,
+} from '@/application/collab.type';
+import { yDocToSlateContent } from '@/application/slate-yjs/utils/convert';
+import { generateId, withTestingYDoc, withTestingYjsEditor } from './withTestingYjsEditor';
+import { createEditor } from 'slate';
+import { expect } from '@jest/globals';
+import * as Y from 'yjs';
+
+export async function runApplyRemoteEventsTest() {
+  const pageId = generateId();
+  const remoteDoc = withTestingYDoc(pageId);
+  const remote = withTestingYjsEditor(createEditor(), remoteDoc);
+
+  const localDoc = new Y.Doc();
+
+  Y.applyUpdateV2(localDoc, Y.encodeStateAsUpdateV2(remoteDoc));
+  const editor = withTestingYjsEditor(createEditor(), localDoc);
+
+  editor.connect();
+  expect(editor.children).toEqual(remote.children);
+
+  // update remote doc
+  insertBlock(remoteDoc, generateId(), pageId, 0);
+  remote.children = yDocToSlateContent(remoteDoc)?.children ?? [];
+
+  // apply remote changes to local doc
+  Y.transact(
+    localDoc,
+    () => {
+      Y.applyUpdateV2(localDoc, Y.encodeStateAsUpdateV2(remoteDoc));
+    },
+    CollabOrigin.Remote
+  );
+
+  expect(editor.children).toEqual(remote.children);
+}
+
+function insertBlock(doc: Y.Doc, blockId: string, parentId: string, index: number) {
+  const sharedRoot = doc.getMap(YjsEditorKey.data_section) as YSharedRoot;
+  const document = sharedRoot.get(YjsEditorKey.document);
+  const blocks = document.get(YjsEditorKey.blocks) as YBlocks;
+  const meta = document.get(YjsEditorKey.meta) as YMeta;
+  const childrenMap = meta.get(YjsEditorKey.children_map) as YChildrenMap;
+  const textMap = meta.get(YjsEditorKey.text_map) as YTextMap;
+
+  const block = new Y.Map();
+
+  block.set(YjsEditorKey.block_id, blockId);
+  block.set(YjsEditorKey.block_children, blockId);
+  block.set(YjsEditorKey.block_type, 'paragraph');
+  block.set(YjsEditorKey.block_data, '{}');
+  block.set(YjsEditorKey.block_external_id, blockId);
+  blocks.set(blockId, block);
+  childrenMap.set(blockId, new Y.Array());
+  childrenMap.get(parentId).insert(index, [blockId]);
+  const text = new Y.Text();
+
+  text.insert(0, 'Hello, World!');
+  textMap.set(blockId, text);
+}

frontend/appflowy_web_app/src/application/slate-yjs/utils/__tests__/convert.ts

@@ -0,0 +1,41 @@
+import { withTestingYDoc, withTestingYjsEditor } from './withTestingYjsEditor';
+import { yDocToSlateContent } from '../convert';
+import { createEditor, Editor } from 'slate';
+import { expect } from '@jest/globals';
+import * as Y from 'yjs';
+
+function normalizedSlateDoc(doc: Y.Doc) {
+  const editor = createEditor();
+
+  const yjsEditor = withTestingYjsEditor(editor, doc);
+
+  editor.children = yDocToSlateContent(doc)?.children ?? [];
+  return yjsEditor.children;
+}
+
+export async function runCollaborationTest() {
+  const doc = withTestingYDoc('1');
+  const editor = createEditor();
+  const yjsEditor = withTestingYjsEditor(editor, doc);
+
+  // Keep the 'local' editor state before applying run.
+  const baseState = Y.encodeStateAsUpdateV2(doc);
+
+  Editor.normalize(editor, { force: true });
+
+  expect(normalizedSlateDoc(doc)).toEqual(yjsEditor.children);
+
+  // Setup remote editor with input base state
+  const remoteDoc = new Y.Doc();
+
+  Y.applyUpdateV2(remoteDoc, baseState);
+  const remote = withTestingYjsEditor(createEditor(), remoteDoc);
+
+  // Apply changes from 'run'
+  Y.applyUpdateV2(remoteDoc, Y.encodeStateAsUpdateV2(yjsEditor.sharedRoot.doc!));
+
+  // Verify remote and editor state are equal
+  expect(normalizedSlateDoc(remoteDoc)).toEqual(remote.children);
+  expect(yjsEditor.children).toEqual(remote.children);
+  expect(normalizedSlateDoc(doc)).toEqual(yjsEditor.children);
+}

frontend/appflowy_web_app/src/application/slate-yjs/utils/__tests__/index.test.ts

@@ -0,0 +1,12 @@
+import { runCollaborationTest } from './convert';
+import { runApplyRemoteEventsTest } from './applyRemoteEvents';
+
+describe('slate-yjs adapter', () => {
+  it('should pass the collaboration test', async () => {
+    await runCollaborationTest();
+  });
+
+  it('should pass the apply remote events test', async () => {
+    await runApplyRemoteEventsTest();
+  });
+});

frontend/appflowy_web_app/src/application/slate-yjs/utils/__tests__/withTestingYjsEditor.ts

@@ -0,0 +1,45 @@
+import { CollabOrigin, YjsEditorKey, YSharedRoot } from '@/application/collab.type';
+import { withYjs } from '@/application/slate-yjs';
+import { Editor } from 'slate';
+import * as Y from 'yjs';
+import { v4 as uuidv4 } from 'uuid';
+
+export function generateId() {
+  return uuidv4();
+}
+
+export function withTestingYjsEditor(editor: Editor, doc: Y.Doc) {
+  const yjdEditor = withYjs(editor, doc, {
+    localOrigin: CollabOrigin.LocalSync,
+  });
+
+  return yjdEditor;
+}
+
+export function withTestingYDoc(docId: string) {
+  const doc = new Y.Doc();
+  const sharedRoot = doc.getMap(YjsEditorKey.data_section) as YSharedRoot;
+  const document = new Y.Map();
+  const blocks = new Y.Map();
+  const meta = new Y.Map();
+  const children_map = new Y.Map();
+  const text_map = new Y.Map();
+  const rootBlock = new Y.Map();
+  const blockOrders = new Y.Array();
+  const pageId = docId;
+
+  sharedRoot.set(YjsEditorKey.document, document);
+  document.set(YjsEditorKey.page_id, pageId);
+  document.set(YjsEditorKey.blocks, blocks);
+  document.set(YjsEditorKey.meta, meta);
+  meta.set(YjsEditorKey.children_map, children_map);
+  meta.set(YjsEditorKey.text_map, text_map);
+  children_map.set(pageId, blockOrders);
+  blocks.set(pageId, rootBlock);
+  rootBlock.set(YjsEditorKey.block_id, pageId);
+  rootBlock.set(YjsEditorKey.block_children, pageId);
+  rootBlock.set(YjsEditorKey.block_type, 'page');
+  rootBlock.set(YjsEditorKey.block_data, '{}');
+  rootBlock.set(YjsEditorKey.block_external_id, '');
+  return doc;
+}

frontend/appflowy_web_app/src/application/slate-yjs/utils/applyToYjs.ts

@@ -0,0 +1,8 @@
+import { Operation, Node } from 'slate';
+import * as Y from 'yjs';
+
+// transform slate op to yjs op and apply it to ydoc
+export function applyToYjs(_ydoc: Y.Doc, _slateRoot: Node, op: Operation) {
+  if (op.type === 'set_selection') return;
+  console.log('applySlateOp', op);
+}

frontend/appflowy_web_app/src/application/slate-yjs/utils/convert.ts

@@ -10,22 +10,14 @@ import {
   BlockData,
   BlockType,
 } from '@/application/collab.type';
+import { BlockJson } from '@/application/slate-yjs/utils/types';
 import { getFontFamily } from '@/utils/font';
 import { uniq } from 'lodash-es';
 import { Element, Text } from 'slate';
 
-interface BlockJson {
-  id: string;
-  ty: string;
-  data?: string;
-  children?: string;
-  external_id?: string;
-}
-
 export function yDocToSlateContent(doc: YDoc): Element | undefined {
   const sharedRoot = doc.getMap(YjsEditorKey.data_section) as YSharedRoot;
 
-  console.log(sharedRoot.toJSON());
   const document = sharedRoot.get(YjsEditorKey.document);
   const pageId = document.get(YjsEditorKey.page_id) as string;
   const blocks = document.get(YjsEditorKey.blocks) as YBlocks;