[WEB-1322] dev: conflict free pages collaboration (#4463)

* chore: pages realtime

* chore: empty binary response

* chore: added a ypy package

* feat: pages collaboration

* chore: update fetching logic

* chore: degrade ypy version

* chore: replace useEffect fetch logic with useSWR

* chore: move all the update logic to the page store

* refactor: remove react-hook-form

* chore: save description_html as well

* chore: migrate old data logic

* fix: added description_binary as field name

* fix: code cleanup

* refactor: create separate hook to handle page description

* fix: build errors

* chore: combine updates instead of using the whole document

* chore: removed ypy package

* chore: added conflict resolving logic to the client side

* chore: add a save changes button

* chore: add read-only validation

* chore: remove saving state information

* chore: added permission class

* chore: removed the migration file

* chore: corrected the model field

* chore: rename pageStore to page

* chore: update collaboration provider

* chore: add try catch to handle error

---------

Co-authored-by: NarayanBavisetti <narayan3119@gmail.com>

packages/editor/document-editor/src/hooks/use-document-editor.ts

@@ -0,0 +1,85 @@
+import { useEffect, useLayoutEffect, useMemo } from "react";
+import { EditorProps } from "@tiptap/pm/view";
+import { IndexeddbPersistence } from "y-indexeddb";
+import * as Y from "yjs";
+// editor-core
+import { EditorRefApi, IMentionHighlight, IMentionSuggestion, TFileHandler, useEditor } from "@plane/editor-core";
+// custom provider
+import { CollaborationProvider } from "src/providers/collaboration-provider";
+// extensions
+import { DocumentEditorExtensions } from "src/ui/extensions";
+
+type DocumentEditorProps = {
+  id: string;
+  fileHandler: TFileHandler;
+  value: Uint8Array;
+  editorClassName: string;
+  onChange: (updates: Uint8Array) => void;
+  editorProps?: EditorProps;
+  forwardedRef?: React.MutableRefObject<EditorRefApi | null>;
+  mentionHandler: {
+    highlights: () => Promise<IMentionHighlight[]>;
+    suggestions?: () => Promise<IMentionSuggestion[]>;
+  };
+  handleEditorReady?: (value: boolean) => void;
+  placeholder?: string | ((isFocused: boolean, value: string) => string);
+  setHideDragHandleFunction: (hideDragHandlerFromDragDrop: () => void) => void;
+  tabIndex?: number;
+};
+
+export const useDocumentEditor = ({
+  id,
+  editorProps = {},
+  value,
+  editorClassName,
+  fileHandler,
+  onChange,
+  forwardedRef,
+  tabIndex,
+  handleEditorReady,
+  mentionHandler,
+  placeholder,
+  setHideDragHandleFunction,
+}: DocumentEditorProps) => {
+  const provider = useMemo(
+    () =>
+      new CollaborationProvider({
+        name: id,
+        onChange,
+      }),
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [id]
+  );
+
+  // update document on value change
+  useEffect(() => {
+    if (value.byteLength > 0) Y.applyUpdate(provider.document, value);
+  }, [value, provider.document]);
+
+  // indexedDB provider
+  useLayoutEffect(() => {
+    const localProvider = new IndexeddbPersistence(id, provider.document);
+    return () => {
+      localProvider?.destroy();
+    };
+  }, [provider, id]);
+
+  const editor = useEditor({
+    id,
+    editorProps,
+    editorClassName,
+    fileHandler,
+    handleEditorReady,
+    forwardedRef,
+    mentionHandler,
+    extensions: DocumentEditorExtensions({
+      uploadFile: fileHandler.upload,
+      setHideDragHandle: setHideDragHandleFunction,
+      provider,
+    }),
+    placeholder,
+    tabIndex,
+  });
+
+  return editor;
+};

packages/editor/document-editor/src/index.ts

@@ -3,6 +3,8 @@ export { DocumentReadOnlyEditor, DocumentReadOnlyEditorWithRef } from "src/ui/re
 
 // hooks
 export { useEditorMarkings } from "src/hooks/use-editor-markings";
+// utils
+export { proseMirrorJSONToBinaryString, applyUpdates, mergeUpdates } from "src/utils/yjs";
 
 export type { EditorRefApi, EditorReadOnlyRefApi, EditorMenuItem, EditorMenuItemNames } from "@plane/editor-core";
 

packages/editor/document-editor/src/providers/collaboration-provider.ts

@@ -0,0 +1,60 @@
+import * as Y from "yjs";
+
+export interface CompleteCollaboratorProviderConfiguration {
+  /**
+   * The identifier/name of your document
+   */
+  name: string;
+  /**
+   * The actual Y.js document
+   */
+  document: Y.Doc;
+  /**
+   * onChange callback
+   */
+  onChange: (updates: Uint8Array) => void;
+}
+
+export type CollaborationProviderConfiguration = Required<Pick<CompleteCollaboratorProviderConfiguration, "name">> &
+  Partial<CompleteCollaboratorProviderConfiguration>;
+
+export class CollaborationProvider {
+  public configuration: CompleteCollaboratorProviderConfiguration = {
+    name: "",
+    // @ts-expect-error cannot be undefined
+    document: undefined,
+    onChange: () => {},
+  };
+
+  constructor(configuration: CollaborationProviderConfiguration) {
+    this.setConfiguration(configuration);
+
+    this.configuration.document = configuration.document ?? new Y.Doc();
+    this.document.on("update", this.documentUpdateHandler.bind(this));
+    this.document.on("destroy", this.documentDestroyHandler.bind(this));
+  }
+
+  public setConfiguration(configuration: Partial<CompleteCollaboratorProviderConfiguration> = {}): void {
+    this.configuration = {
+      ...this.configuration,
+      ...configuration,
+    };
+  }
+
+  get document() {
+    return this.configuration.document;
+  }
+
+  documentUpdateHandler(update: Uint8Array, origin: any) {
+    // return if the update is from the provider itself
+    if (origin === this) return;
+
+    // call onChange with the update
+    this.configuration.onChange?.(update);
+  }
+
+  documentDestroyHandler() {
+    this.document.off("update", this.documentUpdateHandler);
+    this.document.off("destroy", this.documentDestroyHandler);
+  }
+}

packages/editor/document-editor/src/ui/extensions/index.tsx

@@ -2,14 +2,20 @@ import { IssueWidgetPlaceholder } from "src/ui/extensions/widgets/issue-embed-wi
 
 import { SlashCommand, DragAndDrop } from "@plane/editor-extensions";
 import { UploadImage } from "@plane/editor-core";
+import { CollaborationProvider } from "src/providers/collaboration-provider";
+import Collaboration from "@tiptap/extension-collaboration";
 
 type TArguments = {
   uploadFile: UploadImage;
   setHideDragHandle?: (hideDragHandlerFromDragDrop: () => void) => void;
+  provider: CollaborationProvider;
 };
 
-export const DocumentEditorExtensions = ({ uploadFile, setHideDragHandle }: TArguments) => [
+export const DocumentEditorExtensions = ({ uploadFile, setHideDragHandle, provider }: TArguments) => [
   SlashCommand(uploadFile),
   DragAndDrop(setHideDragHandle),
   IssueWidgetPlaceholder(),
+  Collaboration.configure({
+    document: provider.document,
+  }),
 ];

packages/editor/document-editor/src/ui/index.tsx

@@ -1,30 +1,25 @@
 import React, { useState } from "react";
+// editor-core
 import {
-  UploadImage,
-  DeleteImage,
-  RestoreImage,
   getEditorClassNames,
-  useEditor,
   EditorRefApi,
   IMentionHighlight,
   IMentionSuggestion,
+  TFileHandler,
 } from "@plane/editor-core";
-import { DocumentEditorExtensions } from "src/ui/extensions";
+// components
 import { PageRenderer } from "src/ui/components/page-renderer";
+// hooks
+import { useDocumentEditor } from "src/hooks/use-document-editor";
 
 interface IDocumentEditor {
-  initialValue: string;
-  value?: string;
-  fileHandler: {
-    cancel: () => void;
-    delete: DeleteImage;
-    upload: UploadImage;
-    restore: RestoreImage;
-  };
+  id: string;
+  value: Uint8Array;
+  fileHandler: TFileHandler;
   handleEditorReady?: (value: boolean) => void;
   containerClassName?: string;
   editorClassName?: string;
-  onChange: (json: object, html: string) => void;
+  onChange: (updates: Uint8Array) => void;
   forwardedRef?: React.MutableRefObject<EditorRefApi | null>;
   mentionHandler: {
     highlights: () => Promise<IMentionHighlight[]>;
@@ -37,7 +32,7 @@ interface IDocumentEditor {
 const DocumentEditor = (props: IDocumentEditor) => {
   const {
     onChange,
-    initialValue,
+    id,
     value,
     fileHandler,
     containerClassName,
@@ -50,32 +45,24 @@ const DocumentEditor = (props: IDocumentEditor) => {
   } = props;
   // states
   const [hideDragHandleOnMouseLeave, setHideDragHandleOnMouseLeave] = useState<() => void>(() => {});
-
   // this essentially sets the hideDragHandle function from the DragAndDrop extension as the Plugin
   // loads such that we can invoke it from react when the cursor leaves the container
   const setHideDragHandleFunction = (hideDragHandlerFromDragDrop: () => void) => {
     setHideDragHandleOnMouseLeave(() => hideDragHandlerFromDragDrop);
   };
-  // use editor
-  const editor = useEditor({
-    onChange(json, html) {
-      onChange(json, html);
-    },
+
+  // use document editor
+  const editor = useDocumentEditor({
+    id,
     editorClassName,
-    restoreFile: fileHandler.restore,
-    uploadFile: fileHandler.upload,
-    deleteFile: fileHandler.delete,
-    cancelUploadImage: fileHandler.cancel,
-    initialValue,
+    fileHandler,
     value,
+    onChange,
     handleEditorReady,
     forwardedRef,
     mentionHandler,
-    extensions: DocumentEditorExtensions({
-      uploadFile: fileHandler.upload,
-      setHideDragHandle: setHideDragHandleFunction,
-    }),
     placeholder,
+    setHideDragHandleFunction,
     tabIndex,
   });
 

packages/editor/document-editor/src/utils/yjs.ts

@@ -0,0 +1,76 @@
+import { Schema } from "@tiptap/pm/model";
+import { prosemirrorJSONToYDoc } from "y-prosemirror";
+import * as Y from "yjs";
+
+const defaultSchema: Schema = new Schema({
+  nodes: {
+    text: {},
+    doc: { content: "text*" },
+  },
+});
+
+/**
+ * @description converts ProseMirror JSON to Yjs document
+ * @param document prosemirror JSON
+ * @param fieldName
+ * @param schema
+ * @returns {Y.Doc} Yjs document
+ */
+export const proseMirrorJSONToBinaryString = (
+  document: any,
+  fieldName: string | Array<string> = "default",
+  schema?: Schema
+): string => {
+  if (!document) {
+    throw new Error(
+      `You've passed an empty or invalid document to the Transformer. Make sure to pass ProseMirror-compatible JSON. Actually passed JSON: ${document}`
+    );
+  }
+
+  // allow a single field name
+  if (typeof fieldName === "string") {
+    const yDoc = prosemirrorJSONToYDoc(schema ?? defaultSchema, document, fieldName);
+    const docAsUint8Array = Y.encodeStateAsUpdate(yDoc);
+    const base64Doc = Buffer.from(docAsUint8Array).toString("base64");
+    return base64Doc;
+  }
+
+  const yDoc = new Y.Doc();
+
+  fieldName.forEach((field) => {
+    const update = Y.encodeStateAsUpdate(prosemirrorJSONToYDoc(schema ?? defaultSchema, document, field));
+
+    Y.applyUpdate(yDoc, update);
+  });
+
+  const docAsUint8Array = Y.encodeStateAsUpdate(yDoc);
+  const base64Doc = Buffer.from(docAsUint8Array).toString("base64");
+
+  return base64Doc;
+};
+
+/**
+ * @description apply updates to a doc and return the updated doc in base64(binary) format
+ * @param {Uint8Array} document
+ * @param {Uint8Array} updates
+ * @returns {string} base64(binary) form of the updated doc
+ */
+export const applyUpdates = (document: Uint8Array, updates: Uint8Array): string => {
+  const yDoc = new Y.Doc();
+  Y.applyUpdate(yDoc, document);
+  Y.applyUpdate(yDoc, updates);
+
+  const encodedDoc = Y.encodeStateAsUpdate(yDoc);
+  const base64Updates = Buffer.from(encodedDoc).toString("base64");
+  return base64Updates;
+};
+
+/**
+ * @description merge multiple updates into one single update
+ * @param {Uint8Array[]} updates
+ * @returns {Uint8Array} merged updates
+ */
+export const mergeUpdates = (updates: Uint8Array[]): Uint8Array => {
+  const mergedUpdates = Y.mergeUpdates(updates);
+  return mergedUpdates;
+};