[WEB-1116] feat: pages realtime collaboration (#5493)

* [WEB-1116] feat: pages realtime sync (#5057)

* init: live server for editor realtime sync

* chore: authentication added

* chore: updated logic to convert html to binary for old pages

* chore: added description json on page update

* chore: made all functions generic

* chore: save description in json and html formats

* refactor: document editor components

* chore: uncomment ui package components

* fix: without props extensions refactor

* fix: merge conflicts resolved from preview

* chore: init docker compose

* chore: pages custom error codes

* chore: add health check endpoint to the live server

* chore: update without props extensions type

* chore: better error handling

* chore: update react-hook-form versions

---------

Co-authored-by: NarayanBavisetti <narayan3119@gmail.com>
Co-authored-by: sriram veeraghanta <veeraghanta.sriram@gmail.com>

* fix: docker related fixes

* fix: module type fixes

* fix: nginx update

* fix: adding live server workflow

* fix: workflow fixes

* fix: docker compose fixes

* fix: workflow fixes

* fix: path config

* fix: docker compose warnings

* fix: nginx port forwarding

* fix: update docker compose with new env

* fix: env var fixes

* fix: error handling

* fix: docker compose env var

* fix: compose fixes

* chore: update server start message

* chore: handle errors

* fix: build errors

* chore: update port

* chore: update server port

* chore: show error on authentication fail

* chore: show error on authentication fail

* feat: add redis extension

* chore: updated restore version logic

---------

Co-authored-by: NarayanBavisetti <narayan3119@gmail.com>
Co-authored-by: sriram veeraghanta <veeraghanta.sriram@gmail.com>
Co-authored-by: Palanikannan M <akashmalinimurugu@gmail.com>

live/src/core/lib/authentication.ts

@@ -0,0 +1,62 @@
+import { ConnectionConfiguration } from "@hocuspocus/server";
+// services
+import { UserService } from "../services/user.service.js";
+// types
+import { TDocumentTypes } from "../types/common.js";
+
+const userService = new UserService();
+
+type Props = {
+  connection: ConnectionConfiguration;
+  cookie: string;
+  params: URLSearchParams;
+  token: string;
+};
+
+export const handleAuthentication = async (props: Props) => {
+  const { connection, cookie, params, token } = props;
+  // params
+  const workspaceSlug = params.get("workspaceSlug")?.toString();
+  const projectId = params.get("projectId")?.toString();
+  const documentType = params.get("documentType")?.toString() as
+    | TDocumentTypes
+    | undefined;
+  // fetch current user info
+  let response;
+  try {
+    response = await userService.currentUser(cookie);
+  } catch (error) {
+    console.error("Failed to fetch current user:", error);
+    throw error;
+  }
+  if (response.id !== token) {
+    throw Error("Authentication failed: Token doesn't match the current user.");
+  }
+
+  if (documentType === "project_page") {
+    if (!workspaceSlug || !projectId) {
+      throw Error(
+        "Authentication failed: Incomplete query params. Either workspaceSlug or projectId is missing."
+      );
+    }
+    // fetch current user's roles
+    const workspaceRoles = await userService.getUserAllProjectsRole(
+      workspaceSlug,
+      cookie
+    );
+    const currentProjectRole = workspaceRoles[projectId];
+    // make the connection read only for roles lower than a member
+    if (currentProjectRole < 15) {
+      connection.readOnly = true;
+    }
+  } else {
+    throw Error("Authentication failed: Invalid document type provided.");
+  }
+
+  return {
+    user: {
+      id: response.id,
+      name: response.display_name,
+    },
+  };
+};

live/src/core/lib/page.ts

@@ -0,0 +1,144 @@
+import { getSchema } from "@tiptap/core";
+import { generateHTML, generateJSON } from "@tiptap/html";
+import * as Y from "yjs";
+import {
+  prosemirrorJSONToYDoc,
+  yXmlFragmentToProseMirrorRootNode,
+} from "y-prosemirror";
+// editor
+import {
+  CoreEditorExtensionsWithoutProps,
+  DocumentEditorExtensionsWithoutProps,
+} from "@plane/editor/lib";
+// services
+import { PageService } from "../services/page.service.js";
+const pageService = new PageService();
+
+const DOCUMENT_EDITOR_EXTENSIONS = [
+  ...CoreEditorExtensionsWithoutProps,
+  ...DocumentEditorExtensionsWithoutProps,
+];
+const documentEditorSchema = getSchema(DOCUMENT_EDITOR_EXTENSIONS);
+
+export const updatePageDescription = async (
+  params: URLSearchParams,
+  pageId: string,
+  updatedDescription: Uint8Array,
+  cookie: string | undefined
+) => {
+  if (!(updatedDescription instanceof Uint8Array)) {
+    throw new Error(
+      "Invalid updatedDescription: must be an instance of Uint8Array"
+    );
+  }
+
+  const workspaceSlug = params.get("workspaceSlug")?.toString();
+  const projectId = params.get("projectId")?.toString();
+  if (!workspaceSlug || !projectId || !cookie) return;
+  // encode binary description data
+  const base64Data = Buffer.from(updatedDescription).toString("base64");
+  const yDoc = new Y.Doc();
+  Y.applyUpdate(yDoc, updatedDescription);
+  // convert to JSON
+  const type = yDoc.getXmlFragment("default");
+  const contentJSON = yXmlFragmentToProseMirrorRootNode(
+    type,
+    documentEditorSchema
+  ).toJSON();
+  // convert to HTML
+  const contentHTML = generateHTML(contentJSON, DOCUMENT_EDITOR_EXTENSIONS);
+
+  try {
+    const payload = {
+      description_binary: base64Data,
+      description_html: contentHTML,
+      description: contentJSON,
+    };
+
+    await pageService.updateDescription(
+      workspaceSlug,
+      projectId,
+      pageId,
+      payload,
+      cookie
+    );
+  } catch (error) {
+    console.error("Update error:", error);
+    throw error;
+  }
+};
+
+const fetchDescriptionHTMLAndTransform = async (
+  workspaceSlug: string,
+  projectId: string,
+  pageId: string,
+  cookie: string
+) => {
+  if (!workspaceSlug || !projectId || !cookie) return;
+
+  try {
+    const pageDetails = await pageService.fetchDetails(
+      workspaceSlug,
+      projectId,
+      pageId,
+      cookie
+    );
+    // convert already existing html to json
+    const contentJSON = generateJSON(
+      pageDetails.description_html ?? "<p></p>",
+      DOCUMENT_EDITOR_EXTENSIONS
+    );
+    // get editor schema from the DOCUMENT_EDITOR_EXTENSIONS array
+    const schema = getSchema(DOCUMENT_EDITOR_EXTENSIONS);
+    // convert json to Y.Doc format
+    const transformedData = prosemirrorJSONToYDoc(
+      schema,
+      contentJSON,
+      "default"
+    );
+    // convert Y.Doc to Uint8Array format
+    const encodedData = Y.encodeStateAsUpdate(transformedData);
+
+    return encodedData;
+  } catch (error) {
+    console.error("Error while transforming from HTML to Uint8Array", error);
+    throw error;
+  }
+};
+
+export const fetchPageDescriptionBinary = async (
+  params: URLSearchParams,
+  pageId: string,
+  cookie: string | undefined
+) => {
+  const workspaceSlug = params.get("workspaceSlug")?.toString();
+  const projectId = params.get("projectId")?.toString();
+  if (!workspaceSlug || !projectId || !cookie) return null;
+
+  try {
+    const response = await pageService.fetchDescriptionBinary(
+      workspaceSlug,
+      projectId,
+      pageId,
+      cookie
+    );
+    const binaryData = new Uint8Array(response);
+
+    if (binaryData.byteLength === 0) {
+      const binary = await fetchDescriptionHTMLAndTransform(
+        workspaceSlug,
+        projectId,
+        pageId,
+        cookie
+      );
+      if (binary) {
+        return binary;
+      }
+    }
+
+    return binaryData;
+  } catch (error) {
+    console.error("Fetch error:", error);
+    throw error;
+  }
+};

live/src/core/services/api.service.ts

@@ -0,0 +1,46 @@
+import axios, { AxiosInstance } from "axios";
+import { config } from "dotenv";
+
+config();
+
+export const API_BASE_URL = process.env.API_BASE_URL ?? "";
+
+export abstract class APIService {
+  protected baseURL: string;
+  private axiosInstance: AxiosInstance;
+
+  constructor(baseURL: string) {
+    this.baseURL = baseURL;
+    this.axiosInstance = axios.create({
+      baseURL,
+      withCredentials: true,
+    });
+  }
+
+  get(url: string, params = {}, config = {}) {
+    return this.axiosInstance.get(url, {
+      ...params,
+      ...config,
+    });
+  }
+
+  post(url: string, data = {}, config = {}) {
+    return this.axiosInstance.post(url, data, config);
+  }
+
+  put(url: string, data = {}, config = {}) {
+    return this.axiosInstance.put(url, data, config);
+  }
+
+  patch(url: string, data = {}, config = {}) {
+    return this.axiosInstance.patch(url, data, config);
+  }
+
+  delete(url: string, data?: any, config = {}) {
+    return this.axiosInstance.delete(url, { data, ...config });
+  }
+
+  request(config = {}) {
+    return this.axiosInstance(config);
+  }
+}

live/src/core/services/page.service.ts

@@ -0,0 +1,78 @@
+// types
+import { TPage } from "@plane/types";
+// services
+import { API_BASE_URL, APIService } from "./api.service.js";
+
+export class PageService extends APIService {
+  constructor() {
+    super(API_BASE_URL);
+  }
+
+  async fetchDetails(
+    workspaceSlug: string,
+    projectId: string,
+    pageId: string,
+    cookie: string
+  ): Promise<TPage> {
+    return this.get(
+      `/api/workspaces/${workspaceSlug}/projects/${projectId}/pages/${pageId}/`,
+      {
+        headers: {
+          Cookie: cookie,
+        },
+      }
+    )
+      .then((response) => response?.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async fetchDescriptionBinary(
+    workspaceSlug: string,
+    projectId: string,
+    pageId: string,
+    cookie: string
+  ): Promise<any> {
+    return this.get(
+      `/api/workspaces/${workspaceSlug}/projects/${projectId}/pages/${pageId}/description/`,
+      {
+        headers: {
+          "Content-Type": "application/octet-stream",
+          Cookie: cookie,
+        },
+        responseType: "arraybuffer",
+      }
+    )
+      .then((response) => response?.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async updateDescription(
+    workspaceSlug: string,
+    projectId: string,
+    pageId: string,
+    data: {
+      description_binary: string;
+      description_html: string;
+      description: object;
+    },
+    cookie: string
+  ): Promise<any> {
+    return this.patch(
+      `/api/workspaces/${workspaceSlug}/projects/${projectId}/pages/${pageId}/description/`,
+      data,
+      {
+        headers: {
+          Cookie: cookie,
+        },
+      }
+    )
+      .then((response) => response?.data)
+      .catch((error) => {
+        throw error;
+      });
+  }
+}

live/src/core/services/user.service.ts

@@ -0,0 +1,46 @@
+// types
+import type { IUser, IUserProjectsRole } from "@plane/types";
+// services
+import { API_BASE_URL, APIService } from "./api.service.js";
+
+export class UserService extends APIService {
+  constructor() {
+    super(API_BASE_URL);
+  }
+
+  currentUserConfig() {
+    return {
+      url: `${this.baseURL}/api/users/me/`,
+    };
+  }
+
+  async currentUser(cookie: string): Promise<IUser> {
+    return this.get("/api/users/me/", {
+      headers: {
+        Cookie: cookie,
+      },
+    })
+      .then((response) => response?.data)
+      .catch((error) => {
+        throw error;
+      });
+  }
+
+  async getUserAllProjectsRole(
+    workspaceSlug: string,
+    cookie: string
+  ): Promise<IUserProjectsRole> {
+    return this.get(
+      `/api/users/me/workspaces/${workspaceSlug}/project-roles/`,
+      {
+        headers: {
+          Cookie: cookie,
+        },
+      }
+    )
+      .then((response) => response?.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+}

live/src/core/types/common.d.ts

@@ -0,0 +1,3 @@
+import { TAdditionalDocumentTypes } from "@/plane-live/types/common.js";
+
+export type TDocumentTypes = "project_page" & TAdditionalDocumentTypes;