fix: add the redis extension conditionally (#5524)

* fix: add the redis extension conditionally

* chore: import order and stuff

* fix: added logger, error handling and routing

* feat: configured sentry with source maps

* fix: sentry config and returning json

* fix: remove on change logs

* fix: add pretty print

live/.env.example

@@ -1,4 +1,5 @@
 API_BASE_URL="http://api:8000"
+LIVE_BASE_PATH="/live"
 REDIS_URL="redis://localhost:6379"
 
 # If you prefer not to provide a Redis URL, you can set the REDIS_HOST and REDIS_PORT environment variables instead.

live/Dockerfile.live

@@ -33,12 +33,7 @@ RUN yarn turbo build --filter=live
 FROM base AS runner
 WORKDIR /app
 
-# Don't run production as root
-RUN addgroup --system --gid 1001 expressjs
-RUN adduser --system --uid 1001 expressjs
-USER expressjs
-
 COPY --from=installer /app .
 # COPY --from=installer /app/live/node_modules ./node_modules
 
-EXPOSE 3000
+EXPOSE 3000

live/package.json

@@ -7,8 +7,7 @@
   "type": "module",
   "scripts": {
     "build": "babel src --out-dir dist --extensions \".ts,.js\"",
-    "start": "node dist/server.js",
-    "dev": "PORT=3003 nodemon --exec \"node -r esbuild-register ./src/server.ts\" -e .ts"
+    "start": "node dist/server.js"
   },
   "keywords": [],
   "author": "",
@@ -20,14 +19,20 @@
     "@hocuspocus/server": "^2.11.3",
     "@plane/editor": "*",
     "@plane/types": "*",
+    "@sentry/node": "^8.28.0",
+    "@sentry/profiling-node": "^8.28.0",
     "@tiptap/core": "^2.4.0",
     "@tiptap/html": "^2.3.0",
     "axios": "^1.7.2",
+    "cors": "^2.8.5",
     "dotenv": "^16.4.5",
     "express": "^4.19.2",
     "express-ws": "^5.0.2",
+    "ioredis": "^5.4.1",
     "lodash": "^4.17.21",
-    "redis": "^4.7.0",
+    "morgan": "^1.10.0",
+    "pino-http": "^10.3.0",
+    "pino-pretty": "^11.2.2",
     "y-prosemirror": "^1.2.9",
     "y-protocols": "^1.0.6",
     "yjs": "^13.6.14"
@@ -37,6 +42,7 @@
     "@babel/core": "^7.25.2",
     "@babel/preset-env": "^7.25.4",
     "@babel/preset-typescript": "^7.24.7",
+    "@types/cors": "^2.8.17",
     "@types/dotenv": "^8.2.0",
     "@types/express": "^4.17.21",
     "@types/express-ws": "^3.0.4",

live/src/core/config/sentry-config.ts

@@ -0,0 +1,19 @@
+import * as Sentry from "@sentry/node";
+import { nodeProfilingIntegration } from "@sentry/profiling-node";
+
+// Ensure to call this before importing any other modules!
+Sentry.init({
+  dsn: process.env.LIVE_SENTRY_DSN,
+  environment: process.env.LIVE_SENTRY_ENVIRONMENT || "development",
+
+  integrations: [
+    // Add our Profiling integration
+    nodeProfilingIntegration(),
+  ],
+  // Add Tracing by setting tracesSampleRate
+  // We recommend adjusting this value in production
+  tracesSampleRate: Number(process.env.LIVE_SENTRY_TRACES_SAMPLE_RATE) || 0.5,
+  // Set sampling rate for profiling
+  // This is relative to tracesSampleRate
+  profilesSampleRate: 1.0,
+});

live/src/core/extensions/index.ts

@@ -0,0 +1,137 @@
+// Third-party libraries
+import { Redis } from "ioredis";
+
+// Hocuspocus extensions and core
+import { Database } from "@hocuspocus/extension-database";
+import { Extension } from "@hocuspocus/server";
+import { Logger } from "@hocuspocus/extension-logger";
+import { Redis as HocusPocusRedis } from "@hocuspocus/extension-redis";
+
+// Core helpers and utilities
+import { manualLogger } from "@/core/helpers/logger.js";
+import { getRedisUrl } from "@/core/lib/utils/redis-url.js";
+
+// Core libraries
+import {
+  fetchPageDescriptionBinary,
+  updatePageDescription,
+} from "@/core/lib/page.js";
+
+// Core types
+import { TDocumentTypes } from "@/core/types/common.js";
+
+// Plane live libraries
+import { fetchDocument } from "@/plane-live/lib/fetch-document.js";
+import { updateDocument } from "@/plane-live/lib/update-document.js";
+
+export const getExtensions: () => Extension[] = () => {
+  const extensions: Extension[] = [
+    new Logger({
+      onChange: false,
+      log: (message) => {
+        manualLogger.info(message);
+      },
+    }),
+    new Database({
+      fetch: async ({
+        documentName: pageId,
+        requestHeaders,
+        requestParameters,
+      }) => {
+        // request headers
+        const cookie = requestHeaders.cookie?.toString();
+        // query params
+        const params = requestParameters;
+        const documentType = params.get("documentType")?.toString() as
+          | TDocumentTypes
+          | undefined;
+
+        return new Promise(async (resolve) => {
+          try {
+            let fetchedData = null;
+            if (documentType === "project_page") {
+              fetchedData = await fetchPageDescriptionBinary(
+                params,
+                pageId,
+                cookie,
+              );
+            } else {
+              fetchedData = await fetchDocument({
+                cookie,
+                documentType,
+                pageId,
+                params,
+              });
+            }
+            resolve(fetchedData);
+          } catch (error) {
+            console.error("Error in fetching document", error);
+          }
+        });
+      },
+      store: async ({
+        state,
+        documentName: pageId,
+        requestHeaders,
+        requestParameters,
+      }) => {
+        // request headers
+        const cookie = requestHeaders.cookie?.toString();
+        // query params
+        const params = requestParameters;
+        const documentType = params.get("documentType")?.toString() as
+          | TDocumentTypes
+          | undefined;
+
+        return new Promise(async () => {
+          try {
+            if (documentType === "project_page") {
+              await updatePageDescription(params, pageId, state, cookie);
+            } else {
+              await updateDocument({
+                cookie,
+                documentType,
+                pageId,
+                params,
+                updatedDescription: state,
+              });
+            }
+          } catch (error) {
+            console.error("Error in updating document", error);
+          }
+        });
+      },
+    }),
+  ];
+
+  const redisUrl = getRedisUrl();
+
+  // Add the Redis extension only if configured
+  if (redisUrl) {
+    try {
+      const redisClient = new Redis(redisUrl);
+      redisClient.on("error", (error: any) => {
+        // if auth fails or the server is down, disconnect redis
+        if (
+          error?.code === "ENOTFOUND" ||
+          error.message.includes("WRONGPASS") ||
+          error.message.includes("NOAUTH")
+        ) {
+          redisClient.disconnect();
+        }
+        manualLogger.error(
+          `Redis Client wasn't able to connect, continuing without Redis (you won't be able to sync data betwen multiple plane live servers)`,
+        );
+        manualLogger.error(error);
+      });
+      redisClient.on("ready", () => {
+        manualLogger.info("Redis Client connected");
+        extensions.push(new HocusPocusRedis({ redis: redisClient }));
+      });
+    } catch (error) {
+      manualLogger.error("Failed to connect to Redis:", error);
+    }
+  }
+
+  return extensions;
+};

live/src/core/helpers/error-handler.ts

@@ -0,0 +1,21 @@
+import { ErrorRequestHandler } from "express";
+import { manualLogger } from "@/core/helpers/logger.js";
+
+export const errorHandler: ErrorRequestHandler = (err, _req, res, _next) => {
+  // Log the error
+  manualLogger.error(err);
+
+  // Set the response status
+  res.status(err.status || 500);
+
+  // Send the response
+  res.json({
+    error: {
+      message:
+        process.env.NODE_ENV === "production"
+          ? "An unexpected error occurred"
+          : err.message,
+      ...(process.env.NODE_ENV !== "production" && { stack: err.stack }),
+    },
+  });
+};

live/src/core/helpers/logger.ts

@@ -0,0 +1,15 @@
+import { pinoHttp } from "pino-http";
+
+const transport = {
+  target: "pino-pretty",
+  options: {
+    colorize: true,
+  },
+};
+
+export const logger = pinoHttp({
+  level: "info",
+  transport: transport,
+});
+
+export const manualLogger = logger.logger;

live/src/core/hocuspocus-server.ts

@@ -0,0 +1,35 @@
+import { Server } from "@hocuspocus/server";
+
+import { handleAuthentication } from "@/core/lib/authentication.js";
+import { getExtensions } from "@/core/extensions/index.js";
+
+export const HocusPocusServer = Server.configure({
+  onAuthenticate: async ({
+    requestHeaders,
+    requestParameters,
+    connection,
+    // user id used as token for authentication
+    token,
+  }) => {
+    // request headers
+    const cookie = requestHeaders.cookie?.toString();
+    // params
+    const params = requestParameters;
+
+    if (!cookie) {
+      throw Error("Credentials not provided");
+    }
+
+    try {
+      await handleAuthentication({
+        connection,
+        cookie,
+        params,
+        token,
+      });
+    } catch (error) {
+      throw Error("Authentication unsuccessful!");
+    }
+  },
+  extensions: getExtensions(),
+});

live/src/server.ts

@@ -1,146 +1,54 @@
-import { Server } from "@hocuspocus/server";
-import { Redis } from "@hocuspocus/extension-redis";
-import { createClient } from "redis";
+import "@/core/config/sentry-config.js";
 
-import { Database } from "@hocuspocus/extension-database";
-import { Logger } from "@hocuspocus/extension-logger";
 import express from "express";
-import expressWs, { Application } from "express-ws";
-// lib
-import { handleAuthentication } from "@/core/lib/authentication.js";
-import {
-  fetchPageDescriptionBinary,
-  updatePageDescription,
-} from "@/core/lib/page.js";
-// config
-import { getRedisUrl } from "@/core/config/redis-config.js";
-// types
-import { TDocumentTypes } from "@/core/types/common.js";
-// plane live lib
-import { fetchDocument } from "@/plane-live/lib/fetch-document.js";
-import { updateDocument } from "@/plane-live/lib/update-document.js";
+import expressWs from "express-ws";
+import * as Sentry from "@sentry/node";
 
-const redisUrl = getRedisUrl();
-const redisClient = await createClient({ url: redisUrl })
-  .on("error", (err) => console.log("Redis Client Error", err))
-  .connect();
+// cors
+import cors from "cors";
 
-const server = Server.configure({
-  onAuthenticate: async ({
-    requestHeaders,
-    requestParameters,
-    connection,
-    // user id used as token for authentication
-    token,
-  }) => {
-    // request headers
-    const cookie = requestHeaders.cookie?.toString();
-    // params
-    const params = requestParameters;
+// core hocuspocus server
+import { HocusPocusServer } from "@/core/hocuspocus-server.js";
 
-    if (!cookie) {
-      throw Error("Credentials not provided");
-    }
+// helpers
+import { logger, manualLogger } from "@/core/helpers/logger.js";
+import { errorHandler } from "@/core/helpers/error-handler.js";
 
-    try {
-      await handleAuthentication({
-        connection,
-        cookie,
-        params,
-        token,
-      });
-    } catch (error) {
-      throw Error("Authentication unsuccessful!");
-    }
-  },
-  extensions: [
-    new Redis({ redis: redisClient }),
-    new Logger(),
-    new Database({
-      fetch: async ({
-        documentName: pageId,
-        requestHeaders,
-        requestParameters,
-      }) => {
-        // request headers
-        const cookie = requestHeaders.cookie?.toString();
-        // query params
-        const params = requestParameters;
-        const documentType = params.get("documentType")?.toString() as
-          | TDocumentTypes
-          | undefined;
-
-        return new Promise(async (resolve) => {
-          try {
-            let fetchedData = null;
-            if (documentType === "project_page") {
-              fetchedData = await fetchPageDescriptionBinary(
-                params,
-                pageId,
-                cookie,
-              );
-            } else {
-              fetchedData = await fetchDocument({
-                cookie,
-                documentType,
-                pageId,
-                params,
-              });
-            }
-            resolve(fetchedData);
-          } catch (error) {
-            console.error("Error in fetching document", error);
-          }
-        });
-      },
-      store: async ({
-        state,
-        documentName: pageId,
-        requestHeaders,
-        requestParameters,
-      }) => {
-        // request headers
-        const cookie = requestHeaders.cookie?.toString();
-        // query params
-        const params = requestParameters;
-        const documentType = params.get("documentType")?.toString() as
-          | TDocumentTypes
-          | undefined;
-
-        return new Promise(async () => {
-          try {
-            if (documentType === "project_page") {
-              await updatePageDescription(params, pageId, state, cookie);
-            } else {
-              await updateDocument({
-                cookie,
-                documentType,
-                pageId,
-                params,
-                updatedDescription: state,
-              });
-            }
-          } catch (error) {
-            console.error("Error in updating document", error);
-          }
-        });
-      },
-    }),
-  ],
-});
-
-const { app }: { app: Application } = expressWs(express());
+const app = express();
+expressWs(app);
 
 app.set("port", process.env.PORT || 3000);
 
-app.get("/health", (_request, response) => {
-  response.status(200);
+// Logging middleware
+app.use(logger);
+
+// Body parsing middleware
+app.use(express.json());
+app.use(express.urlencoded({ extended: true }));
+
+// cors middleware
+app.use(cors());
+
+const router = express.Router();
+
+router.get("/health", (_req, res) => {
+  res.status(200).json({ status: "OK" });
 });
 
-app.ws("/collaboration", (websocket, request) => {
-  server.handleConnection(websocket, request);
+router.ws("/collaboration", (ws, req) => {
+  HocusPocusServer.handleConnection(ws, req);
 });
 
+app.use(process.env.LIVE_BASE_PATH || "/live", router);
+
+app.use((_req, res, _next) => {
+  res.status(404).send("Not Found");
+});
+
+Sentry.setupExpressErrorHandler(app);
+
+app.use(errorHandler);
+
 app.listen(app.get("port"), () => {
-  console.log("Live server has started at port", app.get("port"));
+  manualLogger.info(`Plane Live server has started at port ${app.get("port")}`);
 });

live/tsconfig.json

@@ -3,18 +3,41 @@
   "compilerOptions": {
     "module": "NodeNext",
     "moduleResolution": "NodeNext",
-    "lib": ["ES2015"],
+
+    "lib": [
+      "ES2015"
+    ],
+
     "outDir": "./dist",
     "rootDir": "./src",
     "baseUrl": ".",
+
     "paths": {
-      "@/core/*": ["./src/core/*"],
-      "@/plane-live/*": ["./src/ce/*"]
+      "@/*": [
+        "./src/*"
+      ],
+      "@/plane-live/*": [
+        "./src/ce/*"
+      ]
     },
+
     "removeComments": true,
     "esModuleInterop": true,
-    "skipLibCheck": true
+    "skipLibCheck": true,
+    "sourceMap": true,
+    "inlineSources": true,
+
+    // Set `sourceRoot` to  "/" to strip the build path prefix
+    // from generated source code references.
+    // This improves issue grouping in Sentry.
+    "sourceRoot": "/"
   },
-  "include": ["src/**/*.ts"],
-  "exclude": ["./dist", "./build", "./node_modules"]
+  "include": [
+    "src/**/*.ts"
+  ],
+  "exclude": [
+    "./dist",
+    "./build",
+    "./node_modules"
+  ]
 }