- Change batch size to 50 for inserting issues (#6085)

- Fallback to server when mentions filter is used
- Split load workspace into multiple transactions

web/core/local-db/storage.sqlite.ts

@@ -21,7 +21,7 @@ import { clearOPFS, getGroupedIssueResults, getSubGroupedIssueResults, log, logE
 
 const DB_VERSION = 1;
 const PAGE_SIZE = 500;
-const BATCH_SIZE = 500;
+const BATCH_SIZE = 50;
 
 type TProjectStatus = {
   issues: { status: undefined | "loading" | "ready" | "error" | "syncing"; sync: Promise<void> | undefined };
@@ -382,10 +382,10 @@ export class Storage {
 
   getIssue = async (issueId: string) => {
     try {
-      if (!rootStore.user.localDBEnabled) return;
+      if (!rootStore.user.localDBEnabled || this.status !== "ready") return;
 
       const issues = await runQuery(`select * from issues where id='${issueId}'`);
-      if (issues.length) {
+      if (Array.isArray(issues) && issues.length) {
         return formatLocalIssue(issues[0]);
       }
     } catch (err) {

web/core/local-db/utils/load-issues.ts

@@ -15,7 +15,7 @@ export const addIssue = async (issue: any) => {
   await persistence.db.exec("COMMIT;");
 };
 
-export const addIssuesBulk = async (issues: any, batchSize = 100) => {
+export const addIssuesBulk = async (issues: any, batchSize = 50) => {
   if (!rootStore.user.localDBEnabled || !persistence.db) return;
   if (!issues.length) return;
   const insertStart = performance.now();
@@ -24,18 +24,20 @@ export const addIssuesBulk = async (issues: any, batchSize = 100) => {
   for (let i = 0; i < issues.length; i += batchSize) {
     const batch = issues.slice(i, i + batchSize);
 
+    const promises = [];
     for (let j = 0; j < batch.length; j++) {
       const issue = batch[j];
       if (!issue.type_id) {
         issue.type_id = "";
       }
-      await stageIssueInserts(issue);
+      promises.push(stageIssueInserts(issue));
     }
+    await Promise.all(promises);
   }
   await persistence.db.exec("COMMIT;");
 
   const insertEnd = performance.now();
-  log("Inserted issues in ", `${insertEnd - insertStart}ms`);
+  log("Inserted issues in ", `${insertEnd - insertStart}ms`, batchSize, issues.length);
 };
 export const deleteIssueFromLocal = async (issue_id: any) => {
   if (!rootStore.user.localDBEnabled || !persistence.db) return;

web/core/local-db/utils/load-workspace.ts

@@ -120,12 +120,28 @@ export const loadWorkSpaceData = async (workspaceSlug: string) => {
   const start = performance.now();
   await persistence.db.exec("BEGIN;");
   await batchInserts(labels, "labels", labelSchema);
+  await persistence.db.exec("COMMIT;");
+
+  await persistence.db.exec("BEGIN;");
   await batchInserts(modules, "modules", moduleSchema);
+  await persistence.db.exec("COMMIT;");
+
+  await persistence.db.exec("BEGIN;");
   await batchInserts(cycles, "cycles", cycleSchema);
+  await persistence.db.exec("COMMIT;");
+
+  await persistence.db.exec("BEGIN;");
   await batchInserts(states, "states", stateSchema);
+  await persistence.db.exec("COMMIT;");
+
+  await persistence.db.exec("BEGIN;");
   await batchInserts(estimates, "estimate_points", estimatePointSchema);
+  await persistence.db.exec("COMMIT;");
+
+  await persistence.db.exec("BEGIN;");
   await batchInserts(members, "members", memberSchema);
   await persistence.db.exec("COMMIT;");
+
   const end = performance.now();
   log("Time taken to load workspace data", end - start);
 };

web/core/services/issue/issue.service.ts

@@ -1,5 +1,3 @@
-import { startSpan } from "@sentry/nextjs";
-import isEmpty from "lodash/isEmpty";
 // types
 import type {
   IIssueDisplayProperties,
@@ -12,6 +10,7 @@ import type {
 } from "@plane/types";
 // helpers
 import { API_BASE_URL } from "@/helpers/common.helper";
+import { getIssuesShouldFallbackToServer } from "@/helpers/issue.helper";
 import { persistence } from "@/local-db/storage.sqlite";
 // services
 
@@ -68,7 +67,7 @@ export class IssueService extends APIService {
   }
 
   async getIssues(workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise<TIssuesResponse> {
-    if (!isEmpty(queries.expand as string) && !queries.group_by)
+    if (getIssuesShouldFallbackToServer(queries))
       return await this.getIssuesFromServer(workspaceSlug, projectId, queries, config);
 
     const response = await persistence.getIssues(workspaceSlug, projectId, queries, config);

web/helpers/issue.helper.ts

@@ -1,4 +1,5 @@
 import differenceInCalendarDays from "date-fns/differenceInCalendarDays";
+import isEmpty from "lodash/isEmpty";
 import set from "lodash/set";
 import { v4 as uuidv4 } from "uuid";
 // types
@@ -308,3 +309,17 @@ export const getComputedDisplayProperties = (
   cycle: displayProperties?.cycle ?? true,
   issue_type: displayProperties?.issue_type ?? true,
 });
+
+/**
+ * This is to check if the issues list api should fall back to server or use local db
+ * @param queries
+ * @returns
+ */
+export const getIssuesShouldFallbackToServer = (queries: any) => {
+  // If there is expand query and is not grouped then fallback to server
+  if (!isEmpty(queries.expand as string) && !queries.group_by) return true;
+  // If query has mentions then fallback to server
+  if (!isEmpty(queries.mentions)) return true;
+
+  return false;
+};