[WEB-2001]feat: Cache issues on the client (#5327)

* use common getIssues from issue service instead of multiple different services for modules and cycles * Use SQLite to store issues locally and load issues from it. * Fix incorrect total count and filtering on assignees. * enable parallel API calls * use common getIssues from issue service instead of multiple different services for modules and cycles * Use SQLite to store issues locally and load issues from it. * Fix incorrect total count and filtering on assignees. * enable parallel API calls * chore: deleted issue list * - Handle local mutations - Implement getting the updates - Use SWR to update/sync data * Wait for sync to complete in get issues * Fix build errors * Fix build issue * - Sync updates to local-db - Fallback to server when the local data is loading - Wait when the updates are being fetched * Add issues in batches * Disable skeleton loaders for first 10 issues * Load issues in bulk * working version of sql lite with grouped issues * Use window queries for group by * - Fix sort by date fields - Fix the total count * - Fix grouping by created by - Fix order by and limit * fix pagination * Fix sorting on issue priority * - Add secondary sort order - Fix group by priority * chore: added timestamp filter for deleted issues * - Extract local DB into its own class - Implement sorting by label names * Implement subgroup by * sub group by changes * Refactor query constructor * Insert or update issues instead of directly adding them. * Segregated queries. Not working though!! * - Get filtered issues and then group them. - Cleanup code. - Implement order by labels. * Fix build issues * Remove debuggers * remove loaders while changing sorting or applying filters * fix loader while clearing all filters * Fix issue with project being synced twice * Improve project sync * Optimize the queries * Make create dummy data more realistic * dev: added total pages in the global paginator * chore: updated total_paged count * chore: added state_group in the issues pagination * chore: removed deleted_at from the issue pagination payload * chore: replaced state_group with state__group * Integrate new getIssues API, and fix sync issues bug. * Fix issue with SWR running twice in workspace wrapper * Fix DB initialization called when opening project for the first time. * Add all the tables required for sorting * Exclude description from getIssues * Add getIssue function. * Add only selected fields to get query. * Fix the count query * Minor query optimization when no joins are required. * fetch issue description from local db * clear local db on signout * Correct dummy data creation * Fix sort by assignee * sync to local changes * chore: added archived issues in the deleted endpoint * Sync deletes to local db. * - Add missing indexes for tables used in sorting in spreadsheet layout. - Add options table * Make fallback optional in getOption * Kanban column virtualization * persist project sync readiness to sqlite and use that as the source of truth for the project issues to be ready * fix build errors * Fix calendar view * fetch slimed down version of modules in project wrapper * fetch toned down modules and then fetch complete modules * Fix multi value order by in spread sheet layout * Fix sort by * Fix the query when ordering by multi field names * Remove unused import * Fix sort by multi value fields * Format queries and fix order by * fix order by for multi issue * fix loaders for spreadsheet * Fallback to manual order whn moving away from spreadsheet layout * fix minor bug * Move fix for order_by when switching from spreadsheet layout to translateQueryParams * fix default rendering of kanban groups * Fix none priority being saved as null * Remove debugger statement * Fix issue load * chore: updated isue paginated query from to * Fix sub issues and start and target date filters * Fix active and backlog filter * Add default order by * Update the Query param to match with backend. * local sqlite db versioning * When window is hidden, do not perform any db versioning * fix error handling and fall back to server when database errors out * Add ability to disable local db cache * remove db version check from getIssues function * change db version to number and remove workspaceInitPromise in storage.sqlite * - Sync the entire workspace in the background - Add get sub issue method with distribution * Make changes to get issues for sync to match backend. * chore: handled workspace and project in v2 paginted issues * disable issue description and title until fetched from server * sync issues post bulk operations * fix server error * fix front end build * Remove full workspace sync * - Remove the toast message on sync. - Update the disable local message. * Add Hardcoded constant to disable the local db caching * fix lint errors * Fix order by in grouping * update yarn lock * fix build * fix plane-web imports * address review comments --------- Co-authored-by: rahulramesha <rahulramesham@gmail.com> Co-authored-by: NarayanBavisetti <narayan3119@gmail.com> Co-authored-by: gurusainath <gurusainath007@gmail.com>
2024-09-24 19:01:34 +05:30 · 2024-09-24 19:01:34 +05:30 · 3df230393a
commit 3df230393a
parent 8dabe839f3
48 changed files with 2085 additions and 155 deletions
--- a/web/core/local-db/utils/constants.ts
+++ b/web/core/local-db/utils/constants.ts
@ -0,0 +1,21 @@
+export const ARRAY_FIELDS = ["label_ids", "assignee_ids", "module_ids"];
+
+export const GROUP_BY_MAP = {
+  state_id: "state_id",
+  priority: "priority",
+  cycle_id: "cycle_id",
+  created_by: "created_by",
+  // Array Props
+  issue_module__module_id: "module_ids",
+  labels__id: "label_ids",
+  assignees__id: "assignee_ids",
+  target_date: "target_date",
+};
+
+export const PRIORITY_MAP = {
+  low: 1,
+  medium: 2,
+  high: 3,
+  urgent: 4,
+  none: 0,
+};
--- a/web/core/local-db/utils/data.utils.ts
+++ b/web/core/local-db/utils/data.utils.ts
@ -0,0 +1,30 @@
+import { runQuery } from "./query-executor";
+
+export const getProjectIds = async () => {
+  const q = `select project_id from states where project_id is not null group by project_id`;
+  return await runQuery(q);
+};
+
+export const getSubIssues = async (issueId: string) => {
+  const q = `select * from issues where parent_id = '${issueId}'`;
+  return await runQuery(q);
+};
+
+export const getSubIssueDistribution = async (issueId: string) => {
+  const q = `select s.'group', group_concat(i.id) as issues from issues i left join states s on s.id = i.state_id  where i.parent_id = '${issueId}' group by s.'group'`;
+
+  const result = await runQuery(q);
+  if (!result.length) {
+    return {};
+  }
+  return result.reduce((acc: Record<string, string[]>, item: { group: string; issues: string }) => {
+    acc[item.group] = item.issues.split(",");
+    return acc;
+  }, {});
+};
+
+export const getSubIssuesWithDistribution = async (issueId: string) => {
+  const promises = [getSubIssues(issueId), getSubIssueDistribution(issueId)];
+  const [sub_issues, state_distribution] = await Promise.all(promises);
+  return { sub_issues, state_distribution };
+};
--- a/web/core/local-db/utils/indexes.ts
+++ b/web/core/local-db/utils/indexes.ts
@ -0,0 +1,68 @@
+import { persistence } from "../storage.sqlite";
+
+const log = console.log;
+export const createIssueIndexes = async () => {
+  const columns = [
+    "state_id",
+    "sort_order",
+    //  "priority",
+    "priority_proxy",
+    "project_id",
+    "created_by",
+    "cycle_id",
+  ];
+
+  const promises: Promise<any>[] = [];
+
+  promises.push(persistence.db.exec({ sql: `CREATE UNIQUE INDEX issues_issue_id_idx ON issues (id)` }));
+
+  columns.forEach((column) => {
+    promises.push(
+      persistence.db.exec({ sql: `CREATE  INDEX issues_issue_${column}_idx ON issues (project_id, ${column})` })
+    );
+  });
+  await Promise.all(promises);
+};
+
+export const createIssueMetaIndexes = async () => {
+  // Drop indexes
+  await persistence.db.exec({ sql: `CREATE INDEX issue_meta_all_idx ON issue_meta (issue_id,key,value)` });
+};
+
+export const createWorkSpaceIndexes = async () => {
+  const promises: Promise<any>[] = [];
+  // Labels
+  promises.push(persistence.db.exec({ sql: `CREATE INDEX labels_name_idx ON labels (id,name,project_id)` }));
+
+  // Modules
+  promises.push(persistence.db.exec({ sql: `CREATE INDEX modules_name_idx ON modules  (id,name,project_id)` }));
+
+  // States
+  promises.push(persistence.db.exec({ sql: `CREATE INDEX states_name_idx ON states  (id,name,project_id)` }));
+  // Cycles
+  promises.push(persistence.db.exec({ sql: `CREATE INDEX cycles_name_idx ON cycles  (id,name,project_id)` }));
+
+  // Members
+  promises.push(persistence.db.exec({ sql: `CREATE INDEX members_name_idx ON members  (id,first_name)` }));
+
+  // Estimate Points @todo
+  promises.push(persistence.db.exec({ sql: `CREATE INDEX estimate_points_name_idx ON estimate_points  (id,value)` }));
+  // Options
+  promises.push(persistence.db.exec({ sql: `CREATE INDEX options_name_idx ON options  (name)` }));
+
+  await Promise.all(promises);
+};
+
+const createIndexes = async () => {
+  log("### Creating indexes");
+  const start = performance.now();
+  const promises = [createIssueIndexes(), createIssueMetaIndexes(), createWorkSpaceIndexes()];
+  try {
+    await Promise.all(promises);
+  } catch (e) {
+    console.log((e as Error).message);
+  }
+  log("### Indexes created in", `${performance.now() - start}ms`);
+};
+
+export default createIndexes;
--- a/web/core/local-db/utils/load-issues.ts
+++ b/web/core/local-db/utils/load-issues.ts
@ -0,0 +1,118 @@
+import { TIssue } from "@plane/types";
+import { rootStore } from "@/lib/store-context";
+import { IssueService } from "@/services/issue";
+import { persistence } from "../storage.sqlite";
+import { ARRAY_FIELDS, PRIORITY_MAP } from "./constants";
+import { issueSchema } from "./schemas";
+
+export const PROJECT_OFFLINE_STATUS: Record<string, boolean> = {};
+
+export const addIssue = async (issue: any) => {
+  if (document.hidden || !rootStore.user.localDBEnabled) return;
+
+  persistence.db.exec("BEGIN TRANSACTION;");
+  stageIssueInserts(issue);
+  persistence.db.exec("COMMIT;");
+};
+
+export const addIssuesBulk = async (issues: any, batchSize = 100) => {
+  if (!rootStore.user.localDBEnabled) return;
+
+  for (let i = 0; i < issues.length; i += batchSize) {
+    const batch = issues.slice(i, i + batchSize);
+
+    persistence.db.exec("BEGIN TRANSACTION;");
+    batch.forEach((issue: any) => {
+      if (!issue.type_id) {
+        issue.type_id = "";
+      }
+      stageIssueInserts(issue);
+    });
+    await persistence.db.exec("COMMIT;");
+  }
+};
+export const deleteIssueFromLocal = async (issue_id: any) => {
+  if (!rootStore.user.localDBEnabled) return;
+
+  const deleteQuery = `delete from issues where id='${issue_id}'`;
+  const deleteMetaQuery = `delete from issue_meta where issue_id='${issue_id}'`;
+
+  persistence.db.exec("BEGIN TRANSACTION;");
+  persistence.db.exec(deleteQuery);
+  persistence.db.exec(deleteMetaQuery);
+  persistence.db.exec("COMMIT;");
+};
+
+export const updateIssue = async (issue: TIssue) => {
+  if (document.hidden || !rootStore.user.localDBEnabled) return;
+
+  const issue_id = issue.id;
+  // delete the issue and its meta data
+  await deleteIssueFromLocal(issue_id);
+  addIssue(issue);
+};
+
+export const syncDeletesToLocal = async (workspaceId: string, projectId: string, queries: any) => {
+  if (!rootStore.user.localDBEnabled) return;
+
+  const issueService = new IssueService();
+  const response = await issueService.getDeletedIssues(workspaceId, projectId, queries);
+  if (Array.isArray(response)) {
+    response.map(async (issue) => deleteIssueFromLocal(issue));
+  }
+};
+
+const stageIssueInserts = (issue: any) => {
+  const issue_id = issue.id;
+  issue.priority_proxy = PRIORITY_MAP[issue.priority as keyof typeof PRIORITY_MAP];
+
+  const keys = Object.keys(issueSchema);
+  const sanitizedIssue = keys.reduce((acc: any, key) => {
+    if (issue[key] || issue[key] === 0) {
+      acc[key] = issue[key];
+    }
+    return acc;
+  }, {});
+
+  const columns = "'" + Object.keys(sanitizedIssue).join("','") + "'";
+
+  const values = Object.values(sanitizedIssue)
+    .map((value) => {
+      if (value === null) {
+        return "";
+      }
+      if (typeof value === "object") {
+        return `'${JSON.stringify(value)}'`;
+      }
+      if (typeof value === "string") {
+        return `'${value}'`;
+      }
+      return value;
+    })
+    .join(", ");
+
+  const query = `INSERT OR REPLACE INTO issues (${columns}) VALUES (${values});`;
+  persistence.db.exec(query);
+
+  persistence.db.exec({
+    sql: `DELETE from issue_meta where issue_id='${issue_id}'`,
+  });
+
+  ARRAY_FIELDS.forEach((field) => {
+    const values = issue[field];
+    if (values && values.length) {
+      values.forEach((val: any) => {
+        persistence.db.exec({
+          sql: `INSERT OR REPLACE  into issue_meta(issue_id,key,value) values (?,?,?) `,
+          bind: [issue_id, field, val],
+        });
+      });
+    } else {
+      // Added for empty fields?
+      persistence.db.exec({
+        sql: `INSERT OR REPLACE  into issue_meta(issue_id,key,value) values (?,?,?) `,
+        bind: [issue_id, field, ""],
+      });
+    }
+  });
+};
--- a/web/core/local-db/utils/load-workspace.ts
+++ b/web/core/local-db/utils/load-workspace.ts
@ -0,0 +1,142 @@
+import { IEstimate, IEstimatePoint, IWorkspaceMember } from "@plane/types";
+import { API_BASE_URL } from "@/helpers/common.helper";
+import { EstimateService } from "@/plane-web/services/project/estimate.service";
+import { CycleService } from "@/services/cycle.service";
+import { IssueLabelService } from "@/services/issue/issue_label.service";
+import { ModuleService } from "@/services/module.service";
+import { ProjectStateService } from "@/services/project";
+import { WorkspaceService } from "@/services/workspace.service";
+import { persistence } from "../storage.sqlite";
+import { cycleSchema, estimatePointSchema, labelSchema, memberSchema, Schema, stateSchema } from "./schemas";
+
+const stageInserts = (table: string, schema: Schema, data: any) => {
+  const keys = Object.keys(schema);
+  // Pick only the keys that are in the schema
+  const filteredData = keys.reduce((acc: any, key) => {
+    if (data[key] || data[key] === 0) {
+      acc[key] = data[key];
+    }
+    return acc;
+  }, {});
+  const columns = "'" + Object.keys(filteredData).join("','") + "'";
+  // Add quotes to column names
+
+  const values = Object.values(filteredData)
+    .map((value) => {
+      if (value === null) {
+        return "";
+      }
+      if (typeof value === "object") {
+        return `'${JSON.stringify(value)}'`;
+      }
+      if (typeof value === "string") {
+        return `'${value}'`;
+      }
+      return value;
+    })
+    .join(", ");
+  const query = `INSERT OR REPLACE INTO ${table} (${columns}) VALUES (${values});`;
+  persistence.db.exec(query);
+};
+
+export const loadLabels = async (workspaceSlug: string, batchSize = 500) => {
+  const issueLabelService = new IssueLabelService();
+  const objects = await issueLabelService.getWorkspaceIssueLabels(workspaceSlug);
+  for (let i = 0; i < objects.length; i += batchSize) {
+    const batch = objects.slice(i, i + batchSize);
+
+    persistence.db.exec("BEGIN TRANSACTION;");
+    batch.forEach((label: any) => {
+      stageInserts("labels", labelSchema, label);
+    });
+    await persistence.db.exec("COMMIT;");
+  }
+};
+
+export const loadModules = async (workspaceSlug: string, batchSize = 500) => {
+  const moduleService = new ModuleService();
+  const objects = await moduleService.getWorkspaceModules(workspaceSlug);
+  for (let i = 0; i < objects.length; i += batchSize) {
+    const batch = objects.slice(i, i + batchSize);
+
+    persistence.db.exec("BEGIN TRANSACTION;");
+    batch.forEach((label: any) => {
+      stageInserts("modules", labelSchema, label);
+    });
+    await persistence.db.exec("COMMIT;");
+  }
+};
+
+export const loadCycles = async (workspaceSlug: string, batchSize = 500) => {
+  const cycleService = new CycleService();
+
+  const objects = await cycleService.getWorkspaceCycles(workspaceSlug);
+  for (let i = 0; i < objects.length; i += batchSize) {
+    const batch = objects.slice(i, i + batchSize);
+
+    persistence.db.exec("BEGIN TRANSACTION;");
+    batch.forEach((cycle: any) => {
+      stageInserts("cycles", cycleSchema, cycle);
+    });
+    await persistence.db.exec("COMMIT;");
+  }
+};
+
+export const loadStates = async (workspaceSlug: string, batchSize = 500) => {
+  const stateService = new ProjectStateService();
+  const objects = await stateService.getWorkspaceStates(workspaceSlug);
+  for (let i = 0; i < objects.length; i += batchSize) {
+    const batch = objects.slice(i, i + batchSize);
+
+    persistence.db.exec("BEGIN TRANSACTION;");
+    batch.forEach((state: any) => {
+      stageInserts("states", stateSchema, state);
+    });
+    await persistence.db.exec("COMMIT;");
+  }
+};
+
+export const loadEstimatePoints = async (workspaceSlug: string, batchSize = 500) => {
+  const estimateService = new EstimateService();
+  const estimates = await estimateService.fetchWorkspaceEstimates(workspaceSlug);
+  const objects: IEstimatePoint[] = [];
+  (estimates || []).forEach((estimate: IEstimate) => {
+    if (estimate?.points) {
+      objects.concat(estimate.points);
+    }
+  });
+  for (let i = 0; i < objects.length; i += batchSize) {
+    const batch = objects.slice(i, i + batchSize);
+
+    persistence.db.exec("BEGIN TRANSACTION;");
+    batch.forEach((point: any) => {
+      stageInserts("estimate_points", estimatePointSchema, point);
+    });
+    await persistence.db.exec("COMMIT;");
+  }
+};
+
+export const loadMembers = async (workspaceSlug: string, batchSize = 500) => {
+  const workspaceService = new WorkspaceService(API_BASE_URL);
+  const members = await workspaceService.fetchWorkspaceMembers(workspaceSlug);
+  const objects = members.map((member: IWorkspaceMember) => member.member);
+  for (let i = 0; i < objects.length; i += batchSize) {
+    const batch = objects.slice(i, i + batchSize);
+    persistence.db.exec("BEGIN TRANSACTION;");
+    batch.forEach((member: any) => {
+      stageInserts("members", memberSchema, member);
+    });
+    await persistence.db.exec("COMMIT;");
+  }
+};
+
+export const loadWorkSpaceData = async (workspaceSlug: string) => {
+  const promises = [];
+  promises.push(loadLabels(workspaceSlug));
+  promises.push(loadModules(workspaceSlug));
+  promises.push(loadCycles(workspaceSlug));
+  promises.push(loadStates(workspaceSlug));
+  promises.push(loadEstimatePoints(workspaceSlug));
+  promises.push(loadMembers(workspaceSlug));
+  await Promise.all(promises);
+};
--- a/web/core/local-db/utils/query-constructor.ts
+++ b/web/core/local-db/utils/query-constructor.ts
@ -0,0 +1,166 @@
+import {
+  getFilteredRowsForGrouping,
+  getIssueFieldsFragment,
+  getMetaKeys,
+  getOrderByFragment,
+  singleFilterConstructor,
+  translateQueryParams,
+} from "./query.utils";
+export const SPECIAL_ORDER_BY = {
+  labels__name: "labels",
+  "-labels__name": "labels",
+  assignees__first_name: "members",
+  "-assignees__first_name": "members",
+  issue_module__module__name: "modules",
+  "-issue_module__module__name": "modules",
+  issue_cycle__cycle__name: "cycles",
+  "-issue_cycle__cycle__name": "cycles",
+  state__name: "states",
+  "-state__name": "states",
+};
+export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => {
+  const {
+    cursor,
+    per_page,
+    group_by,
+    sub_group_by,
+    order_by = "created_at",
+    ...otherProps
+  } = translateQueryParams(queries);
+
+  const [pageSize, page, offset] = cursor.split(":");
+
+  let sql = "";
+
+  const fieldsFragment = getIssueFieldsFragment();
+
+  if (sub_group_by) {
+    const orderByString = getOrderByFragment(order_by);
+    sql = getFilteredRowsForGrouping(projectId, queries);
+    sql += `, ranked_issues AS ( SELECT fi.*,
+    ROW_NUMBER() OVER (PARTITION BY group_id, sub_group_id ${orderByString}) as rank,
+    COUNT(*) OVER (PARTITION by group_id, sub_group_id) as total_issues from fi) 
+    SELECT ri.*, ${fieldsFragment}
+    FROM ranked_issues ri
+    JOIN issues i ON ri.id = i.id
+    WHERE rank <= ${per_page}
+
+    `;
+
+    console.log("###", sql);
+
+    return sql;
+  }
+  if (group_by) {
+    const orderByString = getOrderByFragment(order_by);
+    sql = getFilteredRowsForGrouping(projectId, queries);
+    sql += `, ranked_issues AS ( SELECT fi.*,
+    ROW_NUMBER() OVER (PARTITION BY group_id ${orderByString}) as rank,
+    COUNT(*) OVER (PARTITION by group_id) as total_issues FROM fi)
+    SELECT ri.*, ${fieldsFragment}
+    FROM ranked_issues ri
+    JOIN issues i ON ri.id = i.id
+        WHERE rank <= ${per_page}
+    `;
+
+    console.log("###", sql);
+
+    return sql;
+  }
+
+  if (order_by && Object.keys(SPECIAL_ORDER_BY).includes(order_by)) {
+    const name = order_by.replace("-", "");
+    const orderByString = getOrderByFragment(order_by, "i.");
+
+    sql = `WITH sorted_issues AS (`;
+    sql += getFilteredRowsForGrouping(projectId, queries);
+    sql += `SELECT fi.* , `;
+    if (order_by.includes("assignee")) {
+      sql += ` s.first_name as ${name} `;
+    } else {
+      sql += `  s.name as ${name} `;
+    }
+    sql += `FROM fi `;
+    if (order_by && Object.keys(SPECIAL_ORDER_BY).includes(order_by)) {
+      if (order_by.includes("cycle")) {
+        sql += ` 
+        LEFT JOIN cycles s on fi.cycle_id = s.id`;
+      }
+      if (order_by.includes("estimate_point")) {
+        sql += `
+        LEFT JOIN estimate_points s on fi.estimate_point = s.id`;
+      }
+      if (order_by.includes("state")) {
+        sql += `
+        LEFT JOIN states s on fi.state_id = s.id`;
+      }
+      if (order_by.includes("label")) {
+        sql += ` 
+        LEFT JOIN issue_meta sm ON fi.id = sm.issue_id AND sm.key = 'label_ids'
+        LEFT JOIN labels s ON s.id = sm.value`;
+      }
+      if (order_by.includes("module")) {
+        sql += ` 
+        LEFT JOIN issue_meta sm ON fi.id = sm.issue_id AND sm.key = 'module_ids'
+        LEFT JOIN modules s ON s.id = sm.value`;
+      }
+
+      if (order_by.includes("assignee")) {
+        sql += ` 
+        LEFT JOIN issue_meta sm ON fi.id = sm.issue_id AND sm.key = 'assignee_ids'
+        LEFT JOIN members s ON s.id = sm.value`;
+      }
+
+      sql += ` ORDER BY ${name} ASC NULLS LAST`;
+    }
+    sql += `)`;
+
+    sql += `SELECT ${fieldsFragment}, group_concat(si.${name}) as ${name} from sorted_issues si JOIN issues i ON si.id = i.id
+    `;
+    sql += ` group by i.id ${orderByString} LIMIT ${pageSize} OFFSET ${offset * 1 + page * pageSize};`;
+
+    console.log("######$$$", sql);
+    return sql;
+  }
+
+  const filterJoinFields = getMetaKeys(queries);
+  const orderByString = getOrderByFragment(order_by);
+
+  sql = `SELECT ${fieldsFragment}`;
+  if (otherProps.state_group) {
+    sql += `, states.'group' as state_group`;
+  }
+  sql += ` from issues i
+  `;
+
+  if (otherProps.state_group) {
+    sql += `LEFT JOIN states ON i.state_id = states.id `;
+  }
+  filterJoinFields.forEach((field: string) => {
+    const value = otherProps[field] || "";
+    sql += ` INNER JOIN issue_meta ${field} ON i.id = ${field}.issue_id AND ${field}.key = '${field}' AND ${field}.value  IN ('${value.split(",").join("','")}')
+    `;
+  });
+
+  sql += ` WHERE i.project_id = '${projectId}'    ${singleFilterConstructor(otherProps)} group by i.id  `;
+  sql += orderByString;
+
+  // Add offset and paging to query
+  sql += ` LIMIT  ${pageSize} OFFSET ${offset * 1 + page * pageSize};`;
+
+  console.log("$$$", sql);
+  return sql;
+};
+
+export const issueFilterCountQueryConstructor = (workspaceSlug: string, projectId: string, queries: any) => {
+  //@todo Very crude way to extract count from the actual query. Needs to be refactored
+  // Remove group by from the query to fallback to non group query
+  const { group_by, sub_group_by, order_by, ...otherProps } = queries;
+  let sql = issueFilterQueryConstructor(workspaceSlug, projectId, otherProps);
+  const fieldsFragment = getIssueFieldsFragment();
+
+  sql = sql.replace(`SELECT ${fieldsFragment}`, "SELECT COUNT(DISTINCT i.id) as total_count");
+  // Remove everything after group by i.id
+  sql = `${sql.split("group by i.id")[0]};`;
+  return sql;
+};
--- a/web/core/local-db/utils/query-executor.ts
+++ b/web/core/local-db/utils/query-executor.ts
@ -0,0 +1,13 @@
+// import { SQL } from "./sqlite";
+
+import { persistence } from "../storage.sqlite";
+
+export const runQuery = async (sql: string) => {
+  const data = await persistence.db.exec({
+    sql,
+    rowMode: "object",
+    returnValue: "resultRows",
+  });
+
+  return data.result.resultRows;
+};
--- a/web/core/local-db/utils/query.utils.ts
+++ b/web/core/local-db/utils/query.utils.ts
@ -0,0 +1,335 @@
+import { ARRAY_FIELDS, GROUP_BY_MAP, PRIORITY_MAP } from "./constants";
+import { SPECIAL_ORDER_BY } from "./query-constructor";
+import { issueSchema } from "./schemas";
+import { wrapDateTime } from "./utils";
+
+export const translateQueryParams = (queries: any) => {
+  const { group_by, sub_group_by, labels, assignees, state, cycle, module, priority, type, ...otherProps } = queries;
+
+  const order_by = queries.order_by;
+  if (state) otherProps.state_id = state;
+  if (cycle) otherProps.cycle_id = cycle;
+  if (module) otherProps.module_ids = module;
+  if (labels) otherProps.label_ids = labels;
+  if (assignees) otherProps.assignee_ids = assignees;
+  if (group_by) otherProps.group_by = GROUP_BY_MAP[group_by as keyof typeof GROUP_BY_MAP];
+  if (sub_group_by) otherProps.sub_group_by = GROUP_BY_MAP[sub_group_by as keyof typeof GROUP_BY_MAP];
+  if (priority) {
+    otherProps.priority_proxy = priority
+      .split(",")
+      .map((priority: string) => PRIORITY_MAP[priority as keyof typeof PRIORITY_MAP])
+      .join(",");
+  }
+  if (type) {
+    otherProps.state_group = type === "backlog" ? "backlog" : "unstarted,started";
+  }
+
+  if (order_by?.includes("priority")) {
+    otherProps.order_by = order_by.replace("priority", "priority_proxy");
+  }
+
+  // Fix invalid orderby when switching from spreadsheet layout
+  if ((group_by || sub_group_by) && Object.keys(SPECIAL_ORDER_BY).includes(order_by)) {
+    otherProps.order_by = "sort_order";
+  }
+  // For each property value, replace None with empty string
+  Object.keys(otherProps).forEach((key) => {
+    if (otherProps[key] === "None") {
+      otherProps[key] = "";
+    }
+  });
+
+  return otherProps;
+};
+
+export const getOrderByFragment = (order_by: string, table = "") => {
+  let orderByString = "";
+  if (!order_by) return orderByString;
+
+  if (order_by.startsWith("-")) {
+    orderByString += ` ORDER BY ${wrapDateTime(order_by.slice(1))} DESC NULLS LAST, datetime(${table}created_at) DESC`;
+  } else {
+    orderByString += ` ORDER BY ${wrapDateTime(order_by)} ASC NULLS LAST, datetime(${table}created_at) DESC`;
+  }
+  return orderByString;
+};
+
+export const isMetaJoinRequired = (groupBy: string, subGroupBy: string) =>
+  ARRAY_FIELDS.includes(groupBy) || ARRAY_FIELDS.includes(subGroupBy);
+
+export const getMetaKeysFragment = (queries: any) => {
+  const { group_by, sub_group_by, ...otherProps } = translateQueryParams(queries);
+
+  const fields: Set<string> = new Set();
+  if (ARRAY_FIELDS.includes(group_by)) {
+    fields.add(group_by);
+  }
+
+  if (ARRAY_FIELDS.includes(sub_group_by)) {
+    fields.add(sub_group_by);
+  }
+
+  const keys = Object.keys(otherProps);
+
+  keys.forEach((field: string) => {
+    if (ARRAY_FIELDS.includes(field)) {
+      fields.add(field);
+    }
+  });
+
+  let sql;
+
+  sql = `  ('${Array.from(fields).join("','")}')`;
+
+  return sql;
+};
+
+export const getMetaKeys = (queries: any): string[] => {
+  const { group_by, sub_group_by, ...otherProps } = translateQueryParams(queries);
+
+  const fields: Set<string> = new Set();
+  if (ARRAY_FIELDS.includes(group_by)) {
+    fields.add(group_by);
+  }
+
+  if (ARRAY_FIELDS.includes(sub_group_by)) {
+    fields.add(sub_group_by);
+  }
+
+  const keys = Object.keys(otherProps);
+
+  keys.forEach((field: string) => {
+    if (ARRAY_FIELDS.includes(field)) {
+      fields.add(field);
+    }
+  });
+
+  return Array.from(fields);
+};
+
+const areJoinsRequired = (queries: any) => {
+  const { group_by, sub_group_by, ...otherProps } = translateQueryParams(queries);
+
+  if (ARRAY_FIELDS.includes(group_by) || ARRAY_FIELDS.includes(sub_group_by)) {
+    return true;
+  }
+  if (Object.keys(otherProps).some((field) => ARRAY_FIELDS.includes(field))) {
+    return true;
+  }
+  return false;
+};
+
+// Apply filters to the query
+export const getFilteredRowsForGrouping = (projectId: string, queries: any) => {
+  const { group_by, sub_group_by, ...otherProps } = translateQueryParams(queries);
+
+  const filterJoinFields = getMetaKeys(otherProps);
+
+  const temp = getSingleFilterFields(queries);
+  const issueTableFilterFields = temp.length ? "," + temp.join(",") : "";
+
+  const joinsRequired = areJoinsRequired(queries);
+
+  let sql = "";
+  if (!joinsRequired) {
+    sql = `WITH fi as (SELECT i.id,i.created_at ${issueTableFilterFields}`;
+    if (group_by) {
+      if (group_by === "target_date") {
+        sql += `, date(i.${group_by}) as group_id`;
+      } else {
+        sql += `, i.${group_by} as group_id`;
+      }
+    }
+    if (sub_group_by) {
+      sql += `, i.${sub_group_by} as sub_group_id`;
+    }
+    sql += ` FROM issues i `;
+    if (otherProps.state_group) {
+      sql += `LEFT JOIN states ON i.state_id = states.id `;
+    }
+    sql += `WHERE i.project_id = '${projectId}'
+    `;
+    sql += `${singleFilterConstructor(otherProps)}) 
+    `;
+    return sql;
+  }
+
+  sql = `WITH fi AS (`;
+  sql += `SELECT i.id,i.created_at ${issueTableFilterFields} `;
+  if (group_by) {
+    if (ARRAY_FIELDS.includes(group_by)) {
+      sql += `, ${group_by}.value as group_id
+      `;
+    } else if (group_by === "target_date") {
+      sql += `, date(i.${group_by}) as group_id
+      `;
+    } else {
+      sql += `, i.${group_by} as group_id
+      `;
+    }
+  }
+  if (sub_group_by) {
+    if (ARRAY_FIELDS.includes(sub_group_by)) {
+      sql += `, ${sub_group_by}.value as sub_group_id
+      `;
+    } else {
+      sql += `, i.${sub_group_by} as sub_group_id
+      `;
+    }
+  }
+
+  sql += ` from issues i
+  `;
+  if (otherProps.state_group) {
+    sql += `LEFT JOIN states ON i.state_id = states.id `;
+  }
+  filterJoinFields.forEach((field: string) => {
+    sql += ` INNER JOIN issue_meta ${field} ON i.id = ${field}.issue_id AND ${field}.key = '${field}' AND ${field}.value  IN ('${otherProps[field].split(",").join("','")}')
+    `;
+  });
+
+  // If group by field is not already joined, join it
+  if (ARRAY_FIELDS.includes(group_by) && !filterJoinFields.includes(group_by)) {
+    sql += ` LEFT JOIN issue_meta ${group_by} ON i.id = ${group_by}.issue_id AND ${group_by}.key = '${group_by}'
+    `;
+  }
+  if (ARRAY_FIELDS.includes(sub_group_by) && !filterJoinFields.includes(sub_group_by)) {
+    sql += ` LEFT JOIN issue_meta ${sub_group_by} ON i.id = ${sub_group_by}.issue_id AND ${sub_group_by}.key = '${sub_group_by}'
+    `;
+  }
+
+  sql += ` WHERE i.project_id = '${projectId}'
+  `;
+  sql += singleFilterConstructor(otherProps);
+
+  sql += `)
+  `;
+  return sql;
+};
+
+export const singleFilterConstructor = (queries: any) => {
+  const {
+    order_by,
+    cursor,
+    per_page,
+    group_by,
+    sub_group_by,
+    state_group,
+    sub_issue,
+    target_date,
+    start_date,
+    ...filters
+  } = translateQueryParams(queries);
+
+  let sql = "";
+  if (!sub_issue) {
+    sql += ` AND parent_id IS NULL 
+    `;
+  }
+  if (target_date) {
+    sql += createDateFilter("target_date", target_date);
+  }
+  if (start_date) {
+    sql += createDateFilter("start_date", start_date);
+  }
+  if (state_group) {
+    sql += ` AND state_group in ('${state_group.split(",").join("','")}')
+    `;
+  }
+  const keys = Object.keys(filters);
+
+  keys.forEach((key) => {
+    const value = filters[key] ? filters[key].split(",") : "";
+    if (!value) return;
+    if (!ARRAY_FIELDS.includes(key)) {
+      sql += ` AND ${key} in ('${value.join("','")}')
+      `;
+    }
+  });
+  //
+
+  return sql;
+};
+
+// let q = '2_months;after;fromnow,1_months;after;fromnow,2024-09-01;after,2024-10-06;after,2_weeks;after;fromnow'
+
+// ["2_months;after;fromnow", "1_months;after;fromnow", "2024-09-01;after", "2024-10-06;before", "2_weeks;after;fromnow"];
+
+const createDateFilter = (key: string, q: string) => {
+  let sql = "  ";
+  // get todays date in YYYY-MM-DD format
+  const queries = q.split(",");
+  const customRange: string[] = [];
+  let isAnd = true;
+  queries.forEach((query: string) => {
+    const [date, type, from] = query.split(";");
+    if (from) {
+      // Assuming type is always after
+      let after = "";
+      const [_length, unit] = date.split("_");
+      const length = parseInt(_length);
+
+      if (unit === "weeks") {
+        // get date in yyyy-mm-dd format one week from now
+        after = new Date(new Date().setDate(new Date().getDate() + length * 7)).toISOString().split("T")[0];
+      }
+      if (unit === "months") {
+        after = new Date(new Date().setDate(new Date().getDate() + length * 30)).toISOString().split("T")[0];
+      }
+      sql += ` ${isAnd ? "AND" : "OR"} ${key} >= date('${after}')`;
+      isAnd = false;
+      // sql += ` AND ${key} ${type === "after" ? ">=" : "<="} date('${date}', '${today}')`;
+    } else {
+      customRange.push(query);
+    }
+  });
+
+  if (customRange.length === 2) {
+    const end = customRange.find((date) => date.includes("before"))?.split(";")[0];
+    const start = customRange.find((date) => date.includes("after"))?.split(";")[0];
+    if (end && start) {
+      sql += ` ${isAnd ? "AND" : "OR"} ${key} BETWEEN date('${start}') AND date('${end}')`;
+    }
+  }
+  if (customRange.length === 1) {
+    sql += ` AND ${key}=date('${customRange[0].split(";")[0]}')`;
+  }
+
+  return sql;
+};
+const getSingleFilterFields = (queries: any) => {
+  const { order_by, cursor, per_page, group_by, sub_group_by, sub_issue, state_group, ...otherProps } =
+    translateQueryParams(queries);
+
+  const fields = new Set();
+
+  if (order_by && !order_by.includes("created_at") && !Object.keys(SPECIAL_ORDER_BY).includes(order_by))
+    fields.add(order_by.replace("-", ""));
+
+  const keys = Object.keys(otherProps);
+
+  keys.forEach((field: string) => {
+    if (!ARRAY_FIELDS.includes(field)) {
+      fields.add(field);
+    }
+  });
+
+  if (order_by?.includes("state__name")) {
+    fields.add("state_id");
+  }
+  if (order_by?.includes("cycle__name")) {
+    fields.add("cycle_id");
+  }
+  if (state_group) {
+    fields.add("states.'group' as state_group");
+  }
+  return Array.from(fields);
+};
+
+export const getIssueFieldsFragment = () => {
+  const { description_html, ...filtered } = issueSchema;
+  const keys = Object.keys(filtered);
+  const sql = `  ${keys.map((key, index) => `i.${key}`).join(`,
+    `)}`;
+  return sql;
+};
--- a/web/core/local-db/utils/schemas.ts
+++ b/web/core/local-db/utils/schemas.ts
@ -0,0 +1,135 @@
+export type Schema = {
+  [key: string]: string;
+};
+
+export const issueSchema: Schema = {
+  id: "TEXT UNIQUE",
+  name: "TEXT",
+  state_id: "TEXT",
+  sort_order: "REAL",
+  completed_at: "TEXT",
+  estimate_point: "REAL",
+  priority: "TEXT",
+  priority_proxy: "INTEGER",
+  start_date: "TEXT",
+  target_date: "TEXT",
+  sequence_id: "INTEGER",
+  project_id: "TEXT",
+  parent_id: "TEXT",
+  created_at: "TEXT",
+  updated_at: "TEXT",
+  created_by: "TEXT",
+  updated_by: "TEXT",
+  is_draft: "INTEGER",
+  archived_at: "TEXT",
+  state__group: "TEXT",
+  sub_issues_count: "INTEGER",
+  cycle_id: "TEXT",
+  link_count: "INTEGER",
+  attachment_count: "INTEGER",
+  type_id: "TEXT",
+  label_ids: "TEXT",
+  assignee_ids: "TEXT",
+  module_ids: "TEXT",
+  description_html: "TEXT",
+};
+
+export const issueMetaSchema: Schema = {
+  issue_id: "TEXT",
+  key: "TEXT",
+  value: "TEXT",
+};
+export const moduleSchema: Schema = {
+  id: "TEXT UNIQUE",
+  workspace_id: "TEXT",
+  project_id: "TEXT",
+  name: "TEXT",
+  description: "TEXT",
+  description_text: "TEXT",
+  description_html: "TEXT",
+  start_date: "TEXT",
+  target_date: "TEXT",
+  status: "TEXT",
+  lead_id: "TEXT",
+  member_ids: "TEXT",
+  view_props: "TEXT",
+  sort_order: "INTEGER",
+  external_source: "TEXT",
+  external_id: "TEXT",
+  logo_props: "TEXT",
+  total_issues: "INTEGER",
+  cancelled_issues: "INTEGER",
+  completed_issues: "INTEGER",
+  started_issues: "INTEGER",
+  unstarted_issues: "INTEGER",
+  backlog_issues: "INTEGER",
+  created_at: "TEXT",
+  updated_at: "TEXT",
+  archived_at: "TEXT",
+};
+
+export const labelSchema: Schema = {
+  id: "TEXT UNIQUE",
+  name: "TEXT",
+  color: "TEXT",
+  parent: "TEXT",
+  project_id: "TEXT",
+  workspace_id: "TEXT",
+  sort_order: "INTEGER",
+};
+
+export const cycleSchema: Schema = {
+  id: "TEXT UNIQUE",
+  workspace_id: "TEXT",
+  project_id: "TEXT",
+  name: "TEXT",
+  description: "TEXT",
+  start_date: "TEXT",
+  end_date: "TEXT",
+  owned_by_id: "TEXT",
+  view_props: "TEXT",
+  sort_order: "INTEGER",
+  external_source: "TEXT",
+  external_id: "TEXT",
+  progress_snapshot: "TEXT",
+  logo_props: "TEXT",
+  total_issues: "INTEGER",
+  cancelled_issues: "INTEGER",
+  completed_issues: "INTEGER",
+  started_issues: "INTEGER",
+  unstarted_issues: "INTEGER",
+  backlog_issues: "INTEGER",
+};
+
+export const stateSchema: Schema = {
+  id: "TEXT UNIQUE",
+  project_id: "TEXT",
+  workspace_id: "TEXT",
+  name: "TEXT",
+  color: "TEXT",
+  group: "TEXT",
+  default: "BOOLEAN",
+  description: "TEXT",
+  sequence: "INTEGER",
+};
+
+export const estimatePointSchema: Schema = {
+  id: "TEXT UNIQUE",
+  key: "TEXT",
+  value: "REAL",
+};
+
+export const memberSchema: Schema = {
+  id: "TEXT UNIQUE",
+  first_name: "TEXT",
+  last_name: "TEXT",
+  avatar: "TEXT",
+  is_bot: "BOOLEAN",
+  display_name: "TEXT",
+  email: "TEXT",
+};
+
+export const optionsSchema: Schema = {
+  key: "TEXT UNIQUE",
+  value: "TEXT",
+};
--- a/web/core/local-db/utils/tables.ts
+++ b/web/core/local-db/utils/tables.ts
@ -0,0 +1,39 @@
+import { persistence } from "../storage.sqlite";
+import {
+  labelSchema,
+  moduleSchema,
+  Schema,
+  issueMetaSchema,
+  issueSchema,
+  stateSchema,
+  cycleSchema,
+  estimatePointSchema,
+  memberSchema,
+  optionsSchema,
+} from "./schemas";
+
+const createTableSQLfromSchema = (tableName: string, schema: Schema) => {
+  let sql = `CREATE TABLE IF NOT EXISTS ${tableName} (`;
+  sql += Object.keys(schema)
+    .map((key) => `'${key}' ${schema[key]}`)
+    .join(", ");
+  sql += `);`;
+  console.log("#####", sql);
+  return sql;
+};
+
+export const createTables = async () => {
+  persistence.db.exec("BEGIN TRANSACTION;");
+
+  persistence.db.exec(createTableSQLfromSchema("issues", issueSchema));
+  persistence.db.exec(createTableSQLfromSchema("issue_meta", issueMetaSchema));
+  persistence.db.exec(createTableSQLfromSchema("modules", moduleSchema));
+  persistence.db.exec(createTableSQLfromSchema("labels", labelSchema));
+  persistence.db.exec(createTableSQLfromSchema("states", stateSchema));
+  persistence.db.exec(createTableSQLfromSchema("cycles", cycleSchema));
+  persistence.db.exec(createTableSQLfromSchema("estimate_points", estimatePointSchema));
+  persistence.db.exec(createTableSQLfromSchema("members", memberSchema));
+  persistence.db.exec(createTableSQLfromSchema("options", optionsSchema));
+
+  persistence.db.exec("COMMIT;");
+};
--- a/web/core/local-db/utils/utils.ts
+++ b/web/core/local-db/utils/utils.ts
@ -0,0 +1,134 @@
+import pick from "lodash/pick";
+import { TIssue } from "@plane/types";
+import { rootStore } from "@/lib/store-context";
+import { updateIssue } from "./load-issues";
+
+export const log = console.log;
+
+// export const log = () => {};
+
+export const updatePersistentLayer = async (issueIds: string | string[]) => {
+  if (typeof issueIds === "string") {
+    issueIds = [issueIds];
+  }
+  issueIds.forEach((issueId) => {
+    const issue = rootStore.issue.issues.getIssueById(issueId);
+
+    if (issue) {
+      const issuePartial = pick(JSON.parse(JSON.stringify(issue)), [
+        "id",
+        "name",
+        "state_id",
+        "sort_order",
+        "completed_at",
+        "estimate_point",
+        "priority",
+        "start_date",
+        "target_date",
+        "sequence_id",
+        "project_id",
+        "parent_id",
+        "created_at",
+        "updated_at",
+        "created_by",
+        "updated_by",
+        "is_draft",
+        "archived_at",
+        "state__group",
+        "cycle_id",
+        "link_count",
+        "attachment_count",
+        "sub_issues_count",
+        "assignee_ids",
+        "label_ids",
+        "module_ids",
+        "type_id",
+      ]);
+      updateIssue(issuePartial);
+    }
+  });
+};
+
+export const wrapDateTime = (field: string) => {
+  const DATE_TIME_FIELDS = ["created_at", "updated_at", "completed_at", "start_date", "target_date"];
+
+  if (DATE_TIME_FIELDS.includes(field)) {
+    return `datetime(${field})`;
+  }
+  return field;
+};
+
+export const getGroupedIssueResults = (issueResults: (TIssue & { group_id: string; total_issues: number })[]): any => {
+  const groupedResults: {
+    [key: string]: {
+      results: TIssue[];
+      total_results: number;
+    };
+  } = {};
+
+  for (const issue of issueResults) {
+    const { group_id, total_issues } = issue;
+    const groupId = group_id ? group_id : "None";
+    if (groupedResults?.[groupId] !== undefined && Array.isArray(groupedResults?.[groupId]?.results)) {
+      groupedResults?.[groupId]?.results.push(issue);
+    } else {
+      groupedResults[groupId] = { results: [issue], total_results: total_issues };
+    }
+  }
+
+  return groupedResults;
+};
+
+export const getSubGroupedIssueResults = (
+  issueResults: (TIssue & { group_id: string; total_issues: number; sub_group_id: string })[]
+): any => {
+  const subGroupedResults: {
+    [key: string]: {
+      results: {
+        [key: string]: {
+          results: TIssue[];
+          total_results: number;
+        };
+      };
+      total_results: number;
+    };
+  } = {};
+
+  for (const issue of issueResults) {
+    const { group_id, total_issues, sub_group_id } = issue;
+    const groupId = group_id ? group_id : "None";
+    const subGroupId = sub_group_id ? sub_group_id : "None";
+
+    if (subGroupedResults?.[groupId] === undefined) {
+      subGroupedResults[groupId] = { results: {}, total_results: 0 };
+    }
+
+    if (
+      subGroupedResults[groupId].results[subGroupId] !== undefined &&
+      Array.isArray(subGroupedResults[groupId].results[subGroupId]?.results)
+    ) {
+      subGroupedResults[groupId].results[subGroupId]?.results.push(issue);
+    } else {
+      subGroupedResults[groupId].results[subGroupId] = { results: [issue], total_results: total_issues };
+    }
+  }
+
+  const groupByKeys = Object.keys(subGroupedResults);
+
+  for (const groupByKey of groupByKeys) {
+    let totalIssues = 0;
+    const groupedResults = subGroupedResults[groupByKey]?.results ?? {};
+    const subGroupByKeys = Object.keys(groupedResults);
+
+    for (const subGroupByKey of subGroupByKeys) {
+      const subGroupedResultsCount = groupedResults[subGroupByKey].total_results ?? 0;
+      totalIssues += subGroupedResultsCount;
+    }
+
+    subGroupedResults[groupByKey].total_results = totalIssues;
+  }
+
+  return subGroupedResults;
+};
+
+export const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));