[WEB-2706] chore: Switch to wa-sqlite (#5859)
* fix layout switching when filter is not yet completely fetched * add layout in issue filter params * Handle cases when DB intilization failed * chore: permission layer and updated issues v1 query from workspace to project level * - Switch to using wa-sqlite instead of sqlite-wasm * Code cleanup and fix indexes * Add missing files * - Import only required functions from sentry - Wait till all the tables are created * Skip workspace sync if one is already in progress. * Sync workspace without using transaction * Minor cleanup * Close DB connection before deleting files Fix clear OPFS on safari * Fix type issue * Improve issue insert performance * Refactor workspace sync * Close the DB connection while switching workspaces * Update web/core/local-db/worker/db.ts Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * Worker cleanup and error handling Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * Update web/core/local-db/worker/db.ts Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * Update web/core/local-db/storage.sqlite.ts Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * Update web/core/local-db/worker/db.ts Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * Code cleanup * Set default order by to created at and descending * Wait for transactions to complete. --------- Co-authored-by: rahulramesha <rahulramesham@gmail.com> Co-authored-by: gurusainath <gurusainath007@gmail.com> Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
This commit is contained in:
Satish Gandham
GitHub
parent
ad25a972a1
commit
9fb353ef54
38 changed files with 4318 additions and 220 deletions
|
|
@ -1,4 +1,5 @@
|
|||
import * as Sentry from "@sentry/nextjs";
|
||||
import { getActiveSpan, startSpan } from "@sentry/nextjs";
|
||||
import * as Comlink from "comlink";
|
||||
import set from "lodash/set";
|
||||
// plane
|
||||
import { EIssueGroupBYServerToProperty } from "@plane/constants";
|
||||
|
|
@ -16,15 +17,11 @@ import { loadWorkSpaceData } from "./utils/load-workspace";
|
|||
import { issueFilterCountQueryConstructor, issueFilterQueryConstructor } from "./utils/query-constructor";
|
||||
import { runQuery } from "./utils/query-executor";
|
||||
import { createTables } from "./utils/tables";
|
||||
import { getGroupedIssueResults, getSubGroupedIssueResults, log, logError, logInfo } from "./utils/utils";
|
||||
|
||||
declare module "@sqlite.org/sqlite-wasm" {
|
||||
export function sqlite3Worker1Promiser(...args: any): any;
|
||||
}
|
||||
import { clearOPFS, getGroupedIssueResults, getSubGroupedIssueResults, log, logError } from "./utils/utils";
|
||||
|
||||
const DB_VERSION = 1;
|
||||
const PAGE_SIZE = 1000;
|
||||
const BATCH_SIZE = 200;
|
||||
const PAGE_SIZE = 500;
|
||||
const BATCH_SIZE = 500;
|
||||
|
||||
type TProjectStatus = {
|
||||
issues: { status: undefined | "loading" | "ready" | "error" | "syncing"; sync: Promise<void> | undefined };
|
||||
|
|
@ -43,6 +40,9 @@ export class Storage {
|
|||
}
|
||||
|
||||
reset = () => {
|
||||
if (this.db) {
|
||||
this.db.close();
|
||||
}
|
||||
this.db = null;
|
||||
this.status = undefined;
|
||||
this.projectStatus = {};
|
||||
|
|
@ -51,10 +51,8 @@ export class Storage {
|
|||
|
||||
clearStorage = async () => {
|
||||
try {
|
||||
const storageManager = window.navigator.storage;
|
||||
const fileSystemDirectoryHandle = await storageManager.getDirectory();
|
||||
//@ts-expect-error , clear local issue cache
|
||||
await fileSystemDirectoryHandle.remove({ recursive: true });
|
||||
await this.db.close();
|
||||
await clearOPFS();
|
||||
this.reset();
|
||||
} catch (e) {
|
||||
console.error("Error clearing sqlite sync storage", e);
|
||||
|
|
@ -62,13 +60,13 @@ export class Storage {
|
|||
};
|
||||
|
||||
initialize = async (workspaceSlug: string): Promise<boolean> => {
|
||||
if (document.hidden || !rootStore.user.localDBEnabled) return false; // return if the window gets hidden
|
||||
if (!rootStore.user.localDBEnabled) return false; // return if the window gets hidden
|
||||
|
||||
if (workspaceSlug !== this.workspaceSlug) {
|
||||
this.reset();
|
||||
}
|
||||
try {
|
||||
await Sentry.startSpan({ name: "INIT_DB" }, async () => await this._initialize(workspaceSlug));
|
||||
await startSpan({ name: "INIT_DB" }, async () => await this._initialize(workspaceSlug));
|
||||
return true;
|
||||
} catch (err) {
|
||||
logError(err);
|
||||
|
|
@ -91,71 +89,61 @@ export class Storage {
|
|||
return false;
|
||||
}
|
||||
|
||||
logInfo("Loading and initializing SQLite3 module...");
|
||||
|
||||
this.workspaceSlug = workspaceSlug;
|
||||
this.dbName = workspaceSlug;
|
||||
const { sqlite3Worker1Promiser } = await import("@sqlite.org/sqlite-wasm");
|
||||
|
||||
try {
|
||||
const promiser: any = await new Promise((resolve) => {
|
||||
const _promiser = sqlite3Worker1Promiser({
|
||||
onready: () => resolve(_promiser),
|
||||
});
|
||||
});
|
||||
const { DBClass } = await import("./worker/db");
|
||||
const worker = new Worker(new URL("./worker/db.ts", import.meta.url));
|
||||
const MyWorker = Comlink.wrap<typeof DBClass>(worker);
|
||||
|
||||
const configResponse = await promiser("config-get", {});
|
||||
log("Running SQLite3 version", configResponse.result.version.libVersion);
|
||||
// Add cleanup on window unload
|
||||
window.addEventListener("unload", () => worker.terminate());
|
||||
|
||||
this.workspaceSlug = workspaceSlug;
|
||||
this.dbName = workspaceSlug;
|
||||
const instance = await new MyWorker();
|
||||
await instance.init(workspaceSlug);
|
||||
|
||||
const openResponse = await promiser("open", {
|
||||
filename: `file:${this.dbName}.sqlite3?vfs=opfs`,
|
||||
});
|
||||
const { dbId } = openResponse;
|
||||
this.db = {
|
||||
dbId,
|
||||
exec: async (val: any) => {
|
||||
if (typeof val === "string") {
|
||||
val = { sql: val };
|
||||
}
|
||||
return promiser("exec", { dbId, ...val });
|
||||
},
|
||||
exec: instance.exec,
|
||||
close: instance.close,
|
||||
};
|
||||
|
||||
// dump DB of db version is matching
|
||||
const dbVersion = await this.getOption("DB_VERSION");
|
||||
if (dbVersion !== "" && parseInt(dbVersion) !== DB_VERSION) {
|
||||
await this.clearStorage();
|
||||
this.reset();
|
||||
await this._initialize(workspaceSlug);
|
||||
return false;
|
||||
}
|
||||
|
||||
log(
|
||||
"OPFS is available, created persisted database at",
|
||||
openResponse.result.filename.replace(/^file:(.*?)\?vfs=opfs$/, "$1")
|
||||
);
|
||||
this.status = "ready";
|
||||
// Your SQLite code here.
|
||||
await createTables();
|
||||
|
||||
await this.setOption("DB_VERSION", DB_VERSION.toString());
|
||||
} catch (err) {
|
||||
logError(err);
|
||||
throw err;
|
||||
return true;
|
||||
} catch (error) {
|
||||
this.status = "error";
|
||||
throw new Error(`Failed to initialize database worker: ${error}`);
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
syncWorkspace = async () => {
|
||||
if (document.hidden || !rootStore.user.localDBEnabled) return; // return if the window gets hidden
|
||||
await Sentry.startSpan({ name: "LOAD_WS", attributes: { slug: this.workspaceSlug } }, async () => {
|
||||
await loadWorkSpaceData(this.workspaceSlug);
|
||||
});
|
||||
if (!rootStore.user.localDBEnabled) return;
|
||||
const syncInProgress = await this.getIsWriteInProgress("sync_workspace");
|
||||
if (syncInProgress) {
|
||||
log("Sync in progress, skipping");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await startSpan({ name: "LOAD_WS", attributes: { slug: this.workspaceSlug } }, async () => {
|
||||
this.setOption("sync_workspace", new Date().toUTCString());
|
||||
await loadWorkSpaceData(this.workspaceSlug);
|
||||
this.deleteOption("sync_workspace");
|
||||
});
|
||||
} catch (e) {
|
||||
logError(e);
|
||||
this.deleteOption("sync_workspace");
|
||||
}
|
||||
};
|
||||
|
||||
syncProject = async (projectId: string) => {
|
||||
if (document.hidden || !rootStore.user.localDBEnabled) return false; // return if the window gets hidden
|
||||
if (
|
||||
// document.hidden ||
|
||||
!rootStore.user.localDBEnabled
|
||||
)
|
||||
return false; // return if the window gets hidden
|
||||
|
||||
// Load labels, members, states, modules, cycles
|
||||
await this.syncIssues(projectId);
|
||||
|
|
@ -173,10 +161,11 @@ export class Storage {
|
|||
};
|
||||
|
||||
syncIssues = async (projectId: string) => {
|
||||
if (document.hidden || !rootStore.user.localDBEnabled) return false; // return if the window gets hidden
|
||||
|
||||
if (!rootStore.user.localDBEnabled || !this.db) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
const sync = Sentry.startSpan({ name: `SYNC_ISSUES` }, () => this._syncIssues(projectId));
|
||||
const sync = startSpan({ name: `SYNC_ISSUES` }, () => this._syncIssues(projectId));
|
||||
this.setSync(projectId, sync);
|
||||
await sync;
|
||||
} catch (e) {
|
||||
|
|
@ -186,12 +175,12 @@ export class Storage {
|
|||
};
|
||||
|
||||
_syncIssues = async (projectId: string) => {
|
||||
const activeSpan = Sentry.getActiveSpan();
|
||||
const activeSpan = getActiveSpan();
|
||||
|
||||
log("### Sync started");
|
||||
let status = this.getStatus(projectId);
|
||||
if (status === "loading" || status === "syncing") {
|
||||
logInfo(`Project ${projectId} is already loading or syncing`);
|
||||
log(`Project ${projectId} is already loading or syncing`);
|
||||
return;
|
||||
}
|
||||
const syncPromise = this.getSync(projectId);
|
||||
|
|
@ -222,8 +211,8 @@ export class Storage {
|
|||
const issueService = new IssueService();
|
||||
|
||||
const response = await issueService.getIssuesForSync(this.workspaceSlug, projectId, queryParams);
|
||||
addIssuesBulk(response.results, BATCH_SIZE);
|
||||
|
||||
await addIssuesBulk(response.results, BATCH_SIZE);
|
||||
if (response.total_pages > 1) {
|
||||
const promiseArray = [];
|
||||
for (let i = 1; i < response.total_pages; i++) {
|
||||
|
|
@ -290,7 +279,7 @@ export class Storage {
|
|||
!rootStore.user.localDBEnabled
|
||||
) {
|
||||
if (rootStore.user.localDBEnabled) {
|
||||
logInfo(`Project ${projectId} is loading, falling back to server`);
|
||||
log(`Project ${projectId} is loading, falling back to server`);
|
||||
}
|
||||
const issueService = new IssueService();
|
||||
return await issueService.getIssuesFromServer(workspaceSlug, projectId, queries);
|
||||
|
|
@ -310,11 +299,9 @@ export class Storage {
|
|||
const issueService = new IssueService();
|
||||
return await issueService.getIssuesFromServer(workspaceSlug, projectId, queries);
|
||||
}
|
||||
// const issuesRaw = await runQuery(query);
|
||||
const end = performance.now();
|
||||
|
||||
const { total_count } = count[0];
|
||||
// const total_count = 2300;
|
||||
|
||||
const [pageSize, page, offset] = cursor.split(":");
|
||||
|
||||
|
|
@ -347,7 +334,6 @@ export class Storage {
|
|||
Parsing: parsingEnd - parsingStart,
|
||||
Grouping: groupingEnd - grouping,
|
||||
};
|
||||
log(issueResults);
|
||||
if ((window as any).DEBUG) {
|
||||
console.table(times);
|
||||
}
|
||||
|
|
@ -364,7 +350,7 @@ export class Storage {
|
|||
total_pages,
|
||||
};
|
||||
|
||||
const activeSpan = Sentry.getActiveSpan();
|
||||
const activeSpan = getActiveSpan();
|
||||
activeSpan?.setAttributes({
|
||||
projectId,
|
||||
count: total_count,
|
||||
|
|
@ -413,7 +399,7 @@ export class Storage {
|
|||
set(this.projectStatus, `${projectId}.issues.sync`, sync);
|
||||
};
|
||||
|
||||
getOption = async (key: string, fallback = "") => {
|
||||
getOption = async (key: string, fallback?: string | boolean | number) => {
|
||||
try {
|
||||
const options = await runQuery(`select * from options where key='${key}'`);
|
||||
if (options.length) {
|
||||
|
|
@ -429,6 +415,9 @@ export class Storage {
|
|||
await runQuery(`insert or replace into options (key, value) values ('${key}', '${value}')`);
|
||||
};
|
||||
|
||||
deleteOption = async (key: string) => {
|
||||
await runQuery(` DELETE FROM options where key='${key}'`);
|
||||
};
|
||||
getOptions = async (keys: string[]) => {
|
||||
const options = await runQuery(`select * from options where key in ('${keys.join("','")}')`);
|
||||
return options.reduce((acc: any, option: any) => {
|
||||
|
|
@ -436,6 +425,21 @@ export class Storage {
|
|||
return acc;
|
||||
}, {});
|
||||
};
|
||||
|
||||
getIsWriteInProgress = async (op: string) => {
|
||||
const writeStartTime = await this.getOption(op, false);
|
||||
if (writeStartTime) {
|
||||
// Check if it has been more than 5seconds
|
||||
const current = new Date();
|
||||
const start = new Date(writeStartTime);
|
||||
|
||||
if (current.getTime() - start.getTime() < 5000) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
}
|
||||
|
||||
export const persistence = new Storage();
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue