Skip to content

Commit

Permalink
- Change batch size to 50 for inserting issues (#6085)
Browse files Browse the repository at this point in the history
- Fallback to server when mentions filter is used
- Split load workspace into multiple transactions
  • Loading branch information
SatishGandham authored Nov 26, 2024
1 parent 32048be commit 6376a09
Show file tree
Hide file tree
Showing 5 changed files with 41 additions and 9 deletions.
6 changes: 3 additions & 3 deletions web/core/local-db/storage.sqlite.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import { clearOPFS, getGroupedIssueResults, getSubGroupedIssueResults, log, logE

const DB_VERSION = 1;
const PAGE_SIZE = 500;
const BATCH_SIZE = 500;
const BATCH_SIZE = 50;

type TProjectStatus = {
issues: { status: undefined | "loading" | "ready" | "error" | "syncing"; sync: Promise<void> | undefined };
Expand Down Expand Up @@ -382,10 +382,10 @@ export class Storage {

getIssue = async (issueId: string) => {
try {
if (!rootStore.user.localDBEnabled) return;
if (!rootStore.user.localDBEnabled || this.status !== "ready") return;

const issues = await runQuery(`select * from issues where id='${issueId}'`);
if (issues.length) {
if (Array.isArray(issues) && issues.length) {
return formatLocalIssue(issues[0]);
}
} catch (err) {
Expand Down
8 changes: 5 additions & 3 deletions web/core/local-db/utils/load-issues.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export const addIssue = async (issue: any) => {
await persistence.db.exec("COMMIT;");
};

export const addIssuesBulk = async (issues: any, batchSize = 100) => {
export const addIssuesBulk = async (issues: any, batchSize = 50) => {
if (!rootStore.user.localDBEnabled || !persistence.db) return;
if (!issues.length) return;
const insertStart = performance.now();
Expand All @@ -24,18 +24,20 @@ export const addIssuesBulk = async (issues: any, batchSize = 100) => {
for (let i = 0; i < issues.length; i += batchSize) {
const batch = issues.slice(i, i + batchSize);

const promises = [];
for (let j = 0; j < batch.length; j++) {
const issue = batch[j];
if (!issue.type_id) {
issue.type_id = "";
}
await stageIssueInserts(issue);
promises.push(stageIssueInserts(issue));
}
await Promise.all(promises);
}
await persistence.db.exec("COMMIT;");

const insertEnd = performance.now();
log("Inserted issues in ", `${insertEnd - insertStart}ms`);
log("Inserted issues in ", `${insertEnd - insertStart}ms`, batchSize, issues.length);
};
export const deleteIssueFromLocal = async (issue_id: any) => {
if (!rootStore.user.localDBEnabled || !persistence.db) return;
Expand Down
16 changes: 16 additions & 0 deletions web/core/local-db/utils/load-workspace.ts
Original file line number Diff line number Diff line change
Expand Up @@ -120,12 +120,28 @@ export const loadWorkSpaceData = async (workspaceSlug: string) => {
const start = performance.now();
await persistence.db.exec("BEGIN;");
await batchInserts(labels, "labels", labelSchema);
await persistence.db.exec("COMMIT;");

await persistence.db.exec("BEGIN;");
await batchInserts(modules, "modules", moduleSchema);
await persistence.db.exec("COMMIT;");

await persistence.db.exec("BEGIN;");
await batchInserts(cycles, "cycles", cycleSchema);
await persistence.db.exec("COMMIT;");

await persistence.db.exec("BEGIN;");
await batchInserts(states, "states", stateSchema);
await persistence.db.exec("COMMIT;");

await persistence.db.exec("BEGIN;");
await batchInserts(estimates, "estimate_points", estimatePointSchema);
await persistence.db.exec("COMMIT;");

await persistence.db.exec("BEGIN;");
await batchInserts(members, "members", memberSchema);
await persistence.db.exec("COMMIT;");

const end = performance.now();
log("Time taken to load workspace data", end - start);
};
5 changes: 2 additions & 3 deletions web/core/services/issue/issue.service.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
import { startSpan } from "@sentry/nextjs";
import isEmpty from "lodash/isEmpty";
// types
import type {
IIssueDisplayProperties,
Expand All @@ -12,6 +10,7 @@ import type {
} from "@plane/types";
// helpers
import { API_BASE_URL } from "@/helpers/common.helper";
import { getIssuesShouldFallbackToServer } from "@/helpers/issue.helper";
import { persistence } from "@/local-db/storage.sqlite";
// services

Expand Down Expand Up @@ -68,7 +67,7 @@ export class IssueService extends APIService {
}

async getIssues(workspaceSlug: string, projectId: string, queries?: any, config = {}): Promise<TIssuesResponse> {
if (!isEmpty(queries.expand as string) && !queries.group_by)
if (getIssuesShouldFallbackToServer(queries))
return await this.getIssuesFromServer(workspaceSlug, projectId, queries, config);

const response = await persistence.getIssues(workspaceSlug, projectId, queries, config);
Expand Down
15 changes: 15 additions & 0 deletions web/helpers/issue.helper.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import differenceInCalendarDays from "date-fns/differenceInCalendarDays";
import isEmpty from "lodash/isEmpty";
import set from "lodash/set";
import { v4 as uuidv4 } from "uuid";
// types
Expand Down Expand Up @@ -308,3 +309,17 @@ export const getComputedDisplayProperties = (
cycle: displayProperties?.cycle ?? true,
issue_type: displayProperties?.issue_type ?? true,
});

/**
* This is to check if the issues list api should fall back to server or use local db
* @param queries
* @returns
*/
export const getIssuesShouldFallbackToServer = (queries: any) => {
// If there is expand query and is not grouped then fallback to server
if (!isEmpty(queries.expand as string) && !queries.group_by) return true;
// If query has mentions then fallback to server
if (!isEmpty(queries.mentions)) return true;

return false;
};

0 comments on commit 6376a09

Please sign in to comment.