From a5a855b22fb74807db08e53f370b260afc4969e2 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Mon, 18 Oct 2021 18:06:14 +0200 Subject: [PATCH] keep async but have tmp cache --- .../src/datastore/lmdb/lmdb-datastore.ts | 51 ++++++++++++++----- .../datastore/lmdb/updates/nodes-by-type.ts | 5 +- .../src/datastore/lmdb/updates/nodes.ts | 3 +- 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/packages/gatsby/src/datastore/lmdb/lmdb-datastore.ts b/packages/gatsby/src/datastore/lmdb/lmdb-datastore.ts index 07b9d062ee0ca..ca77c12a83fca 100644 --- a/packages/gatsby/src/datastore/lmdb/lmdb-datastore.ts +++ b/packages/gatsby/src/datastore/lmdb/lmdb-datastore.ts @@ -1,6 +1,6 @@ -import { RootDatabase, open } from "lmdb-store" +import { RootDatabase, open, ArrayLikeIterable } from "lmdb-store" // import { performance } from "perf_hooks" -import { ActionsUnion, IGatsbyNode } from "../../redux/types" +import { ActionsUnion, IDeleteNodeAction, IGatsbyNode } from "../../redux/types" import { updateNodes } from "./updates/nodes" import { updateNodesByType } from "./updates/nodes-by-type" import { IDataStore, ILmdbDatabases, IQueryResult } from "../types" @@ -27,6 +27,8 @@ const lmdbDatastore = { getNodesByType, } +const preSyncDeletedNodeIdsCache = new Set() + function getDefaultDbPath(): string { const dbFileName = process.env.NODE_ENV === `test` @@ -122,10 +124,8 @@ function iterateNodes(): GatsbyIterable { return new GatsbyIterable( nodesDb .getKeys({ snapshot: false }) - .map( - nodeId => (typeof nodeId === `string` ? getNode(nodeId) : undefined)! - ) - .filter(Boolean) + .map(nodeId => (typeof nodeId === `string` ? getNode(nodeId) : undefined)) + .filter(Boolean) as ArrayLikeIterable ) } @@ -134,13 +134,22 @@ function iterateNodesByType(type: string): GatsbyIterable { return new GatsbyIterable( nodesByType .getValues(type) - .map(nodeId => getNode(nodeId)!) - .filter(Boolean) + .map(nodeId => { + if (preSyncDeletedNodeIdsCache.has(nodeId)) { + return undefined + } + + return getNode(nodeId) + }) + .filter(Boolean) as ArrayLikeIterable ) } function getNode(id: string): IGatsbyNode | undefined { - if (!id) return undefined + if (!id || preSyncDeletedNodeIdsCache.has(id)) { + return undefined + } + const { nodes } = getDatabases() return nodes.get(id) } @@ -151,9 +160,11 @@ function getTypes(): Array { function countNodes(typeName?: string): number { if (!typeName) { - const stats = getDatabases().nodes.getStats() - // @ts-ignore - return Number(stats.entryCount || 0) // FIXME: add -1 when restoring shared structures key + const stats = getDatabases().nodes.getStats() as { entryCount: number } + return Math.max( + Number(stats.entryCount) - preSyncDeletedNodeIdsCache.size, + 0 + ) // FIXME: add -1 when restoring shared structures key } const { nodesByType } = getDatabases() @@ -192,15 +203,29 @@ function updateDataStore(action: ActionsUnion): void { break } case `CREATE_NODE`: + case `DELETE_NODE`: case `ADD_FIELD_TO_NODE`: case `ADD_CHILD_NODE_TO_PARENT_NODE`: - case `DELETE_NODE`: case `MATERIALIZE_PAGE_MODE`: { const dbs = getDatabases() lastOperationPromise = Promise.all([ updateNodes(dbs.nodes, action), updateNodesByType(dbs.nodesByType, action), ]) + + // if create is used in the same transaction as delete we should remove it from cache + if (action.type === `CREATE_NODE`) { + preSyncDeletedNodeIdsCache.delete(action.payload.id) + } + + if (action.type === `DELETE_NODE`) { + preSyncDeletedNodeIdsCache.add( + ((action as IDeleteNodeAction).payload as IGatsbyNode).id + ) + lastOperationPromise.then(() => { + preSyncDeletedNodeIdsCache.clear() + }) + } } } } diff --git a/packages/gatsby/src/datastore/lmdb/updates/nodes-by-type.ts b/packages/gatsby/src/datastore/lmdb/updates/nodes-by-type.ts index 4d10f8939681c..daaaad7374d90 100644 --- a/packages/gatsby/src/datastore/lmdb/updates/nodes-by-type.ts +++ b/packages/gatsby/src/datastore/lmdb/updates/nodes-by-type.ts @@ -14,10 +14,7 @@ export function updateNodesByType( } case `DELETE_NODE`: { return action.payload - ? nodesByTypeDb.removeSync( - action.payload.internal.type, - action.payload.id - ) + ? nodesByTypeDb.remove(action.payload.internal.type, action.payload.id) : false } } diff --git a/packages/gatsby/src/datastore/lmdb/updates/nodes.ts b/packages/gatsby/src/datastore/lmdb/updates/nodes.ts index 8f19059ce3310..d2426e3959ef4 100644 --- a/packages/gatsby/src/datastore/lmdb/updates/nodes.ts +++ b/packages/gatsby/src/datastore/lmdb/updates/nodes.ts @@ -15,8 +15,9 @@ export function updateNodes( } case `DELETE_NODE`: { if (action.payload) { - return nodesDb.removeSync(action.payload.id) + return nodesDb.remove(action.payload.id) } + return false } case `MATERIALIZE_PAGE_MODE`: {