Skip to content

chore: consolidate classic-level usage #6795

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jeluard merged 2 commits into from
May 24, 2024
Merged

chore: consolidate classic-level usage #6795

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
82ad73f
chore: do not rely on leveldown
jeluard May 16, 2024
3812189
chore: replace level with classic-level
jeluard May 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions packages/beacon-node/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,12 +154,10 @@
},
"devDependencies": {
"@types/datastore-level": "^3.0.0",
"@types/leveldown": "^4.0.3",
"@types/qs": "^6.9.7",
"@types/tmp": "^0.2.3",
"it-drain": "^3.0.3",
"it-pair": "^2.0.6",
"leveldown": "^6.1.1",
"rewiremock": "^3.14.5",
"rimraf": "^4.4.1",
"tmp": "^0.2.1"
Expand Down
6 changes: 2 additions & 4 deletions packages/beacon-node/test/e2e/eth1/eth1ForBlockProduction.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
import {promisify} from "node:util";
import {describe, it, beforeAll, afterAll, expect} from "vitest";
import leveldown from "leveldown";
import {fromHexString, toHexString} from "@chainsafe/ssz";
import {sleep} from "@lodestar/utils";
import {LevelDbController} from "@lodestar/db";
Expand Down Expand Up @@ -38,7 +36,7 @@ describe.skip("eth1 / Eth1Provider", function () {

beforeAll(async () => {
// Nuke DB to make sure it's empty
await promisify<string>(leveldown.destroy)(dbLocation);
await LevelDbController.destroy(dbLocation);

db = new BeaconDb(config, await LevelDbController.create({name: dbLocation}, {logger}));
});
Expand All @@ -47,7 +45,7 @@ describe.skip("eth1 / Eth1Provider", function () {
clearInterval(interval);
controller.abort();
await db.close();
await promisify<string>(leveldown.destroy)(dbLocation);
await LevelDbController.destroy(dbLocation);
});

it("Should fetch real Pyrmont eth1 data for block proposing", async function () {
Expand Down
4 changes: 2 additions & 2 deletions packages/db/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@
"@chainsafe/ssz": "^0.15.1",
"@lodestar/config": "^1.18.1",
"@lodestar/utils": "^1.18.1",
"it-all": "^3.0.4",
"level": "^8.0.0"
"classic-level": "^1.4.1",

@jeluard jeluard May 16, 2024

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It might make sense to depend on classic-level dependency directly now that we are requiring it.

@wemeetagain wemeetagain May 23, 2024

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Rather than using the level package?

@jeluard jeluard May 23, 2024

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right, the code already expect running under NodeJS and we add classic-level dependency to make sure the right version is used.

@wemeetagain wemeetagain May 23, 2024

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds good to me.
Can you update this PR to include that? Currently it feels like in a halfway state (which I suppose caused your comment here).

@jeluard jeluard May 24, 2024

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@wemeetagain done

"it-all": "^3.0.4"
},
"devDependencies": {
"@lodestar/logger": "^1.18.1"
Expand Down
20 changes: 11 additions & 9 deletions packages/db/src/controller/level.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import {Level} from "level";
import type {ClassicLevel} from "classic-level";
import {ClassicLevel} from "classic-level";
import {Logger} from "@lodestar/utils";
import {DbReqOpts, DatabaseController, DatabaseOptions, FilterOptions, KeyValue} from "./interface.js";
import {LevelDbControllerMetrics} from "./metrics.js";
Expand All @@ -9,10 +8,8 @@ enum Status {
closed = "closed",
}

type LevelNodeJS = ClassicLevel<Uint8Array, Uint8Array>;

export interface LevelDBOptions extends DatabaseOptions {
db?: Level<Uint8Array, Uint8Array>;
db?: ClassicLevel<Uint8Array, Uint8Array>;
}

export type LevelDbControllerModules = {
Expand All @@ -35,7 +32,7 @@ export class LevelDbController implements DatabaseController<Uint8Array, Uint8Ar

constructor(
private readonly logger: Logger,
private readonly db: Level<Uint8Array, Uint8Array>,
private readonly db: ClassicLevel<Uint8Array, Uint8Array>,
private metrics: LevelDbControllerMetrics | null
) {
this.metrics = metrics ?? null;
Expand All @@ -46,7 +43,8 @@ export class LevelDbController implements DatabaseController<Uint8Array, Uint8Ar
}

static async create(opts: LevelDBOptions, {metrics, logger}: LevelDbControllerModules): Promise<LevelDbController> {
const db = opts.db || new Level(opts.name || "beaconchain", {keyEncoding: "binary", valueEncoding: "binary"});
const db =
opts.db || new ClassicLevel(opts.name || "beaconchain", {keyEncoding: "binary", valueEncoding: "binary"});

try {
await db.open();
Expand Down Expand Up @@ -162,14 +160,14 @@ export class LevelDbController implements DatabaseController<Uint8Array, Uint8Ar
* The result might not include recently written data.
*/
approximateSize(start: Uint8Array, end: Uint8Array): Promise<number> {
return (this.db as LevelNodeJS).approximateSize(start, end);
return this.db.approximateSize(start, end);
}

/**
* Manually trigger a database compaction in the range [start..end].
*/
compactRange(start: Uint8Array, end: Uint8Array): Promise<void> {
return (this.db as LevelNodeJS).compactRange(start, end);
return this.db.compactRange(start, end);
}

/** Capture metrics for db.iterator, db.keys, db.values .all() calls */
Expand Down Expand Up @@ -221,6 +219,10 @@ export class LevelDbController implements DatabaseController<Uint8Array, Uint8Ar
})
.finally(timer);
}

static async destroy(location: string): Promise<void> {
return ClassicLevel.destroy(location);
}
}

/** From https://www.npmjs.com/package/level */
Expand Down
8 changes: 1 addition & 7 deletions packages/db/test/unit/controller/level.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import {execSync} from "node:child_process";
import os from "node:os";
import {describe, it, expect, beforeAll, afterAll} from "vitest";
import leveldown from "leveldown";
import all from "it-all";
import {getEnvLogger} from "@lodestar/logger/env";
import {LevelDbController} from "../../../src/controller/index.js";
Expand All @@ -16,12 +15,7 @@ describe("LevelDB controller", () => {

afterAll(async () => {
await db.close();
await new Promise<void>((resolve, reject) => {
leveldown.destroy(dbLocation, (err) => {
if (err) reject(err);
else resolve();
});
});
await LevelDbController.destroy(dbLocation);
});

it("test get not found", async () => {
Expand Down
86 changes: 19 additions & 67 deletions yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2826,11 +2826,6 @@
"@tufjs/canonical-json" "1.0.0"
minimatch "^9.0.0"

"@types/abstract-leveldown@*":
version "5.0.1"
resolved "https://registry.npmjs.org/@types/abstract-leveldown/-/abstract-leveldown-5.0.1.tgz"
integrity sha512-wYxU3kp5zItbxKmeRYCEplS2MW7DzyBnxPGj+GJVHZEUZiK/nn5Ei1sUFgURDh+X051+zsGe28iud3oHjrYWQQ==

"@types/argparse@1.0.38":
version "1.0.38"
resolved "https://registry.yarnpkg.com/@types/argparse/-/argparse-1.0.38.tgz#a81fd8606d481f873a3800c6ebae4f1d768a56a9"
Expand Down Expand Up @@ -2957,14 +2952,6 @@
dependencies:
"@types/node" "*"

"@types/leveldown@^4.0.3":
version "4.0.3"
resolved "https://registry.yarnpkg.com/@types/leveldown/-/leveldown-4.0.3.tgz#4b868fd747808d378df6ffb27de7f889cae46aad"
integrity sha512-fzIXOuUCSZQKkRadWhURwu6mMYbfqi/nRDA+yBwz8kj7AK/L7L//u6A0MqSv0gsilzo7N/5+FlZOx8G6m03EVQ==
dependencies:
"@types/abstract-leveldown" "*"
"@types/node" "*"

"@types/minimatch@^3.0.3":
version "3.0.5"
resolved "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.5.tgz"
Expand Down Expand Up @@ -3627,18 +3614,6 @@ abstract-level@^1.0.0, abstract-level@^1.0.2, abstract-level@^1.0.3:
module-error "^1.0.1"
queue-microtask "^1.2.3"

abstract-leveldown@^7.2.0:
version "7.2.0"
resolved "https://registry.yarnpkg.com/abstract-leveldown/-/abstract-leveldown-7.2.0.tgz#08d19d4e26fb5be426f7a57004851b39e1795a2e"
integrity sha512-DnhQwcFEaYsvYDnACLZhMmCWd3rkOeEvglpa4q5i/5Jlm3UIsWaxVzuXvDLFCSCWRO3yy2/+V/G7FusFgejnfQ==
dependencies:
buffer "^6.0.3"
catering "^2.0.0"
is-buffer "^2.0.5"
level-concat-iterator "^3.0.0"
level-supports "^2.0.1"
queue-microtask "^1.2.3"

abstract-logging@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/abstract-logging/-/abstract-logging-2.0.1.tgz#6b0c371df212db7129b57d2e7fcf282b8bf1c839"
Expand Down Expand Up @@ -4683,7 +4658,7 @@ case@^1.6.3:
resolved "https://registry.npmjs.org/case/-/case-1.6.3.tgz"
integrity sha512-mzDSXIPaFwVDvZAHqZ9VlbyF4yyXRuX6IvB06WvPYkqJVO24kX1PPhv9bfpKNFZyxYFmmgo03HUiD8iklmJYRQ==

catering@^2.0.0, catering@^2.1.0, catering@^2.1.1:
catering@^2.1.0, catering@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/catering/-/catering-2.1.1.tgz#66acba06ed5ee28d5286133982a927de9a04b510"
integrity sha512-K7Qy8O9p76sL3/3m7/zLKbRkyOlSZAgzEaLhyj2mXS8PsCud2Eo4hAb8aLtZqHh0QGqLcb9dlJSu6lHRVENm1w==
Expand Down Expand Up @@ -4819,6 +4794,17 @@ classic-level@^1.2.0:
napi-macros "~2.0.0"
node-gyp-build "^4.3.0"

classic-level@^1.4.1:
version "1.4.1"
resolved "https://registry.yarnpkg.com/classic-level/-/classic-level-1.4.1.tgz#169ecf9f9c6200ad42a98c8576af449c1badbaee"
integrity sha512-qGx/KJl3bvtOHrGau2WklEZuXhS3zme+jf+fsu6Ej7W7IP/C49v7KNlWIsT1jZu0YnfzSIYDGcEWpCa1wKGWXQ==
dependencies:
abstract-level "^1.0.2"
catering "^2.1.0"
module-error "^1.0.1"
napi-macros "^2.2.2"
node-gyp-build "^4.3.0"

clean-stack@^2.0.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b"
Expand Down Expand Up @@ -8531,18 +8517,6 @@ lerna@^7.3.0:
yargs "16.2.0"
yargs-parser "20.2.4"

level-concat-iterator@^3.0.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/level-concat-iterator/-/level-concat-iterator-3.1.0.tgz#5235b1f744bc34847ed65a50548aa88d22e881cf"
integrity sha512-BWRCMHBxbIqPxJ8vHOvKUsaO0v1sLYZtjN3K2iZJsRBYtp+ONsY6Jfi6hy9K3+zolgQRryhIn2NRZjZnWJ9NmQ==
dependencies:
catering "^2.1.0"

level-supports@^2.0.1:
version "2.1.0"
resolved "https://registry.yarnpkg.com/level-supports/-/level-supports-2.1.0.tgz#9af908d853597ecd592293b2fad124375be79c5f"
integrity sha512-E486g1NCjW5cF78KGPrMDRBYzPuueMZ6VBXHT6gC7A8UYWGiM14fGgp+s/L1oFfDWSPV/+SFkYCmZ0SiESkRKA==

level-supports@^4.0.0:
version "4.0.1"
resolved "https://registry.yarnpkg.com/level-supports/-/level-supports-4.0.1.tgz#431546f9d81f10ff0fea0e74533a0e875c08c66a"
Expand All @@ -8564,15 +8538,6 @@ level@^8.0.0:
browser-level "^1.0.1"
classic-level "^1.2.0"

leveldown@^6.1.1:
version "6.1.1"
resolved "https://registry.yarnpkg.com/leveldown/-/leveldown-6.1.1.tgz#0f0e480fa88fd807abf94c33cb7e40966ea4b5ce"
integrity sha512-88c+E+Eizn4CkQOBHwqlCJaTNEjGpaEIikn1S+cINc5E9HEvJ77bqY4JY/HxT5u0caWqsc3P3DcFIKBI1vHt+A==
dependencies:
abstract-leveldown "^7.2.0"
napi-macros "~2.0.0"
node-gyp-build "^4.3.0"

levn@^0.4.1:
version "0.4.1"
resolved "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz"
Expand Down Expand Up @@ -9500,6 +9465,11 @@ nanoid@^3.3.7:
resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.7.tgz#d0c301a691bc8d54efa0a2226ccf3fe2fd656bd8"
integrity sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==

napi-macros@^2.2.2:
version "2.2.2"
resolved "https://registry.yarnpkg.com/napi-macros/-/napi-macros-2.2.2.tgz#817fef20c3e0e40a963fbf7b37d1600bd0201044"
integrity sha512-hmEVtAGYzVQpCKdbQea4skABsdXW4RUh5t5mJ2zzqowJS2OyXZTU1KhDVFhx+NlWZ4ap9mqR9TcDO3LTTttd+g==

napi-macros@~2.0.0:
version "2.0.0"
resolved "https://registry.npmjs.org/napi-macros/-/napi-macros-2.0.0.tgz"
Expand Down Expand Up @@ -11844,16 +11814,7 @@ string-argv@~0.3.1:
resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.2.tgz#2b6d0ef24b656274d957d54e0a4bbf6153dc02b6"
integrity sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==

"string-width-cjs@npm:string-width@^4.2.0":
version "4.2.3"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
dependencies:
emoji-regex "^8.0.0"
is-fullwidth-code-point "^3.0.0"
strip-ansi "^6.0.1"

"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
version "4.2.3"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
Expand Down Expand Up @@ -13488,16 +13449,7 @@ workerpool@6.2.1:
resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.2.1.tgz#46fc150c17d826b86a008e5a4508656777e9c343"
integrity sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==

"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
version "7.0.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
dependencies:
ansi-styles "^4.0.0"
string-width "^4.1.0"
strip-ansi "^6.0.0"

wrap-ansi@^7.0.0:
"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
Expand Down