diff --git a/packages/cli/src/cmds/validator/keymanager/keystoreCache.ts b/packages/cli/src/cmds/validator/keymanager/keystoreCache.ts new file mode 100644 index 000000000000..9ba1475ab78e --- /dev/null +++ b/packages/cli/src/cmds/validator/keymanager/keystoreCache.ts @@ -0,0 +1,89 @@ +import fs from "node:fs"; +import bls from "@chainsafe/bls"; +import {Keystore} from "@chainsafe/bls-keystore"; +import {SignerLocal, SignerType} from "@lodestar/validator"; +import {fromHex, toHex} from "@lodestar/utils"; +import {PointFormat} from "@chainsafe/bls/types"; +import {lockFilepath, unlockFilepath} from "../../../util/lockfile.js"; +import {LocalKeystoreDefinition} from "./interface.js"; + +export async function loadKeystoreCache( + cacheFilepath: string, + keystoreDefinitions: LocalKeystoreDefinition[] +): Promise { + const keystores: Keystore[] = []; + const passwords: string[] = []; + for (const {keystorePath, password} of keystoreDefinitions) { + keystores.push(Keystore.parse(fs.readFileSync(keystorePath, "utf8"))); + passwords.push(password); + } + + if (keystores.length !== passwords.length) { + throw new Error( + `Number of keystores and passwords must be equal. keystores=${keystores.length}, passwords=${passwords.length}` + ); + } + + if (!fs.existsSync(cacheFilepath)) { + throw new Error(`Cache file ${cacheFilepath} does not exists.`); + } + + lockFilepath(cacheFilepath); + + const password = passwords.join(""); + // We can't use Keystore.parse as it validates the `encrypted message` to be only 32 bytes. + const keystore = new Keystore(JSON.parse(fs.readFileSync(cacheFilepath, "utf8"))); + const secretKeyConcatenatedBytes = await keystore.decrypt(password); + + const result: SignerLocal[] = []; + for (const [index, k] of keystores.entries()) { + const secretKeyBytes = Uint8Array.prototype.slice.call(secretKeyConcatenatedBytes, index * 32, (index + 1) * 32); + const secretKey = bls.SecretKey.fromBytes(secretKeyBytes); + const publicKey = secretKey.toPublicKey().toBytes(PointFormat.compressed); + + if (toHex(publicKey) !== toHex(fromHex(k.pubkey))) { + throw new Error( + `Keystore ${k.uuid} does not match the expected pubkey. expected=${toHex(fromHex(k.pubkey))}, found=${toHex( + publicKey + )}` + ); + } + + result.push({ + type: SignerType.Local, + secretKey, + }); + } + + unlockFilepath(cacheFilepath); + + return result; +} + +export async function writeKeystoreCache( + cacheFilepath: string, + signers: SignerLocal[], + passwords: string[] +): Promise { + if (signers.length !== passwords.length) { + throw new Error( + `Number of signers and passwords must be equal. signers=${signers.length}, passwords=${passwords.length}` + ); + } + const secretKeys = signers.map((s) => s.secretKey.toBytes()); + const publicKeys = signers.map((s) => s.secretKey.toPublicKey().toBytes()); + const password = passwords.join(""); + const secretKeyConcatenatedBytes = Buffer.concat(secretKeys); + const publicConcatenatedBytes = Buffer.concat(publicKeys); + const keystore = await Keystore.create(password, secretKeyConcatenatedBytes, publicConcatenatedBytes, cacheFilepath); + lockFilepath(cacheFilepath); + fs.writeFileSync(cacheFilepath, keystore.stringify()); + unlockFilepath(cacheFilepath); +} + +export async function clearKeystoreCache(cacheFilepath: string): Promise { + if (fs.existsSync(cacheFilepath)) { + unlockFilepath(cacheFilepath); + fs.unlinkSync(cacheFilepath); + } +} diff --git a/packages/cli/src/cmds/validator/keymanager/persistedKeys.ts b/packages/cli/src/cmds/validator/keymanager/persistedKeys.ts index 22fa7a305c0d..a1133d57eff5 100644 --- a/packages/cli/src/cmds/validator/keymanager/persistedKeys.ts +++ b/packages/cli/src/cmds/validator/keymanager/persistedKeys.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import bls from "@chainsafe/bls"; import {Keystore} from "@chainsafe/bls-keystore"; -import {Signer, SignerType, ProposerConfig} from "@lodestar/validator"; +import {Signer, SignerType, ProposerConfig, SignerLocal} from "@lodestar/validator"; import {DeletionStatus, ImportStatus, PubkeyHex, SignerDefinition} from "@lodestar/api/keymanager"; import { getPubkeyHexFromKeystore, @@ -14,6 +14,7 @@ import { } from "../../../util/index.js"; import {lockFilepath} from "../../../util/lockfile.js"; import {IPersistedKeysBackend} from "./interface.js"; +import {clearKeystoreCache, loadKeystoreCache, writeKeystoreCache} from "./keystoreCache.js"; export {ImportStatus, DeletionStatus}; @@ -246,11 +247,32 @@ export class PersistedKeysBackend implements IPersistedKeysBackend { } } +type KeystoreDecryptOptions = { + force?: boolean; + onDecrypt?: (index: number, signer: Signer) => void; + // Try to use the cache file if it exists + cacheFilePath?: string; +}; + export async function decryptKeystoreDefinitions( keystoreDefinitions: LocalKeystoreDefinition[], - opts: {force?: boolean; onDecrypt?: (index: number, signer: Signer) => void} + opts: KeystoreDecryptOptions ): Promise { - const signers: Signer[] = []; + if (opts.cacheFilePath) { + try { + const signers = await loadKeystoreCache(opts.cacheFilePath, keystoreDefinitions); + if (opts?.onDecrypt) { + opts?.onDecrypt(signers.length - 1, signers[signers.length - 1]); + } + return signers; + } catch { + // Some error loading the cache, ignore and invalidate cache + await clearKeystoreCache(opts.cacheFilePath); + } + } + + const signers: SignerLocal[] = []; + const passwords: string[] = []; for (const [index, {keystorePath, password}] of keystoreDefinitions.entries()) { try { @@ -281,18 +303,23 @@ export async function decryptKeystoreDefinitions( // const secretKeyBytes = await keystore.decrypt(password); - const signer: Signer = { + const signer: SignerLocal = { type: SignerType.Local, secretKey: bls.SecretKey.fromBytes(secretKeyBytes), }; signers.push(signer); + passwords.push(password); if (opts?.onDecrypt) { opts?.onDecrypt(index, signer); } } + if (opts.cacheFilePath) { + await writeKeystoreCache(opts.cacheFilePath, signers, passwords); + } + return signers; } diff --git a/packages/cli/src/cmds/validator/signers/index.ts b/packages/cli/src/cmds/validator/signers/index.ts index 658313d10ab0..9c9391ba9017 100644 --- a/packages/cli/src/cmds/validator/signers/index.ts +++ b/packages/cli/src/cmds/validator/signers/index.ts @@ -91,7 +91,11 @@ export async function getSignersFromArgs( ); }, }); - return await decryptKeystoreDefinitions(keystoreDefinitions, {...args, onDecrypt: needle}); + return decryptKeystoreDefinitions(keystoreDefinitions, { + ...args, + onDecrypt: needle, + cacheFilePath: `${args.importKeystores[0]}.cache`, + }); } // Remote keys declared manually with --externalSignerPublicKeys @@ -119,7 +123,11 @@ export async function getSignersFromArgs( ); }, }); - const keystoreSigners = await decryptKeystoreDefinitions(keystoreDefinitions, {...args, onDecrypt: needle}); + const keystoreSigners = await decryptKeystoreDefinitions(keystoreDefinitions, { + ...args, + onDecrypt: needle, + cacheFilePath: `${accountPaths.keystoresDir}.cache`, + }); // Read local remote keys, imported via keymanager api const signerDefinitions = persistedKeysBackend.readAllRemoteKeys(); diff --git a/packages/cli/src/util/progress.ts b/packages/cli/src/util/progress.ts index e284677d5a7e..52bd1117e266 100644 --- a/packages/cli/src/util/progress.ts +++ b/packages/cli/src/util/progress.ts @@ -16,6 +16,7 @@ export function showProgress({ let current = 0; let last = 0; let lastProcessTime: number = Date.now(); + let progressIntervalId: NodeJS.Timeout; const needle: NeedleFunc = (needle: number) => { // zero is considered first index in the range @@ -34,21 +35,23 @@ export function showProgress({ current, total, ratePerSec: processTime === 0 ? 0 : ((current - last) / processTime) * 1000, - percentage: (current / total) * 100, + percentage: current && total ? (current / total) * 100 : 100, }); last = current; lastProcessTime = currentTime; if (current >= total) { - clearInterval(internalId); + clearInterval(progressIntervalId); } }; - const internalId = setInterval(processProgress, frequencyMs); + if (total > 0) { + progressIntervalId = setInterval(processProgress, frequencyMs); + } signal.addEventListener("abort", () => { - clearInterval(internalId); + clearInterval(progressIntervalId); }); return needle; diff --git a/packages/cli/test/e2e/propserConfigfromKeymanager.test.ts b/packages/cli/test/e2e/propserConfigfromKeymanager.test.ts index cae2c4b1755b..a22a0e484c3a 100644 --- a/packages/cli/test/e2e/propserConfigfromKeymanager.test.ts +++ b/packages/cli/test/e2e/propserConfigfromKeymanager.test.ts @@ -10,7 +10,7 @@ import {getKeymanagerTestRunner} from "../utils/keymanagerTestRunners.js"; import {getKeystoresStr} from "../utils/keystores.js"; describeCliTest("import keystores from api, test DefaultProposerConfig", function ({spawnCli}) { - const dataDir = path.join(testFilesDir, "import-keystores-test"); + const dataDir = path.join(testFilesDir, "proposer-config-test"); const defaultOptions = { suggestedFeeRecipient: "0x0000000000000000000000000000000000000000", diff --git a/packages/cli/test/unit/cmds/validator/keymanager/keystoreCache.test.ts b/packages/cli/test/unit/cmds/validator/keymanager/keystoreCache.test.ts new file mode 100644 index 000000000000..c27d64dcaf2b --- /dev/null +++ b/packages/cli/test/unit/cmds/validator/keymanager/keystoreCache.test.ts @@ -0,0 +1,88 @@ +import fs from "node:fs"; +import {randomBytes} from "node:crypto"; +import tmp from "tmp"; +import {expect} from "chai"; +import chainAsPromised from "chai-as-promised"; +import chai from "chai"; +import {Keystore} from "@chainsafe/bls-keystore"; +import {interopSecretKey} from "@lodestar/state-transition"; +import bls from "@chainsafe/bls"; +import {SignerLocal, SignerType} from "@lodestar/validator"; +import {loadKeystoreCache, writeKeystoreCache} from "../../../../../src/cmds/validator/keymanager/keystoreCache.js"; +import {LocalKeystoreDefinition} from "../../../../../src/cmds/validator/keymanager/interface.js"; + +chai.use(chainAsPromised); + +const numberOfSigners = 10; + +describe("keystoreCache", () => { + let definitions: LocalKeystoreDefinition[]; + let signers: SignerLocal[]; + let secretKeys: Uint8Array[]; + let passwords: string[]; + let keystoreCacheFile: string; + + beforeEach(async function setup() { + this.timeout(50000); + definitions = []; + signers = []; + secretKeys = []; + passwords = []; + keystoreCacheFile = tmp.tmpNameSync({postfix: ".cache"}); + + for (let i = 0; i < numberOfSigners; i++) { + const secretKey = bls.SecretKey.fromBytes(interopSecretKey(i).toBytes()); + const keystorePath = tmp.tmpNameSync({postfix: ".json"}); + const password = secretKey.toHex(); + const keystore = await Keystore.create( + password, + secretKey.toBytes(), + secretKey.toPublicKey().toBytes(), + keystorePath, + "test-keystore", + // To make the test efficient we use a low iteration count + { + function: "pbkdf2", + params: {dklen: 32, c: 10, prf: "hmac-sha256", salt: randomBytes(32).toString("hex")}, + } + ); + fs.writeFileSync(keystorePath, keystore.stringify()); + + signers.push({type: SignerType.Local, secretKey}); + + // Use secretkey hex as password + definitions.push({password: secretKey.toHex(), keystorePath}); + passwords.push(password); + secretKeys.push(secretKey.toBytes()); + } + }); + + describe("writeKeystoreCache", () => { + it("should write a valid keystore cache file", async () => { + await expect(writeKeystoreCache(keystoreCacheFile, signers, passwords)).to.fulfilled; + expect(fs.existsSync(keystoreCacheFile)).to.be.true; + }); + + it("should throw error if password length are not same as signers", async () => { + await expect(writeKeystoreCache(keystoreCacheFile, signers, [passwords[0]])).to.rejectedWith( + `Number of signers and passwords must be equal. signers=${numberOfSigners}, passwords=1` + ); + }); + }); + + describe("loadKeystoreCache", () => { + it("should load the valid keystore cache", async () => { + await writeKeystoreCache(keystoreCacheFile, signers, passwords); + const result = await loadKeystoreCache(keystoreCacheFile, definitions); + + expect(result.map((r) => r.secretKey.toBytes())).to.eql(secretKeys); + }); + + it("should raise error for mismatch public key", async () => { + await writeKeystoreCache(keystoreCacheFile, signers, passwords); + definitions[0].keystorePath = definitions[1].keystorePath; + + await expect(loadKeystoreCache(keystoreCacheFile, definitions)).to.rejected; + }); + }); +}); diff --git a/packages/cli/test/unit/util/progress.test.ts b/packages/cli/test/unit/util/progress.test.ts index bddb75c28f78..6646152a8558 100644 --- a/packages/cli/test/unit/util/progress.test.ts +++ b/packages/cli/test/unit/util/progress.test.ts @@ -53,6 +53,26 @@ describe("progress", () => { expect(progress.secondCall.args[0]).to.eql({total, current: total, ratePerSec: 0, percentage: 100}); }); + it("should call progress with correct values directly reaches to total", () => { + const progress = sandbox.spy(); + const frequencyMs = 50; + const total = 8; + const needle = showProgress({total, signal: new AbortController().signal, frequencyMs, progress}); + needle(7); + + expect(progress).to.be.calledOnce; + expect(progress.firstCall.args[0]).to.eql({total, current: total, ratePerSec: 0, percentage: 100}); + }); + + it("should not call progress when initiated with zero total", () => { + const progress = sandbox.spy(); + const frequencyMs = 50; + const total = 0; + showProgress({total, signal: new AbortController().signal, frequencyMs, progress}); + + expect(progress).to.be.not.be.called; + }); + it("should not call progress further when abort signal is called", () => { const progress = sandbox.spy(); const frequencyMs = 50;